├── .coveragerc ├── .flake8 ├── .github ├── dependabot.yml └── workflows │ └── main.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .yamllint ├── LICENSE ├── README.md ├── SECURITY.md ├── apps ├── cloud │ ├── LICENSE │ ├── README.md │ ├── odc │ │ └── apps │ │ │ └── cloud │ │ │ ├── __init__.py │ │ │ ├── _version.py │ │ │ ├── azure_to_tar.py │ │ │ ├── gs_to_tar.py │ │ │ ├── redrive_to_queue.py │ │ │ ├── s3_find.py │ │ │ ├── s3_inventory.py │ │ │ ├── s3_to_tar.py │ │ │ └── thredds_to_tar.py │ ├── pyproject.toml │ ├── setup.cfg │ └── setup.py └── dc_tools │ ├── LICENSE │ ├── README.md │ ├── assets │ └── bootstrap-odc.sh │ ├── odc │ └── apps │ │ └── dc_tools │ │ ├── __init__.py │ │ ├── _docs.py │ │ ├── _stac.py │ │ ├── _version.py │ │ ├── add_update_products.py │ │ ├── azure_to_dc.py │ │ ├── cop_dem_to_dc.py │ │ ├── esa_worldcover_to_dc.py │ │ ├── export_md.py │ │ ├── fs_to_dc.py │ │ ├── index_from_tar.py │ │ ├── s3_to_dc.py │ │ ├── sqs_to_dc.py │ │ ├── stac_api_to_dc.py │ │ ├── thredds_to_dc.py │ │ └── utils.py │ ├── pyproject.toml │ ├── setup.cfg │ ├── setup.py │ └── tests │ ├── conftest.py │ ├── data │ ├── 29V-2021.stac-item.json │ ├── 60U-2020.stac-item.json │ ├── LC08_L2SR_081119_20200101_20200823_02_T2.json │ ├── NASADEM_HGT_s56w072.stac-item.json │ ├── S2A_17SPR_20221106_0_L2A.json │ ├── S2A_28QCH_20200714_0_L2A.json │ ├── S2A_28QCH_20200714_0_L2A.odc-metadata.json │ ├── S2A_28QCH_20200714_0_L2A_old.json │ ├── S2B_31QGB_20200831_0_L2A.json │ ├── S2B_42TUM_20220107_0_L2A.json │ ├── S2B_T37MGP_20240710T073012_L2A_C1-odc-metadata.yaml │ ├── S2B_T37MGP_20240710T073012_L2A_C1.json │ ├── S2B_T37MGP_20240710T073012_L2A_C1_rel_links-odc-metadata.yaml │ ├── S2B_T37MGP_20240710T073012_L2A_C1_rel_links.json │ ├── cemp_insar │ │ ├── 10 │ │ │ ├── 10 │ │ │ │ └── alos_cumul_2010-10-10.yaml │ │ │ └── 22 │ │ │ │ └── alos_cumul_2010-10-22.yaml │ │ ├── 01 │ │ │ ├── 19 │ │ │ │ └── alos_cumul_2010-01-19.yaml │ │ │ ├── 31 │ │ │ │ └── alos_cumul_2010-01-31.yaml │ │ │ └── 07 │ │ │ │ └── alos_cumul_2010-01-07.yaml │ │ ├── 02 │ │ │ ├── 12 │ │ │ │ └── alos_cumul_2010-02-12.yaml │ │ │ └── 24 │ │ │ │ └── alos_cumul_2010-02-24.yaml │ │ ├── 03 │ │ │ ├── 20 │ │ │ │ └── alos_cumul_2010-03-20.yaml │ │ │ └── 08 │ │ │ │ └── alos_cumul_2010-03-08.yaml │ │ ├── 04 │ │ │ ├── 13 │ │ │ │ └── alos_cumul_2010-04-13.yaml │ │ │ ├── 25 │ │ │ │ └── alos_cumul_2010-04-25.yaml │ │ │ └── 01 │ │ │ │ └── alos_cumul_2010-04-01.yaml │ │ ├── 05 │ │ │ ├── 19 │ │ │ │ └── alos_cumul_2010-05-19.yaml │ │ │ ├── 31 │ │ │ │ └── alos_cumul_2010-05-31.yaml │ │ │ └── 07 │ │ │ │ └── alos_cumul_2010-05-07.yaml │ │ ├── 06 │ │ │ ├── 12 │ │ │ │ └── alos_cumul_2010-06-12.yaml │ │ │ └── 24 │ │ │ │ └── alos_cumul_2010-06-24.yaml │ │ ├── 07 │ │ │ ├── 18 │ │ │ │ └── alos_cumul_2010-07-18.yaml │ │ │ ├── 30 │ │ │ │ └── alos_cumul_2010-07-30.yaml │ │ │ └── 06 │ │ │ │ └── alos_cumul_2010-07-06.yaml │ │ ├── 08 │ │ │ ├── 11 │ │ │ │ └── alos_cumul_2010-08-11.yaml │ │ │ └── 23 │ │ │ │ └── alos_cumul_2010-08-23.yaml │ │ └── 09 │ │ │ ├── 16 │ │ │ └── alos_cumul_2010-09-16.yaml │ │ │ ├── 28 │ │ │ └── alos_cumul_2010-09-28.yaml │ │ │ └── 04 │ │ │ └── alos_cumul_2010-09-04.yaml │ ├── derivative │ │ └── ga_ls5t_nbart_gm_cyear_3 │ │ │ └── 3-0-0 │ │ │ └── x08 │ │ │ └── y23 │ │ │ └── 1994--P1Y │ │ │ ├── ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final.odc-metadata.yaml │ │ │ └── ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final.proc-info.yaml │ ├── eo3_sentinel_ard.odc-type.yaml │ ├── esa_worldcover │ │ ├── ESA_WorldCover_10m_2020_v100_N03E003_Map.tif │ │ ├── ESA_WorldCover_10m_2020_v100_N03E006_Map.tif │ │ ├── ESA_WorldCover_10m_2020_v100_N06E003_Map.tif │ │ └── ESA_WorldCover_10m_2020_v100_N06E006_Map.tif │ ├── example_product_list.csv │ ├── ga_ls5t_gm_product.yaml │ ├── ga_ls5t_nbart_gm_cyear_3_x30y14_1999--P1Y_final.stac-item.json │ ├── ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.sqs.yaml │ ├── ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.yaml │ ├── ga_ls8c_ard_3-1-0_088080_2020-05-25_final.stac-item.json │ ├── ga_ls8c_ard_3-1-0_088080_2020-05-25_final.stac-item.sqs.json │ ├── ga_ls8c_ard_3-1-1_088080_2020-05-25_final.stac-item.json │ ├── ga_s2am_ard_3-2-1_49JFM_2016-12-14_final.odc-metadata.yaml │ ├── ga_s2am_ard_3-2-1_49JFM_2016-12-14_final.stac-item.json │ ├── lidar_dem.json │ ├── maturity-final.odc-metadata.yaml │ ├── maturity-nrt.odc-metadata.yaml │ └── world-wrapping.stac-item.json │ ├── test_add_update_products.py │ ├── test_cop_dem_to_dc.py │ ├── test_esa_worldcover_to_dc.py │ ├── test_fs_to_dc.py │ ├── test_s3_to_dc.py │ ├── test_sns_publishing.py │ ├── test_sqs_to_dc.py │ ├── test_stac_api_to_dc.py │ └── test_stac_transform.py ├── codecov.yml ├── docker ├── .dockerignore ├── .gitignore ├── Dockerfile ├── Makefile ├── assets │ ├── with-bootstrap │ └── with-test-db ├── constraints.txt ├── nobinary.txt ├── readme.md └── requirements.in ├── docs ├── change-dataset-metadata.md ├── db-core-interface.md ├── normalised-dataset.yaml └── s3-tar.md ├── libs ├── cloud │ ├── LICENSE │ ├── README.md │ ├── odc │ │ ├── aio.py │ │ ├── aws │ │ │ ├── __init__.py │ │ │ ├── _find.py │ │ │ ├── dns.py │ │ │ ├── inventory.py │ │ │ ├── misc.py │ │ │ ├── queue.py │ │ │ └── s3_client.py │ │ ├── azure.py │ │ ├── cloud │ │ │ ├── __init__.py │ │ │ └── _version.py │ │ ├── ppt.py │ │ └── thredds.py │ ├── pyproject.toml │ ├── setup.cfg │ ├── setup.py │ └── tests │ │ ├── test_aws.py │ │ ├── test_azure.py │ │ └── test_thredds.py ├── io │ ├── LICENSE │ ├── README.md │ ├── odc │ │ └── io │ │ │ ├── __init__.py │ │ │ ├── _version.py │ │ │ ├── cgroups.py │ │ │ ├── tar.py │ │ │ ├── text.py │ │ │ └── timer.py │ ├── pyproject.toml │ ├── setup.cfg │ ├── setup.py │ └── tests │ │ └── test_text.py └── ui │ ├── LICENSE │ ├── README.md │ ├── odc │ └── ui │ │ ├── __init__.py │ │ ├── _cmaps.py │ │ ├── _dc_explore.py │ │ ├── _images.py │ │ ├── _map.py │ │ ├── _ui.py │ │ ├── _version.py │ │ └── plt_tools.py │ ├── pyproject.toml │ ├── setup.cfg │ └── setup.py ├── mypy.ini ├── notebooks ├── README.md ├── dask-dc-load.ipynb ├── dask-stream-processing.ipynb ├── dc-explore.ipynb ├── dc-load-progress-bar.ipynb ├── deterministic-uuid.ipynb ├── dscache-example.ipynb ├── geomedian-example.ipynb ├── map-selector-example.ipynb ├── pixel-drill-example.ipynb ├── s3-fetch-example.ipynb ├── s3-inventory-example.ipynb ├── show-image-on-a-map-nci.ipynb └── show-image-on-a-map.ipynb ├── pylintrc ├── pyproject.toml ├── scripts ├── bootstrap-env.sh ├── build-wheels.sh ├── dev-install.sh ├── mk-pip-tree.sh ├── patch_version.py ├── setup-test-db.sh └── sync-publish-branch.sh └── tests └── test-env.yml /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = *tests* 3 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | ignore = E731, E226, E203, W503 4 | # Damien: I want to set max-line-length to 88, as recommended by black 5 | # but black isn't reformatting comments, or seemingly even strings, to 6 | # keep them within this length. 7 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "github-actions" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | - package-ecosystem: "docker" 13 | directory: "/docker" 14 | schedule: 15 | interval: "daily" 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pickle 2 | /test_env 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | .mypy_cache/ 7 | dask-worker-space/ 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *,cover 49 | .hypothesis 50 | .pytest_cache 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | 59 | # Sphinx documentation 60 | docs/_build/ 61 | 62 | # VIM swap files 63 | .*.sw? 64 | 65 | # PyBuilder 66 | target/ 67 | .idea/ 68 | 69 | # iPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # Mac OS X 73 | .DS_Store 74 | docs/html/ 75 | 76 | # Generated Documentation 77 | generate/ 78 | docs/notebooks/ 79 | 80 | #Local Visual Studio Code configurations 81 | .vscode/ 82 | 83 | # used to cache dev install cache 84 | .run/ 85 | 86 | docker/wheels/ 87 | 88 | .jj 89 | .ropeproject 90 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | repos: 3 | - repo: https://github.com/adrienverge/yamllint 4 | rev: v1.37.0 5 | hooks: 6 | - id: yamllint 7 | args: ['-c', '.yamllint'] 8 | - repo: https://github.com/pre-commit/pre-commit-hooks 9 | rev: v5.0.0 10 | hooks: 11 | - id: end-of-file-fixer 12 | - id: check-docstring-first 13 | - id: check-json 14 | - id: check-yaml 15 | - id: debug-statements 16 | - id: name-tests-test 17 | args: ['--django'] 18 | - id: requirements-txt-fixer 19 | - id: check-added-large-files 20 | - id: check-merge-conflict 21 | # isort broke 'libs/ui' by reordering imports 22 | # - repo: https://github.com/pycqa/isort 23 | # rev: 5.10.1 24 | # hooks: 25 | # - id: isort 26 | # name: isort (python) 27 | # args: [ "--profile", "black", "--filter-files" ] 28 | - repo: https://github.com/psf/black 29 | rev: 25.1.0 30 | hooks: 31 | - id: black 32 | - repo: https://github.com/PyCQA/pylint 33 | rev: 'v3.3.6' # Use the sha / tag you want to point at 34 | hooks: 35 | - id: pylint 36 | - repo: https://github.com/PyCQA/flake8 37 | rev: '7.1.2' 38 | hooks: 39 | - id: flake8 40 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | 4 | rules: 5 | # Property-indent errors are very noisy as tools all make different choices. 6 | # Until a good yaml auto-indenter appears, we'll relax this rule. 7 | indentation: disable 8 | comments: disable 9 | comments-indentation: disable 10 | document-start: disable 11 | 12 | # URLs and base64 strings break this a lot, and multi-line URLs aren't 13 | # clearer. Not worth it. 14 | line-length: disable 15 | 16 | # Github Actions config uses the word 'on', but not as a truthy-value. 17 | truthy: 18 | ignore: | 19 | .github/workflows/*.yml 20 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | Please report security issues to `Kirill888@gmail.com` 6 | -------------------------------------------------------------------------------- /apps/cloud/README.md: -------------------------------------------------------------------------------- 1 | odc.apps.cloud 2 | ============== 3 | 4 | Command line utilities for working with cloud "objects/files" 5 | 6 | 7 | Installation 8 | ------------ 9 | 10 | ``` 11 | pip install odc-apps-cloud 12 | ``` 13 | 14 | Usage 15 | ----- 16 | 17 | TODO 18 | -------------------------------------------------------------------------------- /apps/cloud/odc/apps/cloud/__init__.py: -------------------------------------------------------------------------------- 1 | from ._version import __version__ # noqa: F401 2 | -------------------------------------------------------------------------------- /apps/cloud/odc/apps/cloud/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.2.3" 2 | -------------------------------------------------------------------------------- /apps/cloud/odc/apps/cloud/azure_to_tar.py: -------------------------------------------------------------------------------- 1 | import click 2 | import tarfile 3 | from odc.azure import download_yamls, find_blobs 4 | from odc.io.tar import add_txt_file, tar_mode 5 | 6 | 7 | @click.command("azure-to-tar") 8 | @click.argument("account_url", type=str, nargs=1) 9 | @click.argument("container_name", type=str, nargs=1) 10 | @click.argument("credential", type=str, nargs=1) 11 | @click.argument("prefix", type=str, nargs=1) 12 | @click.argument("suffix", type=str, nargs=1) 13 | @click.option( 14 | "--workers", "-w", type=int, default=32, help="Number of threads to download blobs" 15 | ) 16 | @click.option( 17 | "--outfile", type=str, default="metadata.tar.gz", help="Sets the output file name" 18 | ) 19 | def cli( 20 | account_url: str, 21 | container_name: str, 22 | credential: str, 23 | prefix: str, 24 | suffix: str, 25 | workers: int, 26 | outfile: str, 27 | ) -> None: 28 | print(f"Opening AZ Container {container_name} on {account_url}") 29 | print(f"Searching on prefix '{prefix}' for files matching suffix '{suffix}'") 30 | yaml_urls = find_blobs(container_name, credential, prefix, suffix, account_url) 31 | 32 | print(f"Found {len(yaml_urls)} datasets") 33 | yamls = download_yamls(account_url, container_name, credential, yaml_urls, workers) 34 | 35 | url_prefix = (account_url + "/" + container_name + "/")[len("https://") :] 36 | 37 | # jam it all in a tar 38 | tar_opts = { 39 | "name": outfile, 40 | "mode": "w" + tar_mode(gzip=True, xz=True, is_pipe=False), 41 | } 42 | with tarfile.open(**tar_opts) as tar: 43 | for yaml in yamls: 44 | add_txt_file(tar=tar, content=yaml[0], fname=url_prefix + yaml[1]) 45 | 46 | print("Done!") 47 | 48 | 49 | if __name__ == "__main__": 50 | cli() 51 | -------------------------------------------------------------------------------- /apps/cloud/odc/apps/cloud/gs_to_tar.py: -------------------------------------------------------------------------------- 1 | import click 2 | import os 3 | import shutil 4 | import tarfile 5 | from google.cloud import storage 6 | 7 | 8 | @click.command("gs-to-tar") 9 | @click.option( 10 | "--bucket", type=str, help="The Google Storage bucket, without gs:// prefix" 11 | ) 12 | @click.option( 13 | "--prefix", 14 | type=str, 15 | help="The filepath to search for yaml files, i.e. geomedian/v2.1.0/L5", 16 | ) 17 | @click.option( 18 | "--suffix", type=str, default=".yaml", help="The suffix of your metadata files" 19 | ) 20 | @click.option( 21 | "--outfile", type=str, default="metadata.tar.gz", help="Sets the output file name" 22 | ) 23 | def cli(bucket, prefix, suffix, outfile) -> None: 24 | """Download Metadata from GS bucket to tarball 25 | 26 | Example: 27 | 28 | \b 29 | Download files in directory that match `*yaml` and store them as a tar 30 | > gs-to-tar --bucket data.deadev.com --prefix mangrove_cover 31 | 32 | """ 33 | 34 | # Connect to bucket 35 | client = storage.Client.create_anonymous_client() 36 | bucket_object = client.bucket(bucket, user_project=None) 37 | 38 | # Get list of files under prefix 39 | print( 40 | "Searching {bucket}/{prefix} for files".format( 41 | bucket=bucket_object, prefix=prefix 42 | ) 43 | ) 44 | blobs = bucket_object.list_blobs(prefix=prefix) 45 | 46 | # Filter on suffix 47 | files = [blob for blob in blobs if blob.name.endswith(suffix)] 48 | file_count = str(len(files)) 49 | print("Found {0} metadata files".format(file_count)) 50 | 51 | # Download Files to tar with an updating counter 52 | file_num = 1 53 | with tarfile.open(outfile, "w:gz") as tar: 54 | for yaml in files: 55 | count = str(file_num) 56 | filename = "{bucket}/{filepath}".format(bucket=bucket, filepath=yaml.name) 57 | 58 | # ensure dir exists 59 | if not os.path.exists(os.path.dirname("./" + filename)): 60 | os.makedirs(os.path.dirname("./" + filename)) 61 | # download to tar 62 | yaml.download_to_filename(filename=filename, client=client) 63 | tar.add(filename) 64 | os.remove("./" + filename) 65 | 66 | # counter 67 | print( 68 | "{count}/{file_count} Downloaded".format( 69 | count=count, file_count=file_count 70 | ) 71 | ) 72 | file_num += 1 73 | 74 | # Deletes the directory recursively 75 | shutil.rmtree("./" + bucket) 76 | 77 | print("Done!") 78 | 79 | 80 | if __name__ == "__main__": 81 | cli() 82 | -------------------------------------------------------------------------------- /apps/cloud/odc/apps/cloud/redrive_to_queue.py: -------------------------------------------------------------------------------- 1 | import click 2 | import logging 3 | import sys 4 | from odc.aws.queue import redrive_queue 5 | 6 | 7 | @click.command("redrive-to-queue") 8 | @click.argument("queue", required=True) 9 | @click.argument("to-queue", required=False) 10 | @click.option( 11 | "--limit", 12 | "-l", 13 | help="Limit the number of messages to transfer.", 14 | default=None, 15 | ) 16 | @click.option( 17 | "--dryrun", is_flag=True, default=False, help="Don't actually do real work" 18 | ) 19 | def cli(queue, to_queue, limit, dryrun): 20 | """ 21 | Redrives all the messages from the given sqs queue to the destination 22 | """ 23 | 24 | logging.basicConfig( 25 | level=logging.INFO, 26 | format="[%(asctime)s] %(levelname)s - %(message)s", 27 | stream=sys.stdout, 28 | ) 29 | 30 | _log = logging.getLogger(__name__) 31 | 32 | if limit is not None: 33 | try: 34 | limit = int(limit) 35 | except ValueError: 36 | raise ValueError(f"Limit {limit} is not valid") 37 | 38 | if limit < 1: 39 | raise ValueError(f"Limit {limit} is not valid.") 40 | 41 | count = redrive_queue(queue, to_queue, limit, dryrun, max_wait=1) 42 | 43 | if count == 0: 44 | _log.info("No messages to redrive") 45 | return 46 | 47 | if not dryrun: 48 | _log.info("Completed sending %s messages to the queue", count) 49 | else: 50 | _log.warning( 51 | "DRYRUN enabled, would have pushed approx %s messages to the queue", count 52 | ) 53 | 54 | 55 | if __name__ == "__main__": 56 | cli() 57 | -------------------------------------------------------------------------------- /apps/cloud/odc/apps/cloud/s3_find.py: -------------------------------------------------------------------------------- 1 | import click 2 | import sys 3 | from odc.aio import S3Fetcher, s3_find_glob 4 | 5 | 6 | @click.command("s3-find") 7 | @click.option( 8 | "--skip-check", 9 | is_flag=True, 10 | help="Assume file exists when listing exact file rather than wildcard.", 11 | ) 12 | @click.option("--no-sign-request", is_flag=True, help="Do not sign AWS S3 requests") 13 | @click.option( 14 | "--request-payer", 15 | is_flag=True, 16 | help="Needed when accessing requester pays public buckets", 17 | ) 18 | @click.argument("uri", type=str, nargs=1) 19 | def cli(uri, skip_check, no_sign_request=None, request_payer=False) -> None: 20 | """List files on S3 bucket. 21 | 22 | Example: 23 | 24 | \b 25 | List files in directory that match `*yaml` 26 | > s3-find 's3://mybucket/some/path/*yaml' 27 | 28 | \b 29 | List files in directory and all sub-directories that match `*yaml` 30 | > s3-find 's3://mybucket/some/path/**/*yaml' 31 | 32 | \b 33 | List files that match `*yaml` 2 levels deep from known path 34 | > s3-find 's3://mybucket/some/path/*/*/*yaml' 35 | 36 | \b 37 | List directories 2 levels deep from known path 38 | > s3-find 's3://mybucket/some/path/*/*/' 39 | 40 | \b 41 | List all files named `metadata.yaml` 2 directories deep 42 | > s3-find 's3://mybucket/some/path/*/*/metadata.yaml' 43 | """ 44 | flush_freq = 100 45 | 46 | opts = {} 47 | if request_payer: 48 | opts["RequestPayer"] = "requester" 49 | 50 | s3 = S3Fetcher(aws_unsigned=no_sign_request) 51 | 52 | try: 53 | stream = s3_find_glob(uri, skip_check=skip_check, s3=s3, **opts) 54 | for i, o in enumerate(stream): 55 | print( 56 | o.url, flush=(i % flush_freq == 0) 57 | ) # pylint:disable=superfluous-parens 58 | except ValueError as ve: 59 | click.echo(str(ve), err=True) 60 | sys.exit(1) 61 | except Exception as e: # pylint:disable=broad-except 62 | click.echo(str(e), err=True) 63 | sys.exit(1) 64 | 65 | 66 | if __name__ == "__main__": 67 | cli() 68 | -------------------------------------------------------------------------------- /apps/cloud/odc/apps/cloud/s3_inventory.py: -------------------------------------------------------------------------------- 1 | import click 2 | import re 3 | import sys 4 | from fnmatch import fnmatch 5 | from odc.aws import s3_client 6 | from odc.aws.inventory import list_inventory 7 | 8 | 9 | def build_predicate(glob=None, regex=None, prefix=None): 10 | def match_prefix(entry): 11 | return entry.Key.startswith(prefix) 12 | 13 | def match_regex(entry): 14 | return bool(regex.match(entry.Key)) 15 | 16 | def match_glob(entry): 17 | return fnmatch(entry.Key, glob) 18 | 19 | preds = [] 20 | 21 | if prefix: 22 | prefix = prefix.lstrip("/") 23 | preds.append(match_prefix) 24 | 25 | if regex is not None: 26 | regex = re.compile(regex) 27 | preds.append(match_regex) 28 | 29 | if glob is not None: 30 | preds.append(match_glob) 31 | 32 | if len(preds) == 0: 33 | return lambda x: True 34 | elif len(preds) == 1: 35 | return preds[0] 36 | elif len(preds) == 2: 37 | p1, p2 = preds # pylint:disable=unbalanced-tuple-unpacking 38 | return lambda e: p1(e) and p2(e) 39 | else: 40 | raise ValueError("regex and glob are mutually exclusive") 41 | 42 | 43 | @click.command("s3-inventory-dump") 44 | @click.option( 45 | "--inventory", "-i", type=str, help="URL pointing to manifest.json or one level up" 46 | ) 47 | @click.option( 48 | "--prefix", type=str, help="Only print entries with Key starting with `prefix`" 49 | ) 50 | @click.option("--regex", type=str, help="Only print entries matching regex") 51 | @click.option("--aws-profile", type=str, help="Use non-default aws profile") 52 | @click.option("--no-sign-request", is_flag=True, help="Do not sign AWS S3 requests") 53 | @click.option( 54 | "--request-payer", 55 | is_flag=True, 56 | help="Needed when accessing requester pays public buckets", 57 | ) 58 | @click.argument("glob", type=str, default="", nargs=1) 59 | def cli( 60 | inventory, 61 | prefix, 62 | regex, 63 | glob, 64 | aws_profile, 65 | no_sign_request=None, 66 | request_payer=False, 67 | ) -> None: 68 | """List S3 inventory entries. 69 | 70 | prefix can be combined with regex or glob pattern, but supplying both 71 | regex and glob doesn't make sense. 72 | 73 | \b 74 | Example: 75 | s3-inventory s3://my-inventory-bucket/path-to-inventory/ '*yaml' 76 | 77 | """ 78 | 79 | def entry_to_url(entry) -> str: 80 | return "s3://{e.Bucket}/{e.Key}".format(e=entry) 81 | 82 | opts = {} 83 | if request_payer: 84 | opts["RequestPayer"] = "requester" 85 | 86 | flush_freq = 100 87 | s3 = s3_client(profile=aws_profile, aws_unsigned=no_sign_request) 88 | 89 | if glob == "": 90 | glob = None 91 | 92 | if glob is not None and regex is not None: 93 | click.echo("Can not mix regex and shell patterns") 94 | sys.exit(1) 95 | 96 | if inventory is None: 97 | # TODO: read from config file 98 | inventory = "s3://dea-public-data-inventory/dea-public-data/dea-public-data-csv-inventory/" 99 | 100 | predicate = build_predicate(glob=glob, regex=regex, prefix=prefix) 101 | 102 | to_str = entry_to_url 103 | 104 | for i, entry in enumerate(list_inventory(inventory, s3=s3, **opts)): 105 | if predicate(entry): 106 | print(to_str(entry), flush=(i % flush_freq) == 0) 107 | 108 | 109 | if __name__ == "__main__": 110 | cli() 111 | -------------------------------------------------------------------------------- /apps/cloud/odc/apps/cloud/s3_to_tar.py: -------------------------------------------------------------------------------- 1 | import click 2 | import logging 3 | import signal 4 | import sys 5 | import tarfile 6 | from odc.aio import S3Fetcher 7 | from odc.io import read_stdin_lines 8 | from odc.io.tar import add_txt_file, tar_mode 9 | from odc.io.timer import RateEstimator 10 | from sys import stderr, stdout 11 | 12 | 13 | @click.command("s3-to-tar") 14 | @click.option("-n", type=int, help="Number of concurrent async connections to S3") 15 | @click.option("--verbose", "-v", is_flag=True, help="Be verbose") 16 | @click.option("--gzip", is_flag=True, help="Compress with gzip") 17 | @click.option("--xz", is_flag=True, help="Compress with xz") 18 | @click.option("--no-sign-request", is_flag=True, help="Do not sign AWS S3 requests") 19 | @click.option( 20 | "--request-payer", 21 | is_flag=True, 22 | help="Needed when accessing requester pays public buckets", 23 | ) 24 | @click.argument("outfile", type=str, nargs=1, default="-") 25 | def cli( 26 | n, verbose, gzip, xz, outfile, no_sign_request=None, request_payer=False 27 | ) -> None: 28 | """Fetch a bunch of s3 files into a tar archive. 29 | 30 | \b 31 | For every non-empty line in stdin 32 | - Treat line as a URI and fetch document from it 33 | - Write content of the file to a tar archive using `bucket-name/path/to/file` as file name 34 | """ 35 | 36 | opts = {} 37 | if request_payer: 38 | opts["RequestPayer"] = "requester" 39 | 40 | logging.basicConfig( 41 | format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s", 42 | level=logging.ERROR, 43 | ) 44 | 45 | nconnections = 24 if n is None else n 46 | exit_early = False 47 | 48 | def dump_to_tar(data_stream, tar) -> None: 49 | nonlocal exit_early 50 | fps = RateEstimator() 51 | 52 | for d in data_stream: 53 | fps() 54 | fname = d.url[5:] 55 | 56 | if d.data is not None: 57 | if verbose: 58 | if fps.every(10): 59 | print(".", file=stderr, end="", flush=True) 60 | 61 | if fps.every(100): 62 | print(" {}".format(str(fps)), file=stderr) 63 | 64 | add_txt_file(tar, fname, d.data, last_modified=d.last_modified) 65 | else: 66 | print("Failed %s (%s)" % (d.url, str(d.error)), file=stderr) 67 | 68 | if exit_early: 69 | break 70 | 71 | if verbose: 72 | print(" {}".format(str(fps)), file=stderr) 73 | 74 | fetcher = S3Fetcher(nconcurrent=nconnections, aws_unsigned=no_sign_request) 75 | is_pipe = outfile == "-" 76 | tar_opts = {"mode": "w" + tar_mode(gzip=gzip, xz=xz, is_pipe=is_pipe)} 77 | if is_pipe: 78 | if stdout.isatty(): 79 | click.echo("Will not write to a terminal", err=True) 80 | sys.exit(1) 81 | # TODO: on windows switch stdout to binary mode 82 | tar_opts["fileobj"] = stdout.buffer 83 | else: 84 | tar_opts["name"] = outfile 85 | 86 | urls = read_stdin_lines(skip_empty=True) 87 | 88 | def on_ctrlc(sig, frame) -> None: 89 | nonlocal exit_early 90 | print("Shutting down...", file=sys.stderr) 91 | exit_early = True 92 | 93 | signal.signal(signal.SIGINT, on_ctrlc) 94 | 95 | with tarfile.open(**tar_opts) as tar: 96 | dump_to_tar(fetcher(urls, **opts), tar) 97 | 98 | fetcher.close() 99 | 100 | 101 | if __name__ == "__main__": 102 | cli() 103 | -------------------------------------------------------------------------------- /apps/cloud/odc/apps/cloud/thredds_to_tar.py: -------------------------------------------------------------------------------- 1 | import click 2 | import tarfile 3 | from odc.io.tar import add_txt_file, tar_mode 4 | from odc.thredds import download_yamls, thredds_find_glob 5 | 6 | 7 | @click.command("thredds-to-tar") 8 | @click.option( 9 | "--thredds_catalogue", 10 | "-c", 11 | type=str, 12 | required=True, 13 | help="The THREDDS catalogue endpoint", 14 | ) 15 | @click.option( 16 | "--skips", 17 | "-s", 18 | type=str, 19 | multiple=True, 20 | help="Pattern to ignore when THREDDS crawling", 21 | ) 22 | @click.option( 23 | "--select", 24 | "-t", 25 | type=str, 26 | required=True, 27 | help="Target file pattern to match for yaml", 28 | ) 29 | @click.option( 30 | "--workers", 31 | "-w", 32 | type=int, 33 | default=4, 34 | help="Number of thredds crawler workers to use", 35 | ) 36 | @click.option( 37 | "--outfile", type=str, default="metadata.tar.gz", help="Sets the output file name" 38 | ) 39 | def cli(thredds_catalogue, skips, select, workers, outfile) -> None: 40 | """Download Metadata from THREDDS server to tarball 41 | 42 | Example: 43 | 44 | \b 45 | Download files in directory that match `*yaml` and store them as a tar 46 | > thredds-to-tar -c "http://dapds00.nci.org.au/thredds/catalog/if87/2018-11-29/" 47 | -t ".*ARD-METADATA.yaml" -s '.*NBAR.*' -s '.*SUPPLEMENTARY.*' 48 | -s '.*NBART.*' -s '.*/QA/.*' -w 8 --outfile 2018-11-29.tar.gz 49 | 50 | """ 51 | print( 52 | "Searching {thredds_catalogue} for matching files".format( 53 | thredds_catalogue=thredds_catalogue 54 | ) 55 | ) 56 | urls = thredds_find_glob(thredds_catalogue, skips, [select], workers) 57 | 58 | print("Found {0} metadata urls".format(str(len(urls)))) 59 | 60 | yamls = download_yamls(urls, workers) 61 | 62 | # jam it all in a tar 63 | tar_opts = { 64 | "name": outfile, 65 | "mode": "w" + tar_mode(gzip=True, xz=True, is_pipe=False), 66 | } 67 | with tarfile.open(**tar_opts) as tar: 68 | for yaml in yamls: 69 | add_txt_file(tar=tar, content=yaml[0], fname=yaml[1]) 70 | 71 | print("Done!") 72 | 73 | 74 | if __name__ == "__main__": 75 | cli() 76 | -------------------------------------------------------------------------------- /apps/cloud/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=51.0.0", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.pytest.ini_options] 6 | filterwarnings = [ 7 | "ignore:datetime.datetime.utcnow*:DeprecationWarning:botocore.*" 8 | ] 9 | -------------------------------------------------------------------------------- /apps/cloud/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = odc-apps-cloud 3 | description = CLI utils for working with objects/files in the cloud 4 | version = attr: odc.apps.cloud.__version__ 5 | author = Open Data Cube 6 | author_email = 7 | maintainer = Open Data Cube 8 | maintainer_email = 9 | long_description_content_type = text/markdown 10 | long_description = file: README.md 11 | platforms = any 12 | license = Apache License 2.0 13 | url = https://github.com/opendatacube/odc-tools/ 14 | 15 | [options] 16 | include_package_data = true 17 | zip_safe = false 18 | packages = find_namespace: 19 | python_requires = >=3.9 20 | tests_require = pytest 21 | install_requires = 22 | odc-cloud[ASYNC] 23 | odc-io 24 | click 25 | 26 | [options.extras_require] 27 | THREDDS = odc-cloud[THREDDS] 28 | AZURE = odc-cloud[AZURE] 29 | GCP = google-cloud-storage 30 | 31 | [options.entry_points] 32 | console_scripts = 33 | thredds-to-tar = odc.apps.cloud.thredds_to_tar:cli [THREDDS] 34 | gs-to-tar = odc.apps.cloud.gs_to_tar:cli [GCP] 35 | s3-find = odc.apps.cloud.s3_find:cli 36 | s3-inventory-dump = odc.apps.cloud.s3_inventory:cli 37 | s3-to-tar = odc.apps.cloud.s3_to_tar:cli 38 | redrive-queue = odc.apps.cloud.redrive_to_queue:cli 39 | azure-to-tar = odc.apps.cloud.azure_to_tar:cli [AZURE] 40 | 41 | 42 | [options.packages.find] 43 | include = 44 | odc* 45 | -------------------------------------------------------------------------------- /apps/cloud/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /apps/dc_tools/assets/bootstrap-odc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################ 3 | # This script allows creation of all metadata/products # 4 | # in an ODC instance from a given CSV catalog definition # 5 | # In local development activate conda environment # 6 | # In kubernetes pod executor run from # 7 | # container # 8 | ############################################################ 9 | set -o errexit 10 | set -o pipefail 11 | set -o nounset 12 | 13 | product_catalog=$1 14 | metadata_catalog=$2 15 | 16 | echo "BOOTSTRAP: Initialising DB." 17 | datacube system init --no-default-types --no-init-users 18 | # Created using : datacube metadata list | awk '{print $1}' | xargs datacube metadata show 19 | echo "BOOTSTRAP: Adding metadata types." 20 | datacube metadata add "$metadata_catalog" 21 | echo "BOOTSTRAP: Adding products." 22 | python -m odc.apps.dc_tools.add_update_products "$product_catalog" 23 | -------------------------------------------------------------------------------- /apps/dc_tools/odc/apps/dc_tools/__init__.py: -------------------------------------------------------------------------------- 1 | from ._version import __version__ 2 | 3 | __all__ = ["__version__"] 4 | -------------------------------------------------------------------------------- /apps/dc_tools/odc/apps/dc_tools/_docs.py: -------------------------------------------------------------------------------- 1 | """These should probably be in datacube library.""" 2 | 3 | import json 4 | import sys 5 | from typing import Sequence, Union 6 | from uuid import UUID, uuid5 7 | 8 | from datacube.index.hl import Doc2Dataset 9 | from datacube.utils.documents import parse_yaml 10 | 11 | # Some random UUID to be ODC namespace 12 | ODC_NS = UUID("6f34c6f4-13d6-43c0-8e4e-42b6c13203af") 13 | 14 | 15 | def odc_uuid( 16 | algorithm: str, 17 | algorithm_version: str, 18 | sources: Sequence[Union[UUID, str]], 19 | deployment_id: str = "", 20 | **other_tags, 21 | ) -> UUID: 22 | """ 23 | Generate deterministic UUID for a derived Dataset. 24 | 25 | :param algorithm: Name of the algorithm 26 | :param algorithm_version: Version string of the algorithm 27 | :param sources: Sequence of input Dataset UUIDs 28 | :param deployment_id: Some sort of identifier for installation that performs 29 | the run, for example Docker image hash, or dea module version on NCI. 30 | :param **other_tags: Any other identifiers necessary to uniquely identify dataset 31 | """ 32 | tags = [f"{k}={str(v)}" for k, v in other_tags.items()] 33 | 34 | stringified_sources = ( 35 | [str(algorithm), str(algorithm_version), str(deployment_id)] 36 | + sorted(tags) 37 | + [str(u) for u in sorted(sources)] 38 | ) 39 | 40 | srcs_hashes = "\n".join(s.lower() for s in stringified_sources) 41 | return uuid5(ODC_NS, srcs_hashes) 42 | 43 | 44 | def from_metadata_stream(metadata_stream, index, **kwargs): 45 | """ 46 | Raw metadata to Dataset stream converter. 47 | 48 | Given a stream of (uri, metadata) tuples convert them into Datasets, using 49 | supplied index and options for Doc2Dataset. 50 | 51 | 52 | **kwargs**: 53 | - skip_lineage 54 | - verify_lineage 55 | - fail_on_missing_lineage 56 | - products 57 | - exclude_products 58 | 59 | returns a sequence of tuples where each tuple is either 60 | 61 | (Dataset, None) or (None, error_message) 62 | """ 63 | doc2ds = Doc2Dataset(index, **kwargs) 64 | 65 | for uri, metadata in metadata_stream: 66 | if metadata is None: 67 | yield (None, f"Error: empty doc {uri}") 68 | else: 69 | ds, err = doc2ds(metadata, uri) 70 | if ds is not None: 71 | yield (ds, None) 72 | else: 73 | yield (None, f"Error: {uri}, {err}") 74 | 75 | 76 | def parse_doc_stream(doc_stream, on_error=None, transform=None): 77 | """ 78 | Replace doc bytes/strings with parsed dicts. 79 | 80 | Stream[(uri, bytes)] -> Stream[(uri, dict)] 81 | 82 | 83 | :param doc_stream: sequence of (uri, doc: bytes|string) 84 | :param on_error: Callback uri, doc -> None 85 | :param transform: dict -> dict if supplied also apply further transform on parsed document 86 | 87 | On output doc is replaced with python dict parsed from yaml, or with None 88 | if parsing/transform error occurred. 89 | """ 90 | for uri, doc in doc_stream: 91 | try: 92 | if uri.endswith(".json"): 93 | metadata = json.loads(doc) 94 | else: 95 | metadata = parse_yaml(doc) 96 | 97 | if transform is not None: 98 | metadata = transform(metadata) 99 | except Exception: # pylint: disable=broad-except 100 | if on_error is not None: 101 | on_error(uri, doc) 102 | metadata = None 103 | 104 | yield uri, metadata 105 | 106 | 107 | def from_yaml_doc_stream(doc_stream, index, logger=None, transform=None, **kwargs): 108 | """ 109 | Stream of yaml documents to a stream of Dataset results. 110 | 111 | Stream[(path, bytes|str)] -> Stream[(Dataset, None)|(None, error_message)] 112 | 113 | :param doc_stream: sequence of (uri, doc: byges|string) 114 | :param on_error: Callback uri, doc -> None 115 | :param logger: Logger object for printing errors or None 116 | :param transform: dict -> dict if supplied also apply further transform on parsed document 117 | :param kwargs: passed on to from_metadata_stream 118 | 119 | """ 120 | 121 | def on_parse_error(uri, doc) -> None: 122 | # pylint: disable=unused-argument 123 | if logger is not None: 124 | logger.error(f"Failed to parse: {uri}") 125 | else: 126 | print(f"Failed to parse: {uri}", file=sys.stderr) 127 | 128 | metadata_stream = parse_doc_stream( 129 | doc_stream, on_error=on_parse_error, transform=transform 130 | ) 131 | return from_metadata_stream(metadata_stream, index, **kwargs) 132 | -------------------------------------------------------------------------------- /apps/dc_tools/odc/apps/dc_tools/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.9.1" 2 | -------------------------------------------------------------------------------- /apps/dc_tools/odc/apps/dc_tools/fs_to_dc.py: -------------------------------------------------------------------------------- 1 | import click 2 | import json 3 | import logging 4 | import yaml 5 | from pathlib import Path 6 | 7 | import datacube 8 | from datacube.index.hl import Doc2Dataset 9 | from datacube.ui.click import environment_option, pass_config 10 | from odc.apps.dc_tools._stac import stac_transform 11 | from odc.apps.dc_tools.utils import ( 12 | allow_unsafe, 13 | archive_less_mature, 14 | index_update_dataset, 15 | update_if_exists_flag, 16 | publish_action, 17 | statsd_setting, 18 | statsd_gauge_reporting, 19 | transform_stac, 20 | ) 21 | 22 | logging.basicConfig( 23 | level=logging.WARNING, 24 | format="%(asctime)s: %(levelname)s: %(message)s", 25 | datefmt="%m/%d/%Y %I:%M:%S", 26 | ) 27 | 28 | 29 | @click.command("fs-to-dc") 30 | @environment_option 31 | @pass_config 32 | @click.argument("input_directory", type=str, nargs=1) 33 | @update_if_exists_flag 34 | @allow_unsafe 35 | @archive_less_mature 36 | @transform_stac 37 | @statsd_setting 38 | @publish_action 39 | @click.option( 40 | "--glob", 41 | default=None, 42 | help="File system glob to use, defaults to **/*.yaml or **/*.json for STAC.", 43 | ) 44 | def cli( 45 | cfg_env, 46 | input_directory, 47 | update_if_exists, 48 | allow_unsafe, 49 | stac, 50 | statsd_setting, 51 | glob, 52 | archive_less_mature, 53 | publish_action, 54 | ) -> None: 55 | dc = datacube.Datacube(env=cfg_env) 56 | doc2ds = Doc2Dataset(dc.index) 57 | 58 | if glob is None: 59 | glob = "**/*.json" if stac else "**/*.yaml" 60 | 61 | files_to_process = Path(input_directory).glob(glob) 62 | 63 | added, failed = 0, 0 64 | 65 | for in_file in files_to_process: 66 | with in_file.open() as f: 67 | try: 68 | if in_file.suffix in (".yml", ".yaml"): 69 | metadata = yaml.safe_load(f) 70 | else: 71 | metadata = json.load(f) 72 | # Do the STAC Transform if it's flagged 73 | stac_doc = None 74 | if stac: 75 | stac_doc = metadata 76 | metadata = stac_transform(metadata) 77 | index_update_dataset( 78 | metadata, 79 | in_file.absolute().as_uri(), 80 | dc=dc, 81 | doc2ds=doc2ds, 82 | update_if_exists=update_if_exists, 83 | allow_unsafe=allow_unsafe, 84 | archive_less_mature=archive_less_mature, 85 | publish_action=publish_action, 86 | stac_doc=stac_doc, 87 | ) 88 | added += 1 89 | except Exception: # pylint: disable=broad-except 90 | logging.exception("Failed to add dataset %s", in_file) 91 | failed += 1 92 | 93 | logging.info("Added %s and failed %s datasets.", added, failed) 94 | if statsd_setting: 95 | statsd_gauge_reporting(added, ["app:fs_to_dc", "action:added"], statsd_setting) 96 | statsd_gauge_reporting( 97 | failed, ["app:fs_to_dc", "action:failed"], statsd_setting 98 | ) 99 | 100 | 101 | if __name__ == "__main__": 102 | cli() 103 | -------------------------------------------------------------------------------- /apps/dc_tools/odc/apps/dc_tools/thredds_to_dc.py: -------------------------------------------------------------------------------- 1 | """Crawl Thredds for prefixes and fetch YAML's for indexing 2 | and dump them into a Datacube instance 3 | """ 4 | 5 | import click 6 | import logging 7 | from odc.thredds import download_yamls, thredds_find_glob 8 | from typing import List, Tuple 9 | 10 | from datacube import Datacube 11 | from datacube.cfg import ODCEnvironment 12 | from datacube.ui.click import environment_option, pass_config 13 | from odc.apps.dc_tools.utils import statsd_gauge_reporting, statsd_setting 14 | from ._docs import from_yaml_doc_stream 15 | 16 | 17 | def dump_list_to_odc( 18 | yaml_content_list: List[Tuple[bytes, str, str]], 19 | dc: Datacube, 20 | products: List[str], 21 | **kwargs, 22 | ): 23 | expand_stream = ( 24 | ("https://" + d[1], d[0]) for d in yaml_content_list if d[0] is not None 25 | ) 26 | 27 | ds_stream = from_yaml_doc_stream( 28 | expand_stream, dc.index, products=products, **kwargs 29 | ) 30 | ds_added = 0 31 | ds_failed = 0 32 | # Consume chained streams to DB 33 | for result in ds_stream: 34 | ds, err = result 35 | if err is not None: 36 | logging.error(err) 37 | ds_failed += 1 38 | else: 39 | logging.info(ds) 40 | # TODO: Potentially wrap this in transactions and batch to DB 41 | # TODO: Capture UUID's from YAML and perform a bulk has 42 | try: 43 | dc.index.datasets.add(ds) 44 | ds_added += 1 45 | except Exception as e: # pylint: disable=broad-except 46 | logging.error(e) 47 | ds_failed += 1 48 | 49 | return ds_added, ds_failed 50 | 51 | 52 | @click.command("thredds-to-dc") 53 | @environment_option 54 | @pass_config 55 | @click.option( 56 | "--skip-lineage", 57 | is_flag=True, 58 | default=False, 59 | help="Default is not to skip lineage. Set to skip lineage altogether.", 60 | ) 61 | @click.option( 62 | "--fail-on-missing-lineage/--auto-add-lineage", 63 | is_flag=True, 64 | default=True, 65 | help=( 66 | "Default is to fail if lineage documents not present in the database. " 67 | "Set auto add to try to index lineage documents." 68 | ), 69 | ) 70 | @click.option( 71 | "--verify-lineage", 72 | is_flag=True, 73 | default=False, 74 | help="Default is no verification. Set to verify parent dataset definitions.", 75 | ) 76 | @statsd_setting 77 | @click.argument("uri", type=str, nargs=1) 78 | @click.argument("product", type=str, nargs=1) 79 | def cli( 80 | cfg_env: ODCEnvironment, 81 | skip_lineage: bool, 82 | fail_on_missing_lineage: bool, 83 | verify_lineage: bool, 84 | statsd_setting: str, 85 | uri: str, 86 | product: str, 87 | ) -> None: 88 | skips = [".*NBAR.*", ".*SUPPLEMENTARY.*", ".*NBART.*", ".*/QA/.*"] 89 | select = [".*ARD-METADATA.yaml"] 90 | candidate_products = product.split() 91 | print(f"Crawling {uri} on Thredds") 92 | print(f"Matching to {candidate_products}") 93 | yaml_urls = thredds_find_glob(uri, skips, select) 94 | print(f"Found {len(yaml_urls)} datasets") 95 | 96 | yaml_contents = download_yamls(yaml_urls) 97 | 98 | # Consume generator and fetch YAML's 99 | dc = Datacube(env=cfg_env) 100 | added, failed = dump_list_to_odc( 101 | yaml_contents, 102 | dc, 103 | candidate_products, 104 | skip_lineage=skip_lineage, 105 | fail_on_missing_lineage=fail_on_missing_lineage, 106 | verify_lineage=verify_lineage, 107 | ) 108 | 109 | print(f"Added {added} Datasets, Failed {failed} Datasets") 110 | if statsd_setting: 111 | statsd_gauge_reporting( 112 | added, ["app:thredds_to_dc", "action:added"], statsd_setting 113 | ) 114 | statsd_gauge_reporting( 115 | failed, ["app:thredds_to_dc", "action:failed"], statsd_setting 116 | ) 117 | -------------------------------------------------------------------------------- /apps/dc_tools/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=51.0.0", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.pytest.ini_options] 6 | addopts = [ 7 | "--import-mode=importlib", 8 | ] 9 | filterwarnings = [ 10 | "ignore:datetime.datetime.utcnow*:DeprecationWarning:botocore.*" 11 | ] 12 | -------------------------------------------------------------------------------- /apps/dc_tools/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = odc-apps-dc-tools 3 | description = CLI utils for working with a datacube index 4 | version = attr: odc.apps.dc_tools._version.__version__ 5 | author = Open Data Cube 6 | author_email = 7 | maintainer = Open Data Cube 8 | maintainer_email = 9 | long_description_content_type = text/markdown 10 | long_description = file: README.md 11 | platforms = any 12 | license = Apache License 2.0 13 | url = https://github.com/opendatacube/odc-tools/ 14 | 15 | [options] 16 | include_package_data = true 17 | zip_safe = false 18 | packages = find_namespace: 19 | python_requires = >= 3.10 20 | tests_require = 21 | pytest 22 | pytest_httpserver 23 | deepdiff 24 | docker 25 | flask 26 | flask_cors 27 | moto 28 | psycopg2 29 | 30 | install_requires = 31 | click 32 | fsspec 33 | pystac-client>=0.2.0 34 | toolz 35 | pyyaml 36 | datacube>=1.9.0 37 | odc_io 38 | odc-cloud[ASYNC]>=0.2.3 39 | odc-geo 40 | odc-stac 41 | pystac>=1.0.0 42 | rio-stac 43 | urlpath 44 | datadog 45 | eodatasets3>=1.9 46 | 47 | [options.extras_require] 48 | tests = 49 | pytest 50 | pytest_httpserver 51 | deepdiff 52 | docker 53 | flask 54 | flask_cors 55 | moto 56 | psycopg2 57 | 58 | AZURE = odc-cloud[AZURE] 59 | THREDDS = odc-cloud[THREDDS] 60 | 61 | [options.entry_points] 62 | console_scripts = 63 | dc-sync-products = odc.apps.dc_tools.add_update_products:cli 64 | dc-index-from-tar = odc.apps.dc_tools.index_from_tar:cli 65 | dc-index-export-md = odc.apps.dc_tools.export_md:cli 66 | s3-to-dc = odc.apps.dc_tools.s3_to_dc:cli 67 | thredds-to-dc = odc.apps.dc_tools.thredds_to_dc:cli [THREDDS] 68 | sqs-to-dc = odc.apps.dc_tools.sqs_to_dc:cli 69 | stac-to-dc = odc.apps.dc_tools.stac_api_to_dc:cli 70 | azure-to-dc = odc.apps.dc_tools.azure_to_dc:cli [AZURE] 71 | cop-dem-to-dc = odc.apps.dc_tools.cop_dem_to_dc:cli 72 | fs-to-dc = odc.apps.dc_tools.fs_to_dc:cli 73 | esa-wc-to-dc = odc.apps.dc_tools.esa_worldcover_to_dc:cli 74 | 75 | 76 | [options.packages.find] 77 | include = 78 | odc* 79 | 80 | [options.package_data] 81 | * = 82 | *.txt 83 | *.sh 84 | -------------------------------------------------------------------------------- /apps/dc_tools/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/60U-2020.stac-item.json: -------------------------------------------------------------------------------- 1 | {"id":"60U-2020","bbox":[171.00908750245537,50.46857176742808,178.62016649866072,54.14235160249478],"type":"Feature","links":[{"rel":"collection","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/io-lulc"},{"rel":"parent","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/io-lulc"},{"rel":"root","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/"},{"rel":"self","type":"application/geo+json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/io-lulc/items/60U-2020"},{"rel":"preview","href":"https://planetarycomputer.microsoft.com/api/data/v1/item/map?collection=io-lulc&item=60U-2020","title":"Map of item","type":"text/html"}],"assets":{"data":{"href":"https://ai4edataeuwest.blob.core.windows.net/io-lulc/io-lulc-model-001-v01-composite-v03-supercell-v02-clip-v01/60U_20200101-20210101.tif","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"raster:bands":[{"nodata":0,"spatial_resolution":10}]},"tilejson":{"title":"TileJSON with default rendering","href":"https://planetarycomputer.microsoft.com/api/data/v1/item/tilejson.json?collection=io-lulc&items=60U-2020&assets=data&colormap_name=io-lulc","type":"application/json","roles":["tiles"]},"rendered_preview":{"title":"Rendered preview","rel":"preview","href":"https://planetarycomputer.microsoft.com/api/data/v1/item/preview.png?collection=io-lulc&items=60U-2020&assets=data&colormap_name=io-lulc","roles":["overview"],"type":"image/png"}},"geometry":{"type":"Polygon","coordinates":[[[178.55462086,51.40060111],[178.5238168,50.46857177],[177.00873816,50.47854712],[177.00890852,51.37732737],[175.62632353,51.36959067],[174.1374672,51.34300635],[174.0801884,52.23623042],[174.0487866,52.23722251],[173.98870791,51.33985258],[172.44577282,51.36906847],[172.47487755,52.26739734],[171.0090875,52.27729032],[171.00927664,53.17591543],[171.00927626,53.17591543],[171.00928362,53.2090495],[171.00929095,53.24387311],[171.00929135,53.24387311],[171.0094909,54.1423516],[172.54025416,54.13176875],[172.54028045,54.13249834],[172.59764252,54.131372],[172.65501577,54.13097535],[172.65498753,54.130246],[174.18377597,54.10022743],[174.11735976,53.20835167],[175.51075066,53.23385872],[175.51072667,53.23456994],[175.56734638,53.23489475],[175.6239581,53.23593107],[175.62398028,53.23521964],[177.0092908,53.24316663],[177.00929095,53.24387311],[177.06548595,53.243489],[177.12168131,53.24381137],[177.12167936,53.24310491],[178.6201665,53.23286237],[178.58591446,52.29954132],[178.55462086,51.40060111]]]},"collection":"io-lulc","properties":{"datetime":"2020-06-01T00:00:00Z","proj:bbox":[186554.85147020232,5712384.68704876,608384.8514702023,5892994.68704876],"proj:epsg":32660,"label:type":"raster","proj:shape":[18061,42183],"end_datetime":"2021-01-01T00:00:00Z","label:classes":[{"name":"","classes":["nodata","water","trees","grass","flooded veg","crops","scrub","built area","bare","snow/ice","clouds"]}],"proj:transform":[10.0,0.0,186554.85147020232,0.0,-10.0,5892994.68704876],"start_datetime":"2020-01-01T00:00:00Z","io:supercell_id":"60U","label:description":"lulc"},"stac_version":"1.0.0","stac_extensions":["https://stac-extensions.github.io/projection/v1.0.0/schema.json","https://stac-extensions.github.io/label/v1.0.0/schema.json","https://stac-extensions.github.io/raster/v1.0.0/schema.json"]} 2 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/NASADEM_HGT_s56w072.stac-item.json: -------------------------------------------------------------------------------- 1 | {"id":"NASADEM_HGT_s56w072","bbox":[-72.000139,-56.000139,-70.999861,-54.999861],"type":"Feature","links":[{"rel":"collection","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/nasadem"},{"rel":"parent","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/nasadem"},{"rel":"root","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/"},{"rel":"self","type":"application/geo+json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/nasadem/items/NASADEM_HGT_s56w072"}],"assets":{"elevation":{"href":"https://nasademeuwest.blob.core.windows.net/nasadem-cog/v001/NASADEM_HGT_s56w072.tif","type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"title":"Elevation"}},"geometry":{"type":"Polygon","coordinates":[[[-70.99986111,-56.00013889],[-70.99986111,-54.99986111],[-72.00013889,-54.99986111],[-72.00013889,-56.00013889],[-70.99986111,-56.00013889]]]},"collection":"nasadem","properties":{"datetime":"2000-02-20T00:00:00Z","proj:bbox":[-72.00013888888888,-56.000138888888884,-70.9998611111111,-54.99986111111111],"proj:epsg":4326,"proj:shape":[3601,3601],"proj:transform":[0.0002777777777777778,0.0,-72.00013888888888,0.0,-0.0002777777777777778,-54.99986111111111,0.0,0.0,1.0]},"stac_version":"1.0.0","stac_extensions":["https://stac-extensions.github.io/projection/v1.0.0/schema.json"]} 2 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/S2A_28QCH_20200714_0_L2A.odc-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schemas.opendatacube.org/dataset", 3 | "id": "503fa092-0f0f-5544-89fb-b45c1da0d5e1", 4 | "crs": "EPSG:32628", 5 | "grids": { 6 | "g320m": { 7 | "shape": [ 8 | 343, 9 | 343 10 | ], 11 | "transform": [ 12 | 320.0, 13 | 0.0, 14 | 300000.0, 15 | 0.0, 16 | -320.0, 17 | 2300040.0, 18 | 0.0, 19 | 0.0, 20 | 1.0 21 | ] 22 | }, 23 | "g60m": { 24 | "shape": [ 25 | 1830, 26 | 1830 27 | ], 28 | "transform": [ 29 | 60.0, 30 | 0.0, 31 | 300000.0, 32 | 0.0, 33 | -60.0, 34 | 2300040.0, 35 | 0.0, 36 | 0.0, 37 | 1.0 38 | ] 39 | }, 40 | "g20m": { 41 | "shape": [ 42 | 5490, 43 | 5490 44 | ], 45 | "transform": [ 46 | 20.0, 47 | 0.0, 48 | 300000.0, 49 | 0.0, 50 | -20.0, 51 | 2300040.0, 52 | 0.0, 53 | 0.0, 54 | 1.0 55 | ] 56 | }, 57 | "default": { 58 | "shape": [ 59 | 10980, 60 | 10980 61 | ], 62 | "transform": [ 63 | 10.0, 64 | 0.0, 65 | 300000.0, 66 | 0.0, 67 | -10.0, 68 | 2300040.0, 69 | 0.0, 70 | 0.0, 71 | 1.0 72 | ] 73 | } 74 | }, 75 | "product": { 76 | "name": "s2_l2a" 77 | }, 78 | "label": "S2A_MSIL2A_20200714T115221_N0214_R123_T28QCH_20200714T191310", 79 | "properties": { 80 | "datetime": "2020-07-14T11:55:52Z", 81 | "odc:processing_datetime": "2020-07-14T11:55:52Z", 82 | "eo:cloud_cover": 2.36, 83 | "eo:gsd": 10, 84 | "eo:instrument": "MSI", 85 | "eo:platform": "sentinel-2a", 86 | "odc:file_format": "GeoTIFF", 87 | "odc:region_code": "28QCH", 88 | "eo:constellation": "sentinel-2", 89 | "data_coverage": 6.85, 90 | "eo:off_nadir": 0, 91 | "proj:epsg": 32628, 92 | "sentinel:latitude_band": "Q", 93 | "sentinel:grid_square": "CH", 94 | "sentinel:sequence": "0", 95 | "sentinel:product_id": "S2A_MSIL2A_20200714T115221_N0214_R123_T28QCH_20200714T191310" 96 | }, 97 | "measurements": { 98 | "overview": { 99 | "path": "L2A_PVI.tif", 100 | "grid": "g320m" 101 | }, 102 | "visual": { 103 | "path": "TCI.tif" 104 | }, 105 | "B01": { 106 | "path": "B01.tif", 107 | "grid": "g60m" 108 | }, 109 | "B02": { 110 | "path": "B02.tif" 111 | }, 112 | "B03": { 113 | "path": "B03.tif" 114 | }, 115 | "B04": { 116 | "path": "B04.tif" 117 | }, 118 | "B05": { 119 | "path": "B05.tif", 120 | "grid": "g20m" 121 | }, 122 | "B06": { 123 | "path": "B06.tif", 124 | "grid": "g20m" 125 | }, 126 | "B07": { 127 | "path": "B07.tif", 128 | "grid": "g20m" 129 | }, 130 | "B08": { 131 | "path": "B08.tif" 132 | }, 133 | "B8A": { 134 | "path": "B8A.tif", 135 | "grid": "g20m" 136 | }, 137 | "B09": { 138 | "path": "B09.tif", 139 | "grid": "g60m" 140 | }, 141 | "B11": { 142 | "path": "B11.tif", 143 | "grid": "g20m" 144 | }, 145 | "B12": { 146 | "path": "B12.tif", 147 | "grid": "g20m" 148 | }, 149 | "AOT": { 150 | "path": "AOT.tif", 151 | "grid": "g60m" 152 | }, 153 | "WVP": { 154 | "path": "WVP.tif" 155 | }, 156 | "SCL": { 157 | "path": "SCL.tif", 158 | "grid": "g20m" 159 | } 160 | }, 161 | "lineage": {}, 162 | "geometry": { 163 | "type": "Polygon", 164 | "coordinates": [ 165 | [ 166 | [ 167 | 300111.0, 168 | 2218486.0 169 | ], 170 | [ 171 | 300101.0, 172 | 2300039.0 173 | ], 174 | [ 175 | 320356.0, 176 | 2300039.0 177 | ], 178 | [ 179 | 300111.0, 180 | 2218486.0 181 | ] 182 | ] 183 | ] 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1-odc-metadata.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Dataset 3 | # url: https://explorer.dev.digitalearth.africa/dataset/83d02080-0394-5bb1-bd1d-cfc23e0db47d.odc-metadata.yaml 4 | $schema: https://schemas.opendatacube.org/dataset 5 | id: 83d02080-0394-5bb1-bd1d-cfc23e0db47d 6 | 7 | label: S2B_T37MGP_20240710T073012_L2A 8 | product: 9 | name: s2_l2a_c1 10 | 11 | crs: EPSG:32737 12 | geometry: 13 | type: Polygon 14 | coordinates: 15 | [ 16 | [ 17 | [6.99961e+05, 9.399999e+06], 18 | [6.99961e+05, 9.290201e+06], 19 | [8.09759e+05, 9.290201e+06], 20 | [8.09759e+05, 9.399999e+06], 21 | [6.99961e+05, 9.399999e+06], 22 | ], 23 | ] 24 | grids: 25 | g20m: 26 | shape: [5490, 5490] 27 | transform: [2.e+01, 0.e+00, 6.9996e+05, 0.e+00, -2.e+01, 9.4e+06] 28 | g60m: 29 | shape: [1830, 1830] 30 | transform: [6.e+01, 0.e+00, 6.9996e+05, 0.e+00, -6.e+01, 9.4e+06] 31 | g320m: 32 | shape: [343, 343] 33 | transform: 34 | [ 35 | 3.2e+02, 36 | 0.e+00, 37 | 6.9996e+05, 38 | 0.e+00, 39 | -3.2e+02, 40 | 9.4e+06, 41 | 0.e+00, 42 | 0.e+00, 43 | 1.e+00, 44 | ] 45 | default: 46 | shape: [10980, 10980] 47 | transform: [1.e+01, 0.e+00, 6.9996e+05, 0.e+00, -1.e+01, 9.4e+06] 48 | 49 | properties: 50 | created: "2024-07-10T13:38:13.736092+00:00" 51 | datetime: "2024-07-10T07:41:33.657000Z" 52 | earthsearch:payload_id: roda-sentinel-2-c1-l2a/workflow-sentinel-2-c1-l2a-to-stac/c76ef495ac22038115a5d0865aafac19 53 | eo:azimuth: 1.85518023779404e+02 54 | eo:cloud_cover: 8.3505428e+01 55 | eo:constellation: sentinel-2 56 | eo:instrument: MSI 57 | eo:platform: sentinel-2b 58 | eo:sun_azimuth: 3.99264277845747e+01 59 | eo:sun_elevation: 5.29617721681249e+01 60 | grid:code: MGRS-37MGP 61 | mgrs:grid_square: GP 62 | mgrs:latitude_band: M 63 | mgrs:utm_zone: 37 64 | odc:file_format: GeoTIFF 65 | odc:processing_datetime: "2024-07-10T13:38:13.736092+00:00" 66 | odc:region_code: 37MGP 67 | processing:software: 68 | sentinel-2-c1-l2a-to-stac: v2024.02.01 69 | proj:centroid: 70 | lat: -5.92002e+00 71 | lon: 4.130189e+01 72 | proj:epsg: 32737 73 | s2:cloud_shadow_percentage: 0.e+00 74 | s2:dark_features_percentage: 0.e+00 75 | s2:datastrip_id: S2B_OPER_MSI_L2A_DS_2BPS_20240710T103747_S20240710T073012_N05.10 76 | s2:datatake_id: GS2B_20240710T071619_038356_N05.10 77 | s2:datatake_type: INS-NOBS 78 | s2:degraded_msi_data_percentage: 7.9e-03 79 | s2:generation_time: "2024-07-10T10:37:47.000000Z" 80 | s2:high_proba_clouds_percentage: 3.4168363e+01 81 | s2:medium_proba_clouds_percentage: 2.7591509e+01 82 | s2:nodata_pixel_percentage: 0.e+00 83 | s2:not_vegetated_percentage: 2.3e-05 84 | s2:processing_baseline: "05.10" 85 | s2:product_type: S2MSI2A 86 | s2:product_uri: S2B_MSIL2A_20240710T071619_N0510_R006_T37MGP_20240710T103747.SAFE 87 | s2:reflectance_conversion_factor: 9.67377332839659e-01 88 | s2:saturated_defective_pixel_percentage: 0.e+00 89 | s2:snow_ice_percentage: 0.e+00 90 | s2:thin_cirrus_percentage: 2.1745561e+01 91 | s2:tile_id: S2B_OPER_MSI_L2A_TL_2BPS_20240710T103747_A038356_T37MGP_N05.10 92 | s2:unclassified_percentage: 0.e+00 93 | s2:vegetation_percentage: 1.e-05 94 | s2:water_percentage: 1.6494533e+01 95 | storage:platform: AWS 96 | storage:region: us-west-2 97 | storage:requester_pays: false 98 | updated: "2024-07-10T13:38:13.736092+00:00" 99 | view:incidence_angle: 2.79725898254359e+00 100 | 101 | measurements: 102 | aot: 103 | grid: g20m 104 | path: AOT.tif 105 | nir: 106 | path: B08.tif 107 | red: 108 | path: B04.tif 109 | scl: 110 | grid: g20m 111 | path: SCL.tif 112 | wvp: 113 | grid: g20m 114 | path: WVP.tif 115 | blue: 116 | path: B02.tif 117 | snow: 118 | grid: g20m 119 | path: SNW_20m.tif 120 | cloud: 121 | grid: g20m 122 | path: CLD_20m.tif 123 | green: 124 | path: B03.tif 125 | nir08: 126 | grid: g20m 127 | path: B8A.tif 128 | nir09: 129 | grid: g60m 130 | path: B09.tif 131 | swir16: 132 | grid: g20m 133 | path: B11.tif 134 | swir22: 135 | grid: g20m 136 | path: B12.tif 137 | visual: 138 | path: TCI.tif 139 | coastal: 140 | grid: g60m 141 | path: B01.tif 142 | preview: 143 | grid: g320m 144 | path: L2A_PVI.tif 145 | rededge1: 146 | grid: g20m 147 | path: B05.tif 148 | rededge2: 149 | grid: g20m 150 | path: B06.tif 151 | rededge3: 152 | grid: g20m 153 | path: B07.tif 154 | 155 | accessories: 156 | thumbnail: 157 | path: https://e84-earth-search-sentinel-data.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/L2A_PVI.jpg 158 | granule_metadata: 159 | path: https://e84-earth-search-sentinel-data.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/metadata.xml 160 | product_metadata: 161 | path: https://e84-earth-search-sentinel-data.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/product_metadata.xml 162 | tileinfo_metadata: 163 | path: https://e84-earth-search-sentinel-data.s3.us-west-2.amazonaws.com/sentinel-2-c1-l2a/37/M/GP/2024/7/S2B_T37MGP_20240710T073012_L2A/tileInfo.json 164 | 165 | lineage: {} 166 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/S2B_T37MGP_20240710T073012_L2A_C1_rel_links-odc-metadata.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Dataset 3 | # url: https://explorer.dev.digitalearth.africa/dataset/83d02080-0394-5bb1-bd1d-cfc23e0db47d.odc-metadata.yaml 4 | $schema: https://schemas.opendatacube.org/dataset 5 | id: 83d02080-0394-5bb1-bd1d-cfc23e0db47d 6 | 7 | label: S2B_T37MGP_20240710T073012_L2A 8 | product: 9 | name: s2_l2a_c1 10 | 11 | crs: EPSG:32737 12 | geometry: 13 | type: Polygon 14 | coordinates: 15 | [ 16 | [ 17 | [6.99961e+05, 9.399999e+06], 18 | [6.99961e+05, 9.290201e+06], 19 | [8.09759e+05, 9.290201e+06], 20 | [8.09759e+05, 9.399999e+06], 21 | [6.99961e+05, 9.399999e+06], 22 | ], 23 | ] 24 | grids: 25 | g20m: 26 | shape: [5490, 5490] 27 | transform: [2.e+01, 0.e+00, 6.9996e+05, 0.e+00, -2.e+01, 9.4e+06] 28 | g60m: 29 | shape: [1830, 1830] 30 | transform: [6.e+01, 0.e+00, 6.9996e+05, 0.e+00, -6.e+01, 9.4e+06] 31 | g320m: 32 | shape: [343, 343] 33 | transform: 34 | [ 35 | 3.2e+02, 36 | 0.e+00, 37 | 6.9996e+05, 38 | 0.e+00, 39 | -3.2e+02, 40 | 9.4e+06, 41 | 0.e+00, 42 | 0.e+00, 43 | 1.e+00, 44 | ] 45 | default: 46 | shape: [10980, 10980] 47 | transform: [1.e+01, 0.e+00, 6.9996e+05, 0.e+00, -1.e+01, 9.4e+06] 48 | 49 | properties: 50 | created: "2024-07-10T13:38:13.736092+00:00" 51 | datetime: "2024-07-10T07:41:33.657000Z" 52 | earthsearch:payload_id: roda-sentinel-2-c1-l2a/workflow-sentinel-2-c1-l2a-to-stac/c76ef495ac22038115a5d0865aafac19 53 | eo:azimuth: 1.85518023779404e+02 54 | eo:cloud_cover: 8.3505428e+01 55 | eo:constellation: sentinel-2 56 | eo:instrument: MSI 57 | eo:platform: sentinel-2b 58 | eo:sun_azimuth: 3.99264277845747e+01 59 | eo:sun_elevation: 5.29617721681249e+01 60 | grid:code: MGRS-37MGP 61 | mgrs:grid_square: GP 62 | mgrs:latitude_band: M 63 | mgrs:utm_zone: 37 64 | odc:file_format: GeoTIFF 65 | odc:processing_datetime: "2024-07-10T13:38:13.736092+00:00" 66 | odc:region_code: 37MGP 67 | processing:software: 68 | sentinel-2-c1-l2a-to-stac: v2024.02.01 69 | proj:centroid: 70 | lat: -5.92002e+00 71 | lon: 4.130189e+01 72 | proj:epsg: 32737 73 | s2:cloud_shadow_percentage: 0.e+00 74 | s2:dark_features_percentage: 0.e+00 75 | s2:datastrip_id: S2B_OPER_MSI_L2A_DS_2BPS_20240710T103747_S20240710T073012_N05.10 76 | s2:datatake_id: GS2B_20240710T071619_038356_N05.10 77 | s2:datatake_type: INS-NOBS 78 | s2:degraded_msi_data_percentage: 7.9e-03 79 | s2:generation_time: "2024-07-10T10:37:47.000000Z" 80 | s2:high_proba_clouds_percentage: 3.4168363e+01 81 | s2:medium_proba_clouds_percentage: 2.7591509e+01 82 | s2:nodata_pixel_percentage: 0.e+00 83 | s2:not_vegetated_percentage: 2.3e-05 84 | s2:processing_baseline: "05.10" 85 | s2:product_type: S2MSI2A 86 | s2:product_uri: S2B_MSIL2A_20240710T071619_N0510_R006_T37MGP_20240710T103747.SAFE 87 | s2:reflectance_conversion_factor: 9.67377332839659e-01 88 | s2:saturated_defective_pixel_percentage: 0.e+00 89 | s2:snow_ice_percentage: 0.e+00 90 | s2:thin_cirrus_percentage: 2.1745561e+01 91 | s2:tile_id: S2B_OPER_MSI_L2A_TL_2BPS_20240710T103747_A038356_T37MGP_N05.10 92 | s2:unclassified_percentage: 0.e+00 93 | s2:vegetation_percentage: 1.e-05 94 | s2:water_percentage: 1.6494533e+01 95 | storage:platform: AWS 96 | storage:region: us-west-2 97 | storage:requester_pays: false 98 | updated: "2024-07-10T13:38:13.736092+00:00" 99 | view:incidence_angle: 2.79725898254359e+00 100 | 101 | measurements: 102 | aot: 103 | grid: g20m 104 | path: AOT.tif 105 | nir: 106 | path: B08.tif 107 | red: 108 | path: B04.tif 109 | scl: 110 | grid: g20m 111 | path: SCL.tif 112 | wvp: 113 | grid: g20m 114 | path: WVP.tif 115 | blue: 116 | path: B02.tif 117 | snow: 118 | grid: g20m 119 | path: SNW_20m.tif 120 | cloud: 121 | grid: g20m 122 | path: CLD_20m.tif 123 | green: 124 | path: B03.tif 125 | nir08: 126 | grid: g20m 127 | path: B8A.tif 128 | nir09: 129 | grid: g60m 130 | path: B09.tif 131 | swir16: 132 | grid: g20m 133 | path: B11.tif 134 | swir22: 135 | grid: g20m 136 | path: B12.tif 137 | visual: 138 | path: TCI.tif 139 | coastal: 140 | grid: g60m 141 | path: B01.tif 142 | preview: 143 | grid: g320m 144 | path: L2A_PVI.tif 145 | rededge1: 146 | grid: g20m 147 | path: B05.tif 148 | rededge2: 149 | grid: g20m 150 | path: B06.tif 151 | rededge3: 152 | grid: g20m 153 | path: B07.tif 154 | 155 | accessories: 156 | thumbnail: 157 | path: L2A_PVI.jpg 158 | granule_metadata: 159 | path: metadata.xml 160 | product_metadata: 161 | path: product_metadata.xml 162 | tileinfo_metadata: 163 | path: tileInfo.json 164 | 165 | lineage: {} 166 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/01/07/alos_cumul_2010-01-07.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 70cc0091-081d-4e12-adca-fe98ff68f898 4 | label: cemp_insar_alos_displacement_2010-01-07 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-01-07 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:43.928235Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-01-07.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-01-07.tif 25 | ud: 26 | path: alos_cumul_up_2010-01-07.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-01-07.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/01/19/alos_cumul_2010-01-19.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 551710e9-5d26-4d0e-b1ba-c58a2f609abf 4 | label: cemp_insar_alos_displacement_2010-01-19 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-01-19 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:45.184652Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-01-19.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-01-19.tif 25 | ud: 26 | path: alos_cumul_up_2010-01-19.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-01-19.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/01/31/alos_cumul_2010-01-31.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: a9feec8a-0a05-4e32-83c6-181eae97ad6b 4 | label: cemp_insar_alos_displacement_2010-01-31 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-01-31 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:46.556399Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-01-31.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-01-31.tif 25 | ud: 26 | path: alos_cumul_up_2010-01-31.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-01-31.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/02/12/alos_cumul_2010-02-12.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 6b1a661d-f760-441e-ae28-c9e2c02ae4a0 4 | label: cemp_insar_alos_displacement_2010-02-12 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-02-12 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:47.754428Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-02-12.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-02-12.tif 25 | ud: 26 | path: alos_cumul_up_2010-02-12.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-02-12.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/02/24/alos_cumul_2010-02-24.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 96365d07-cf0a-4016-b1ac-fa50a7180644 4 | label: cemp_insar_alos_displacement_2010-02-24 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-02-24 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:49.004510Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-02-24.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-02-24.tif 25 | ud: 26 | path: alos_cumul_up_2010-02-24.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-02-24.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/03/08/alos_cumul_2010-03-08.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 1b96cdb0-3fbb-4eaa-905e-55ef7557440b 4 | label: cemp_insar_alos_displacement_2010-03-08 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-03-08 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:50.371186Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-03-08.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-03-08.tif 25 | ud: 26 | path: alos_cumul_up_2010-03-08.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-03-08.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/03/20/alos_cumul_2010-03-20.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 13aab384-d1f1-480d-ab4d-a38175282be0 4 | label: cemp_insar_alos_displacement_2010-03-20 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-03-20 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:51.571458Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-03-20.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-03-20.tif 25 | ud: 26 | path: alos_cumul_up_2010-03-20.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-03-20.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/04/01/alos_cumul_2010-04-01.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 7ce155d4-782e-476c-9152-af9f65f6f02a 4 | label: cemp_insar_alos_displacement_2010-04-01 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-04-01 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:52.778114Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-04-01.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-04-01.tif 25 | ud: 26 | path: alos_cumul_up_2010-04-01.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-04-01.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/04/13/alos_cumul_2010-04-13.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 674febac-fb40-4eb6-a0e4-68fa2bee7677 4 | label: cemp_insar_alos_displacement_2010-04-13 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-04-13 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:54.129615Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-04-13.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-04-13.tif 25 | ud: 26 | path: alos_cumul_up_2010-04-13.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-04-13.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/04/25/alos_cumul_2010-04-25.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: bcfaac13-3b1e-4e0b-b299-a6073ce24a81 4 | label: cemp_insar_alos_displacement_2010-04-25 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-04-25 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:55.549837Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-04-25.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-04-25.tif 25 | ud: 26 | path: alos_cumul_up_2010-04-25.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-04-25.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/05/07/alos_cumul_2010-05-07.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 8ee1406c-a2a7-443f-9623-ff39f743f684 4 | label: cemp_insar_alos_displacement_2010-05-07 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-05-07 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:56.765738Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-05-07.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-05-07.tif 25 | ud: 26 | path: alos_cumul_up_2010-05-07.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-05-07.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/05/19/alos_cumul_2010-05-19.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 4e3e1512-9ab5-45fa-b44a-69c1a3d2a1f4 4 | label: cemp_insar_alos_displacement_2010-05-19 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-05-19 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:58.235947Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-05-19.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-05-19.tif 25 | ud: 26 | path: alos_cumul_up_2010-05-19.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-05-19.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/05/31/alos_cumul_2010-05-31.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 67b4aab7-cb5f-4afa-8b1a-092c86d1dc82 4 | label: cemp_insar_alos_displacement_2010-05-31 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254974.87113444862, 6178175.000166063], [274425.0, 6178125.0], [338774.9999826678, 6178224.958368041], [338825.0, 6238325.0], [338725.0, 6259925.0], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-05-31 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:51:59.751631Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-05-31.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-05-31.tif 25 | ud: 26 | path: alos_cumul_up_2010-05-31.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-05-31.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/06/12/alos_cumul_2010-06-12.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 4d7ab777-2f0d-4ee1-9796-7868c3369c49 4 | label: cemp_insar_alos_displacement_2010-06-12 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254925.0, 6178225.0], [274375.0, 6178125.0], [329025.0457875266, 6178175.000020965], [329124.99996934057, 6223374.944629048], [329074.99995230616, 6259875.069060707], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-06-12 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:52:01.229092Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-06-12.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-06-12.tif 25 | ud: 26 | path: alos_cumul_up_2010-06-12.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-06-12.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/06/24/alos_cumul_2010-06-24.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: fe8d4b14-97c4-48b1-a222-07fa5b1b261e 4 | label: cemp_insar_alos_displacement_2010-06-24 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254925.0, 6178225.0], [274375.0, 6178125.0], [329025.0457875266, 6178175.000020965], [329124.99996934057, 6223374.944629048], [329074.99995230616, 6259875.069060707], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-06-24 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:52:02.440236Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-06-24.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-06-24.tif 25 | ud: 26 | path: alos_cumul_up_2010-06-24.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-06-24.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/07/06/alos_cumul_2010-07-06.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 1f2e5d1b-df64-40c5-9827-aff4275e064e 4 | label: cemp_insar_alos_displacement_2010-07-06 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254925.0, 6178225.0], [274375.0, 6178125.0], [329025.0457875266, 6178175.000020965], [329124.99996934057, 6223374.944629048], [329074.99995230616, 6259875.069060707], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-07-06 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:52:03.904773Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-07-06.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-07-06.tif 25 | ud: 26 | path: alos_cumul_up_2010-07-06.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-07-06.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/07/18/alos_cumul_2010-07-18.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: fe622f32-8be4-4e46-912e-0e76d5f3a5b4 4 | label: cemp_insar_alos_displacement_2010-07-18 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254925.0, 6178225.0], [274375.0, 6178125.0], [329025.0457875266, 6178175.000020965], [329124.99996934057, 6223374.944629048], [329074.99995230616, 6259875.069060707], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-07-18 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:52:05.087282Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-07-18.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-07-18.tif 25 | ud: 26 | path: alos_cumul_up_2010-07-18.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-07-18.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/07/30/alos_cumul_2010-07-30.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: b9d8e238-a285-41ff-9d32-a6989026ec3a 4 | label: cemp_insar_alos_displacement_2010-07-30 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254925.0, 6178225.0], [274375.0, 6178125.0], [329025.0457875266, 6178175.000020965], [329124.99996934057, 6223374.944629048], [329074.99995230616, 6259875.069060707], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-07-30 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:52:06.392504Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-07-30.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-07-30.tif 25 | ud: 26 | path: alos_cumul_up_2010-07-30.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-07-30.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/08/11/alos_cumul_2010-08-11.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 2e9c8d42-7d47-4f74-a8ae-ce38f5a97c5b 4 | label: cemp_insar_alos_displacement_2010-08-11 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254925.0, 6178225.0], [274375.0, 6178125.0], [329025.0457875266, 6178175.000020965], [329124.99996934057, 6223374.944629048], [329074.99995230616, 6259875.069060707], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-08-11 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:52:07.600386Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-08-11.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-08-11.tif 25 | ud: 26 | path: alos_cumul_up_2010-08-11.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-08-11.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/08/23/alos_cumul_2010-08-23.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 5e851088-bdb4-4b1e-80eb-59718780881e 4 | label: cemp_insar_alos_displacement_2010-08-23 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254925.0, 6178225.0], [274375.0, 6178125.0], [329025.0457875266, 6178175.000020965], [329124.99996934057, 6223374.944629048], [329074.99995230616, 6259875.069060707], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-08-23 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:52:09.033965Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-08-23.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-08-23.tif 25 | ud: 26 | path: alos_cumul_up_2010-08-23.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-08-23.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/09/04/alos_cumul_2010-09-04.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: ebc6f11f-7046-4bff-a6e5-0ab5b3f7a939 4 | label: cemp_insar_alos_displacement_2010-09-04 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254925.0, 6178225.0], [274375.0, 6178125.0], [329025.0457875266, 6178175.000020965], [329124.99996934057, 6223374.944629048], [329074.99995230616, 6259875.069060707], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-09-04 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:52:10.227706Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-09-04.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-09-04.tif 25 | ud: 26 | path: alos_cumul_up_2010-09-04.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-09-04.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/09/16/alos_cumul_2010-09-16.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 3452742d-cce7-4a45-8853-0d706e4a3eb6 4 | label: cemp_insar_alos_displacement_2010-09-16 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254925.0, 6178225.0], [274375.0, 6178125.0], [329025.0457875266, 6178175.000020965], [329124.99996934057, 6223374.944629048], [329074.99995230616, 6259875.069060707], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-09-16 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:52:11.431575Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-09-16.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-09-16.tif 25 | ud: 26 | path: alos_cumul_up_2010-09-16.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-09-16.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/09/28/alos_cumul_2010-09-28.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: ae11ce9d-7685-40fb-8a4f-e128462d8ac2 4 | label: cemp_insar_alos_displacement_2010-09-28 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[254975.0, 6259925.0], [254925.0, 6178225.0], [274375.0, 6178125.0], [329025.0457875266, 6178175.000020965], [329124.99996934057, 6223374.944629048], [329074.99995230616, 6259875.069060707], [254975.0, 6259925.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-09-28 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:52:12.816882Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-09-28.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-09-28.tif 25 | ud: 26 | path: alos_cumul_up_2010-09-28.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-09-28.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/10/10/alos_cumul_2010-10-10.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 1f45d5f3-6d23-4e74-87ac-e9072e73050e 4 | label: cemp_insar_alos_displacement_2010-10-10 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[275175.0, 6251175.0], [275125.0, 6184575.0], [294625.0, 6184475.0], [329025.0727801267, 6184525.00005297], [329124.99995848397, 6223374.935567064], [329074.99991705426, 6251125.09107453], [275175.0, 6251175.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-10-10 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:52:14.025084Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-10-10.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-10-10.tif 25 | ud: 26 | path: alos_cumul_up_2010-10-10.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-10-10.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/cemp_insar/10/22/alos_cumul_2010-10-22.yaml: -------------------------------------------------------------------------------- 1 | # Dataset 2 | $schema: https://schemas.opendatacube.org/dataset 3 | id: 26d2d4d0-a543-48b0-a2a9-9b56c1e42be9 4 | label: cemp_insar_alos_displacement_2010-10-22 5 | product: 6 | name: cemp_insar_alos_displacement 7 | crs: EPSG:32756 8 | geometry: 9 | type: Polygon 10 | coordinates: [[[275175.0, 6251175.0], [275125.0, 6184575.0], [294625.0, 6184475.0], [329025.0727801267, 6184525.00005297], [329124.99995848397, 6223374.935567064], [329074.99991705426, 6251125.09107453], [275175.0, 6251175.0]]] 11 | grids: 12 | default: 13 | shape: [1731, 1703] 14 | transform: [50.00000000000001, 0.0, 254475.0, 0.0, -50.00000000000001, 6260125.0, 0.0, 0.0, 1.0] 15 | properties: 16 | datetime: 2010-10-22 01:01:01Z 17 | odc:file_format: GeoTIFF 18 | odc:processing_datetime: 2019-12-19 20:52:15.157234Z 19 | odc:product_family: cemp_insar_alos_displacement 20 | measurements: 21 | ew: 22 | path: alos_cumul_east_2010-10-22.tif 23 | ewstd: 24 | path: alos_cumulstd_east_2010-10-22.tif 25 | ud: 26 | path: alos_cumul_up_2010-10-22.tif 27 | upstd: 28 | path: alos_cumulstd_up_2010-10-22.tif 29 | accessories: {} 30 | lineage: {} 31 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/derivative/ga_ls5t_nbart_gm_cyear_3/3-0-0/x08/y23/1994--P1Y/ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final.odc-metadata.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Dataset 3 | $schema: https://schemas.opendatacube.org/dataset 4 | id: 6c22734e-66b7-4746-b28c-1f10792f4b2a 5 | 6 | label: ga_ls5t_nbart_gm_cyear_3_x08y23_1994-01-01_final 7 | product: 8 | name: ga_ls5t_nbart_gm_cyear_3 9 | href: https://collections.dea.ga.gov.au/product/ga_ls5t_nbart_gm_cyear_3 10 | 11 | crs: EPSG:3577 12 | geometry: 13 | type: Polygon 14 | coordinates: [[[-1920000.0, -3168000.0], [-1920000.0, -3264000.0], [-1824000.0, 15 | -3264000.0], [-1824000.0, -3168000.0], [-1920000.0, -3168000.0]]] 16 | grids: 17 | default: 18 | shape: [3200, 3200] 19 | transform: [30.0, 0.0, -1920000.0, 0.0, -30.0, -3168000.0, 0.0, 0.0, 1.0] 20 | 21 | properties: 22 | datetime: 1994-01-01 00:00:00Z 23 | dea:dataset_maturity: final 24 | dtr:end_datetime: 1994-12-31 23:59:59.999999Z 25 | dtr:start_datetime: 1994-01-01 00:00:00Z 26 | eo:gsd: 30.0 # Ground sample distance (m) 27 | eo:instrument: TM 28 | eo:platform: landsat-5 29 | odc:collection_number: 3 30 | odc:dataset_version: 3.0.0 31 | odc:file_format: GeoTIFF 32 | odc:processing_datetime: 2021-10-20 00:18:28.907451Z 33 | odc:producer: ga.gov.au 34 | odc:product: ga_ls5t_nbart_gm_cyear_3 35 | odc:product_family: geomedian 36 | odc:region_code: x08y23 37 | 38 | measurements: 39 | bcdev: 40 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final_bcdev.tif 41 | blue: 42 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final_blue.tif 43 | count: 44 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final_count.tif 45 | edev: 46 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final_edev.tif 47 | green: 48 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final_green.tif 49 | nir: 50 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final_nir.tif 51 | red: 52 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final_red.tif 53 | sdev: 54 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final_sdev.tif 55 | swir1: 56 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final_swir1.tif 57 | swir2: 58 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final_swir2.tif 59 | 60 | accessories: 61 | thumbnail: 62 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final_thumbnail.jpg 63 | checksum:sha1: 64 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final.sha1 65 | metadata:processor: 66 | path: ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final.proc-info.yaml 67 | 68 | lineage: 69 | ard: 70 | - 00dcb2c3-e718-4d15-920a-62bd40c4150c 71 | - 186503c3-4132-44d7-b4c8-ff848203bc44 72 | - 52e3ef55-b236-47fc-911b-806f7ac6d3ca 73 | - b59d3fce-36e6-4719-bc16-bb8cd1c05257 74 | - af9cdb2a-3244-4680-95f3-026c88e463a7 75 | - cad759df-351a-4a17-bb0c-a2dda2285faa 76 | - dcce8c66-4bda-4849-a002-43b44903881a 77 | - 1d080a31-5e94-4b09-9f05-5d2e42b7279e 78 | - e1f73ffb-6fc7-47f8-829b-a99a6e04dd74 79 | - 84a7b17d-56f9-49cc-a239-51aca2495c99 80 | - 0eea7649-fdf2-4b45-8f47-d18d5bbbb5cb 81 | - 786af085-90b0-4ca1-8540-35af431b860d 82 | - 681ec1df-3e4a-4a8a-ba71-fa576c034ba0 83 | - 7b0585dd-9aec-4be5-8f70-3375738ed9b8 84 | - 6c27bdd4-c02e-4e92-9d77-4921049d9b51 85 | - 71667715-2eec-4abd-823c-bf29d661b295 86 | ... 87 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/derivative/ga_ls5t_nbart_gm_cyear_3/3-0-0/x08/y23/1994--P1Y/ga_ls5t_nbart_gm_cyear_3_x08y23_1994--P1Y_final.proc-info.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | input-products: 3 | - ga_ls5t_ard_3 4 | odc-stats-config: 5 | name: ga_ls5t_nbart_gm_cyear_3 6 | version: 3.0.0 7 | short_name: ga_ls5t_nbart_gm_cyear_3 8 | location: s3://dea-public-data-dev/ga_ls5t_nbart_gm_cyear_3/3-0-0 9 | properties: 10 | odc:file_format: GeoTIFF 11 | odc:product_family: geomedian 12 | odc:producer: ga.gov.au 13 | measurements: 14 | - red 15 | - green 16 | - blue 17 | - nir 18 | - swir1 19 | - swir2 20 | - sdev 21 | - edev 22 | - bcdev 23 | - count 24 | href: https://collections.dea.ga.gov.au/product/ga_ls5t_nbart_gm_cyear_3 25 | region_code_format: x{x:02d}y{y:02d} 26 | cfg: 27 | naming_conventions_values: dea_c3 28 | explorer_path: https://explorer.dea.ga.gov.au/ 29 | inherit_skip_properties: 30 | - eo:cloud_cover 31 | - fmask:clear 32 | - fmask:snow 33 | - fmask:cloud 34 | - fmask:water 35 | - fmask:cloud_shadow 36 | - eo:sun_elevation 37 | - eo:sun_azimuth 38 | - gqa:iterative_stddev_x 39 | - gqa:iterative_stddev_y 40 | - gqa:iterative_stddev_xy 41 | - gqa:stddev_xy 42 | - gqa:stddev_x 43 | - gqa:stddev_y 44 | - gqa:mean_xy 45 | - gqa:mean_x 46 | - gqa:mean_y 47 | - gqa:abs_xy 48 | - gqa:abs_x 49 | - gqa:abs_y 50 | - gqa:abs_iterative_mean_y 51 | - gqa:abs_iterative_mean_x 52 | - gqa:abs_iterative_mean_xy 53 | - gqa:iterative_mean_xy 54 | - gqa:iterative_mean_x 55 | - gqa:iterative_mean_y 56 | - gqa:cep90 57 | - landsat:landsat_product_id 58 | - landsat:landsat_scene_id 59 | - landsat:collection_category 60 | - landsat:collection_number 61 | - landsat:wrs_path 62 | - landsat:wrs_row 63 | preview_image_ows_style: 64 | name: simple_rgb 65 | title: Simple RGB 66 | abstract: Simple true-colour image, using the red, green and blue bands 67 | components: 68 | red: 69 | red: 1 70 | green: 71 | green: 1 72 | blue: 73 | blue: 1 74 | scale_range: 75 | - 0 76 | - 3000 77 | classifier: ard 78 | maturity: final 79 | collection_number: 3 80 | nodata: 81 | software_versions: 82 | - name: eodatasets3 83 | url: https://github.com/GeoscienceAustralia/eo-datasets 84 | version: 0.22.2 85 | - name: odc-stats 86 | url: https://github.com/opendatacube/odc-tools 87 | version: 1.0.2-dev3083 88 | - name: gm_ls_annual 89 | url: https://github.com/opendatacube/odc-tools 90 | version: 3.0.0 91 | - name: datacube-ows 92 | url: https://github.com/opendatacube/datacube-ows 93 | version: 1.8.20 94 | ... 95 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N03E003_Map.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatacube/odc-tools/b1a12fa3adf90aa5e9f565fa405d2881e1baea01/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N03E003_Map.tif -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N03E006_Map.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatacube/odc-tools/b1a12fa3adf90aa5e9f565fa405d2881e1baea01/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N03E006_Map.tif -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N06E003_Map.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatacube/odc-tools/b1a12fa3adf90aa5e9f565fa405d2881e1baea01/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N06E003_Map.tif -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N06E006_Map.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatacube/odc-tools/b1a12fa3adf90aa5e9f565fa405d2881e1baea01/apps/dc_tools/tests/data/esa_worldcover/ESA_WorldCover_10m_2020_v100_N06E006_Map.tif -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/example_product_list.csv: -------------------------------------------------------------------------------- 1 | product,definition 2 | s2_l2a,https://gist.githubusercontent.com/Ariana-B/90caca4bf5275da19b7c2936385a0f54/raw/f09dc109b51d83da5e07192d64122c32fa18a7ca/esa_s2_l2a.odc-product.yaml 3 | cemp_insar_alos_displacement,https://raw.githubusercontent.com/GeoscienceAustralia/digitalearthau/885e3d3a00bcc9c9fe3ed43e82201d7121898aed/digitalearthau/config/products/cemp_insar_alos_displacement.yaml 4 | ls5_c2l2_sr;ls7_c2l2_sr;ls8_c2l2_sr,https://raw.githubusercontent.com/opendatacube/datacube-dataset-config/4b4e67e7fac9b15beb57fc87f3e0556c23872adb/products/lsX_c2l2_sr.odc-product.yaml 5 | nasadem,https://raw.githubusercontent.com/opendatacube/datacube-dataset-config/main/products/nasadem.odc-product.yaml 6 | dem_cop_30,https://raw.githubusercontent.com/opendatacube/datacube-dataset-config/main/products/dem_cop_30.odc-product.yaml 7 | dem_cop_90,https://raw.githubusercontent.com/opendatacube/datacube-dataset-config/main/products/dem_cop_90.odc-product.yaml 8 | io_lulc,https://raw.githubusercontent.com/opendatacube/datacube-dataset-config/main/products/io_lulc.odc-product.yaml 9 | esa_worldcover_2020,https://raw.githubusercontent.com/opendatacube/datacube-dataset-config/main/products/esa_worldcover_2020.odc-product.yaml 10 | ga_ls5t_nbart_gm_cyear_3,https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/414c7bb065d15e838679c309285137f15cfb885c/products/baseline_satellite_data/geomedian-au/ga_ls5t_nbart_gm_cyear_3.yaml 11 | ga_s2am_ard_3,https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/products/baseline_satellite_data/c3/ga_s2am_ard_3.odc-product.yaml 12 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/ga_ls5t_gm_product.yaml: -------------------------------------------------------------------------------- 1 | name: ga_ls5t_nbart_gm_cyear_3 2 | description: Geoscience Australia Landsat Nadir BRDF Adjusted Reflectance Terrain, Landsat 5 Geomedian Calendar Year Collection 3 3 | metadata_type: eo3 4 | 5 | metadata: 6 | properties: 7 | odc:file_format: GeoTIFF 8 | odc:product_family: geomedian 9 | product: 10 | name: ga_ls5t_nbart_gm_cyear_3 11 | 12 | measurements: 13 | - name: blue 14 | dtype: int16 15 | nodata: -999 16 | units: '1' 17 | - name: green 18 | dtype: int16 19 | nodata: -999 20 | units: '1' 21 | - name: red 22 | dtype: int16 23 | nodata: -999 24 | units: '1' 25 | - name: nir 26 | dtype: int16 27 | nodata: -999 28 | units: '1' 29 | - name: swir1 30 | dtype: int16 31 | nodata: -999 32 | units: '1' 33 | - name: swir2 34 | dtype: int16 35 | nodata: -999 36 | units: '1' 37 | - name: sdev 38 | dtype: float32 39 | nodata: .nan 40 | units: '1' 41 | - name: edev 42 | dtype: float32 43 | nodata: .nan 44 | units: '1' 45 | - name: bcdev 46 | dtype: float32 47 | nodata: .nan 48 | units: '1' 49 | - name: count 50 | dtype: int16 51 | nodata: -999 52 | units: '1' 53 | storage: 54 | crs: EPSG:3577 55 | resolution: 56 | x: 30 57 | y: -30 58 | tile_size: 59 | x: 3200.0 60 | y: 3200.0 61 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.sqs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Dataset 3 | $schema: https://schemas.opendatacube.org/dataset 4 | id: 2aa69fcf-aa55-4747-9d95-3652f9fe79b0 5 | 6 | label: ga_ls8c_ard_3-1-0_088080_2020-05-25_final 7 | product: 8 | name: ga_ls8c_ard_3 9 | href: https://collections.dea.ga.gov.au/product/ga_ls8c_ard_3 10 | 11 | crs: EPSG:32656 12 | geometry: 13 | type: Polygon 14 | coordinates: [[[572743.2722121085, -3077151.309749513], [572745.0, -3077145.0], 15 | [572969.4472838908, -3077198.798209872], [573250.996461604, -3077257.9131992455], 16 | [758606.0005942872, -3121687.9141904702], [759791.1380343756, -3121972.947862498], 17 | [759832.5, -3122002.5], [714367.0886436494, -3312160.9887643186], [714322.5, 18 | -3312262.5], [714228.5968569319, -3312239.9605382206], [714225.0, -3312255.0], 19 | [527767.9988114258, -3267524.1716190595], [527167.7239312489, -3267374.1042750045], 20 | [527095.752540109, -3267322.682433342], [527032.5, -3267307.5], [528337.9117568319, 21 | -3261784.0095614507], [541042.9139480147, -3208729.0004160986], [553732.9148397002, 22 | -3155853.9967016927], [572107.9166454426, -3079413.9891924215], [572647.9478624978, 23 | -3077178.8619656246], [572677.5, -3077137.5], [572743.2722121085, -3077151.309749513]]] 24 | grids: 25 | default: 26 | shape: [7841, 7781] 27 | transform: [30.0, 0.0, 526785.0, 0.0, -30.0, -3077085.0, 0.0, 0.0, 1.0] 28 | g15m: 29 | shape: [15681, 15561] 30 | transform: [15.0, 0.0, 526792.5, 0.0, -15.0, -3077092.5, 0.0, 0.0, 1.0] 31 | 32 | properties: 33 | datetime: 2020-05-25 23:35:47.745731Z 34 | dea:dataset_maturity: final 35 | dtr:end_datetime: 2020-05-25 23:36:02.557719Z 36 | dtr:start_datetime: 2020-05-25 23:35:32.815426Z 37 | eo:cloud_cover: 42.42102985910952 38 | eo:gsd: 15.0 # Ground sample distance (m) 39 | eo:instrument: OLI_TIRS 40 | eo:platform: landsat-8 41 | eo:sun_azimuth: 34.22994171 42 | eo:sun_elevation: 31.7895917 43 | fmask:clear: 1.0427704509544131 44 | fmask:cloud: 42.42102985910952 45 | fmask:cloud_shadow: 2.919727915379625 46 | fmask:snow: 0.00034972190505407335 47 | fmask:water: 53.61612205265139 48 | gqa:abs_iterative_mean_x: 0.17 49 | gqa:abs_iterative_mean_xy: 0.24 50 | gqa:abs_iterative_mean_y: 0.16 51 | gqa:abs_x: 0.28 52 | gqa:abs_xy: 0.4 53 | gqa:abs_y: 0.28 54 | gqa:cep90: 0.4 55 | gqa:iterative_mean_x: 0.03 56 | gqa:iterative_mean_xy: 0.12 57 | gqa:iterative_mean_y: 0.12 58 | gqa:iterative_stddev_x: 0.21 59 | gqa:iterative_stddev_xy: 0.26 60 | gqa:iterative_stddev_y: 0.15 61 | gqa:mean_x: -0.08 62 | gqa:mean_xy: 0.15 63 | gqa:mean_y: 0.12 64 | gqa:stddev_x: 0.42 65 | gqa:stddev_xy: 0.58 66 | gqa:stddev_y: 0.4 67 | landsat:collection_category: T1 68 | landsat:collection_number: 1 69 | landsat:landsat_product_id: LC08_L1TP_088080_20200525_20200608_01_T1 70 | landsat:landsat_scene_id: LC80880802020146LGN00 71 | landsat:wrs_path: 88 72 | landsat:wrs_row: 80 73 | odc:dataset_version: 3.1.0 74 | odc:file_format: GeoTIFF 75 | odc:processing_datetime: 2020-06-18 07:07:06.303867Z 76 | odc:producer: ga.gov.au 77 | odc:product_family: ard 78 | odc:region_code: '088080' 79 | 80 | measurements: 81 | nbart_blue: 82 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band02.tif 83 | nbart_coastal_aerosol: 84 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band01.tif 85 | nbart_green: 86 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band03.tif 87 | nbart_nir: 88 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band05.tif 89 | nbart_panchromatic: 90 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band08.tif 91 | grid: g15m 92 | nbart_red: 93 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band04.tif 94 | nbart_swir_1: 95 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band06.tif 96 | nbart_swir_2: 97 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band07.tif 98 | oa_azimuthal_exiting: 99 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_azimuthal-exiting.tif 100 | oa_azimuthal_incident: 101 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_azimuthal-incident.tif 102 | oa_combined_terrain_shadow: 103 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_combined-terrain-shadow.tif 104 | oa_exiting_angle: 105 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_exiting-angle.tif 106 | oa_fmask: 107 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_fmask.tif 108 | oa_incident_angle: 109 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_incident-angle.tif 110 | oa_nbart_contiguity: 111 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_nbart-contiguity.tif 112 | oa_relative_azimuth: 113 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_relative-azimuth.tif 114 | oa_relative_slope: 115 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_relative-slope.tif 116 | oa_satellite_azimuth: 117 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_satellite-azimuth.tif 118 | oa_satellite_view: 119 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_satellite-view.tif 120 | oa_solar_azimuth: 121 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_solar-azimuth.tif 122 | oa_solar_zenith: 123 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_solar-zenith.tif 124 | oa_time_delta: 125 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_time-delta.tif 126 | 127 | accessories: 128 | thumbnail:nbart: 129 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_thumbnail.jpg 130 | checksum:sha1: 131 | path: ga_ls8c_ard_3-1-0_088080_2020-05-25_final.sha1 132 | metadata:processor: 133 | path: ga_ls8c_ard_3-1-0_088080_2020-05-25_final.proc-info.yaml 134 | 135 | lineage: 136 | level1: 137 | - b5f234fe-bba8-5483-9bc0-250360d429cf 138 | ... 139 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Dataset 3 | $schema: https://schemas.opendatacube.org/dataset 4 | id: 2aa69fcf-aa55-4747-9d95-3652f9fe79b0 5 | 6 | label: ga_ls8c_ard_3-1-0_088080_2020-05-25_final 7 | product: 8 | name: ga_ls8c_ard_3 9 | href: https://collections.dea.ga.gov.au/product/ga_ls8c_ard_3 10 | 11 | crs: EPSG:32656 12 | geometry: 13 | type: Polygon 14 | coordinates: 15 | [ 16 | [ 17 | [572743.0, -3077151.0], 18 | [572745.0, -3077145.0], 19 | [572969.0, -3077199.0], 20 | [573251.0, -3077258.0], 21 | [758606.0, -3121688.0], 22 | [759791.0, -3121973.0], 23 | [759832.0, -3122002.0], 24 | [714367.0, -3312161.0], 25 | [714322.0, -3312262.0], 26 | [714229.0, -3312240.0], 27 | [714225.0, -3312255.0], 28 | [527768.0, -3267524.0], 29 | [527168.0, -3267374.0], 30 | [527096.0, -3267323.0], 31 | [527032.0, -3267308.0], 32 | [528338.0, -3261784.0], 33 | [541043.0, -3208729.0], 34 | [553733.0, -3155854.0], 35 | [572108.0, -3079414.0], 36 | [572648.0, -3077179.0], 37 | [572677.0, -3077138.0], 38 | [572743.0, -3077151.0], 39 | ], 40 | ] 41 | grids: 42 | default: 43 | shape: [7841, 7781] 44 | transform: [30.0, 0.0, 526785.0, 0.0, -30.0, -3077085.0, 0.0, 0.0, 1.0] 45 | g15m: 46 | shape: [15681, 15561] 47 | transform: [15.0, 0.0, 526792.5, 0.0, -15.0, -3077092.5, 0.0, 0.0, 1.0] 48 | 49 | properties: 50 | datetime: 2020-05-25T23:35:47.745731Z 51 | dea:dataset_maturity: final 52 | dtr:end_datetime: 2020-05-25T23:36:02.557719Z 53 | dtr:start_datetime: 2020-05-25T23:35:32.815426Z 54 | eo:cloud_cover: 42.42102985910952 55 | eo:gsd: 15.0 # Ground sample distance (m) 56 | eo:instrument: OLI_TIRS 57 | eo:platform: landsat-8 58 | eo:sun_azimuth: 34.22994171 59 | eo:sun_elevation: 31.7895917 60 | fmask:clear: 1.0427704509544131 61 | fmask:cloud: 42.42102985910952 62 | fmask:cloud_shadow: 2.919727915379625 63 | fmask:snow: 0.00034972190505407335 64 | fmask:water: 53.61612205265139 65 | gqa:abs_iterative_mean_x: 0.17 66 | gqa:abs_iterative_mean_xy: 0.24 67 | gqa:abs_iterative_mean_y: 0.16 68 | gqa:abs_x: 0.28 69 | gqa:abs_xy: 0.4 70 | gqa:abs_y: 0.28 71 | gqa:cep90: 0.4 72 | gqa:iterative_mean_x: 0.03 73 | gqa:iterative_mean_xy: 0.12 74 | gqa:iterative_mean_y: 0.12 75 | gqa:iterative_stddev_x: 0.21 76 | gqa:iterative_stddev_xy: 0.26 77 | gqa:iterative_stddev_y: 0.15 78 | gqa:mean_x: -0.08 79 | gqa:mean_xy: 0.15 80 | gqa:mean_y: 0.12 81 | gqa:stddev_x: 0.42 82 | gqa:stddev_xy: 0.58 83 | gqa:stddev_y: 0.4 84 | landsat:collection_category: T1 85 | landsat:collection_number: 1 86 | landsat:landsat_product_id: LC08_L1TP_088080_20200525_20200608_01_T1 87 | landsat:landsat_scene_id: LC80880802020146LGN00 88 | landsat:wrs_path: 88 89 | landsat:wrs_row: 80 90 | odc:dataset_version: 3.1.0 91 | odc:file_format: GeoTIFF 92 | odc:processing_datetime: 2020-06-18T07:07:06.303867Z 93 | odc:producer: ga.gov.au 94 | odc:product_family: ard 95 | odc:region_code: "088080" 96 | 97 | measurements: 98 | nbart_blue: 99 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band02.tif 100 | nbart_coastal_aerosol: 101 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band01.tif 102 | nbart_green: 103 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band03.tif 104 | nbart_nir: 105 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band05.tif 106 | nbart_panchromatic: 107 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band08.tif 108 | grid: g15m 109 | nbart_red: 110 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band04.tif 111 | nbart_swir_1: 112 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band06.tif 113 | nbart_swir_2: 114 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band07.tif 115 | oa_azimuthal_exiting: 116 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_azimuthal-exiting.tif 117 | oa_azimuthal_incident: 118 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_azimuthal-incident.tif 119 | oa_combined_terrain_shadow: 120 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_combined-terrain-shadow.tif 121 | oa_exiting_angle: 122 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_exiting-angle.tif 123 | oa_fmask: 124 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_fmask.tif 125 | oa_incident_angle: 126 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_incident-angle.tif 127 | oa_nbart_contiguity: 128 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_nbart-contiguity.tif 129 | oa_relative_azimuth: 130 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_relative-azimuth.tif 131 | oa_relative_slope: 132 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_relative-slope.tif 133 | oa_satellite_azimuth: 134 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_satellite-azimuth.tif 135 | oa_satellite_view: 136 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_satellite-view.tif 137 | oa_solar_azimuth: 138 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_solar-azimuth.tif 139 | oa_solar_zenith: 140 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_solar-zenith.tif 141 | oa_time_delta: 142 | path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_time-delta.tif 143 | 144 | accessories: 145 | thumbnail:nbart: 146 | path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_thumbnail.jpg 147 | checksum:sha1: 148 | path: ga_ls8c_ard_3-1-0_088080_2020-05-25_final.sha1 149 | metadata:processor: 150 | path: ga_ls8c_ard_3-1-0_088080_2020-05-25_final.proc-info.yaml 151 | 152 | lineage: 153 | level1: 154 | - b5f234fe-bba8-5483-9bc0-250360d429cf 155 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/lidar_dem.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "Feature", 3 | "stac_version": "1.0.0-beta.2", 4 | "id": "lidar_id", 5 | "properties": { 6 | "start_datetime": "2012-01-01T00:00:00Z", 7 | "end_datetime": "2012-01-01T00:00:00Z", 8 | "resolution": 1.0, 9 | "data_type": "float32", 10 | "derived_from": "Category 1 Lidar", 11 | "platform": "Aircraft", 12 | "interpolation_type": "TIN", 13 | "horizontal_datum": "GDA94", 14 | "vertical_datum": "AHD71 - using local Geoid model", 15 | "model_type": "DEM", 16 | "horizontal_accuracy": "+/-0.80 @95% Confidence Interval", 17 | "vertical_accuracy": "+/-0.30 @95% Confidence Interval", 18 | "sensor": "ALS50 (SN101)", 19 | "proj:epsg": 28355, 20 | "proj:shape": [ 21 | 2000, 22 | 2000 23 | ], 24 | "proj:transform": [ 25 | 1.0, 26 | 0.0, 27 | 766000.0, 28 | 0.0, 29 | -1.0, 30 | 6732000.0, 31 | 0.0, 32 | 0.0, 33 | 1.0 34 | ], 35 | "datetime": null 36 | }, 37 | "geometry": { 38 | "type": "Polygon", 39 | "coordinates": [ 40 | [ 41 | [ 42 | 149.74413486135487, 43 | -29.513331085946845 44 | ], 45 | [ 46 | 149.74462179934187, 47 | -29.531360829758146 48 | ], 49 | [ 50 | 149.76523815043552, 51 | -29.53093320024493 52 | ], 53 | [ 54 | 149.7647475661126, 55 | -29.512903768601603 56 | ], 57 | [ 58 | 149.74413486135487, 59 | -29.513331085946845 60 | ] 61 | ] 62 | ] 63 | }, 64 | "links": [ 65 | { 66 | "rel": "root", 67 | "href": "s3://example-bucket/catalog.json", 68 | "type": "application/json" 69 | }, 70 | { 71 | "rel": "collection", 72 | "href": "s3://example-bucket/lidar_collection/collection.json", 73 | "type": "application/json" 74 | }, 75 | { 76 | "rel": "parent", 77 | "href": "s3://example-bucket/lidar_collection/collection.json", 78 | "type": "application/json" 79 | }, 80 | { 81 | "rel": "self", 82 | "href": "s3://example-bucket/lidar_collection/lidar_id.json", 83 | "type": "application/json" 84 | } 85 | ], 86 | "assets": { 87 | "dem": { 88 | "href": "s3://example-bucket/lidar_id.tif", 89 | "type": "image/tiff; application=geotiff; profile=cloud-optimized", 90 | "title": "Cloud-Optimized Geotiff" 91 | } 92 | }, 93 | "bbox": [ 94 | 149.74413486135487, 95 | -29.531360829758146, 96 | 149.76523815043552, 97 | -29.512903768601603 98 | ], 99 | "stac_extensions": [ 100 | "projection" 101 | ], 102 | "collection": "lidar_collection" 103 | } 104 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/maturity-final.odc-metadata.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Dataset 3 | # url: https://explorer.dea.ga.gov.au/dataset/9f27a15e-3cdf-4e3f-a58e-dd624b2c3bef.odc-metadata.yaml 4 | $schema: https://schemas.opendatacube.org/dataset 5 | id: 9f27a15e-3cdf-4e3f-a58e-dd624b2c3bef 6 | 7 | label: ga_ls5t_nbart_gm_cyear_3_x33y24_1990-01-01_final 8 | product: 9 | href: https://collections.dea.ga.gov.au/product/ga_ls5t_nbart_gm_cyear_3 10 | name: ga_ls5t_nbart_gm_cyear_3 11 | 12 | crs: EPSG:3577 13 | geometry: 14 | type: Polygon 15 | coordinates: [[[4.8e+05, -3.072e+06], [4.8e+05, -3.168e+06], [5.76e+05, -3.168e+06], 16 | [5.76e+05, -3.072e+06], [4.8e+05, -3.072e+06]]] 17 | grids: 18 | default: 19 | shape: [3200, 3200] 20 | transform: [3.e+01, 0.e+00, 4.8e+05, 0.e+00, -3.e+01, -3.072e+06, 0.e+00, 0.e+00, 21 | 1.e+00] 22 | 23 | properties: 24 | datetime: '1990-01-01 00:00:00Z' 25 | dea:dataset_maturity: final 26 | dtr:end_datetime: '1990-12-31 23:59:59.999999Z' 27 | dtr:start_datetime: '1990-01-01 00:00:00Z' 28 | eo:gsd: 3.e+01 # Ground sample distance (m) 29 | eo:instrument: TM 30 | eo:platform: landsat-5 31 | odc:collection_number: 3 32 | odc:dataset_version: 3.0.0 33 | odc:file_format: GeoTIFF 34 | odc:processing_datetime: '2021-10-20 00:00:29.270331Z' 35 | odc:producer: ga.gov.au 36 | odc:product: ga_ls5t_nbart_gm_cyear_3 37 | odc:product_family: geomedian 38 | odc:region_code: x33y24 39 | 40 | measurements: 41 | nir: 42 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final_nir.tif 43 | red: 44 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final_red.tif 45 | blue: 46 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final_blue.tif 47 | edev: 48 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final_edev.tif 49 | sdev: 50 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final_sdev.tif 51 | bcdev: 52 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final_bcdev.tif 53 | count: 54 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final_count.tif 55 | green: 56 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final_green.tif 57 | swir1: 58 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final_swir1.tif 59 | swir2: 60 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final_swir2.tif 61 | 62 | accessories: 63 | thumbnail: 64 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final_thumbnail.jpg 65 | checksum:sha1: 66 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final.sha1 67 | metadata:processor: 68 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_final.proc-info.yaml 69 | ... 70 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/data/maturity-nrt.odc-metadata.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Dataset 3 | # url: https://explorer.dea.ga.gov.au/dataset/9f27a15e-3cdf-4e3f-a58e-dd624b2c3bef.odc-metadata.yaml 4 | $schema: https://schemas.opendatacube.org/dataset 5 | id: 2e9f4623-c51c-5233-869a-bb690f8c2cac 6 | 7 | label: ga_ls5t_nbart_gm_cyear_3_x33y24_1990-01-01_nrt 8 | product: 9 | href: https://collections.dea.ga.gov.au/product/ga_ls5t_nbart_gm_cyear_3 10 | name: ga_ls5t_nbart_gm_cyear_3 11 | 12 | crs: EPSG:3577 13 | geometry: 14 | type: Polygon 15 | coordinates: [[[4.8e+05, -3.072e+06], [4.8e+05, -3.168e+06], [5.76e+05, -3.168e+06], 16 | [5.76e+05, -3.072e+06], [4.8e+05, -3.072e+06]]] 17 | grids: 18 | default: 19 | shape: [3200, 3200] 20 | transform: [3.e+01, 0.e+00, 4.8e+05, 0.e+00, -3.e+01, -3.072e+06, 0.e+00, 0.e+00, 21 | 1.e+00] 22 | 23 | properties: 24 | datetime: '1990-01-01 00:00:00Z' 25 | dea:dataset_maturity: nrt 26 | dtr:end_datetime: '1990-12-31 23:59:59.999999Z' 27 | dtr:start_datetime: '1990-01-01 00:00:00Z' 28 | eo:gsd: 3.e+01 # Ground sample distance (m) 29 | eo:instrument: TM 30 | eo:platform: landsat-5 31 | odc:collection_number: 3 32 | odc:dataset_version: 3.0.0 33 | odc:file_format: GeoTIFF 34 | odc:processing_datetime: '2021-10-20 00:00:29.270331Z' 35 | odc:producer: ga.gov.au 36 | odc:product: ga_ls5t_nbart_gm_cyear_3 37 | odc:product_family: geomedian 38 | odc:region_code: x33y24 39 | 40 | measurements: 41 | nir: 42 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt_nir.tif 43 | red: 44 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt_red.tif 45 | blue: 46 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt_blue.tif 47 | edev: 48 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt_edev.tif 49 | sdev: 50 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt_sdev.tif 51 | bcdev: 52 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt_bcdev.tif 53 | count: 54 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt_count.tif 55 | green: 56 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt_green.tif 57 | swir1: 58 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt_swir1.tif 59 | swir2: 60 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt_swir2.tif 61 | 62 | accessories: 63 | thumbnail: 64 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt_thumbnail.jpg 65 | checksum:sha1: 66 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt.sha1 67 | metadata:processor: 68 | path: ga_ls5t_nbart_gm_cyear_3_x33y24_1990--P1Y_nrt.proc-info.yaml 69 | ... 70 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/test_add_update_products.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from click.testing import CliRunner 3 | from pathlib import Path 4 | 5 | from odc.apps.dc_tools.add_update_products import _get_product, _parse_csv 6 | from odc.apps.dc_tools.add_update_products import cli as add_update_products_cli 7 | 8 | TEST_DATA_FOLDER: Path = Path(__file__).parent.joinpath("data") 9 | LOCAL_EXAMPLE: str = "example_product_list.csv" 10 | PRODUCT_EXAMPLE: str = ( 11 | "https://raw.githubusercontent.com/digitalearthafrica/" 12 | "config/master/products/esa_s2_l2a.odc-product.yaml" 13 | ) 14 | 15 | 16 | def test_parse_local_csv(local_csv) -> None: 17 | local_contents = [x for x in _parse_csv(local_csv)] 18 | 19 | assert len(local_contents) == 12 20 | assert local_contents[0].name == "s2_l2a" 21 | 22 | 23 | def test_parse_remote_csv(remote_csv) -> None: 24 | remote_contents = [x for x in _parse_csv(remote_csv)] 25 | assert len(remote_contents) == 12 26 | assert remote_contents[0].name == "s2_l2a" 27 | 28 | 29 | def test_load_product_def(remote_product) -> None: 30 | products = _get_product(remote_product) 31 | 32 | assert products[0]["name"] == "s2_l2a" 33 | 34 | 35 | def test_add_products(local_csv, odc_db, env_name) -> None: 36 | runner = CliRunner() 37 | # This will fail if requester pays is enabled 38 | result = runner.invoke( 39 | add_update_products_cli, 40 | [ 41 | local_csv, 42 | "--update-if-exists", 43 | "--env", 44 | env_name, 45 | ], 46 | ) 47 | print(f"CLI Output: {result.output}") 48 | assert result.exit_code == 0 49 | 50 | 51 | @pytest.fixture 52 | def remote_product(): 53 | return PRODUCT_EXAMPLE 54 | 55 | 56 | @pytest.fixture 57 | def local_csv(): 58 | return str(TEST_DATA_FOLDER / LOCAL_EXAMPLE) 59 | 60 | 61 | @pytest.fixture 62 | def remote_csv(httpserver, local_csv): 63 | httpserver.expect_request("/some.csv").respond_with_data(open(local_csv).read()) 64 | yield httpserver.url_for("/some.csv") 65 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/test_cop_dem_to_dc.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from click.testing import CliRunner 3 | 4 | from odc.apps.dc_tools.cop_dem_to_dc import get_dem_tile_uris, cli as cop_dem_to_dc_cli 5 | 6 | PRODUCTS = ["cop_30", "cop_90"] 7 | 8 | 9 | @pytest.fixture 10 | def bbox() -> str: 11 | return "5,5,7,7" 12 | 13 | 14 | @pytest.fixture 15 | def bbox_africa() -> str: 16 | return "-26.359944882003788,-47.96476498374171,64.4936701740102,38.34459242512347" 17 | 18 | 19 | @pytest.mark.parametrize("product", PRODUCTS) 20 | def test_get_dem_tile_uris(bbox, product, odc_db) -> None: 21 | uris = list(get_dem_tile_uris(bbox, product)) 22 | 23 | if product == "cop_30": 24 | assert uris[0][0] == ( 25 | "https://copernicus-dem-30m.s3.eu-central-1.amazonaws.com/" 26 | "Copernicus_DSM_COG_10_N05_00_E005_00_DEM/Copernicus_DSM_COG_10_N05_00_E005_00_DEM.tif" 27 | ) 28 | else: 29 | assert uris[0][0] == ( 30 | "https://copernicus-dem-90m.s3.eu-central-1.amazonaws.com/" 31 | "Copernicus_DSM_COG_30_N05_00_E005_00_DEM/Copernicus_DSM_COG_30_N05_00_E005_00_DEM.tif" 32 | ) 33 | 34 | assert len(uris) == 4 35 | 36 | 37 | def test_complex_bbox(bbox_africa) -> None: 38 | uris = list(get_dem_tile_uris(bbox_africa, "cop_30")) 39 | 40 | assert len(uris) == 8004 41 | 42 | 43 | # Test the actual process 44 | @pytest.mark.parametrize("product", PRODUCTS) 45 | def test_indexing_cli(bbox, product, odc_db, env_name) -> None: 46 | runner = CliRunner() 47 | result = runner.invoke( 48 | cop_dem_to_dc_cli, 49 | [ 50 | "--add-product", 51 | "--bbox", 52 | bbox, 53 | "--product", 54 | product, 55 | "--env", 56 | env_name, 57 | ], 58 | ) 59 | assert result.exit_code == 0 60 | assert f"Product definition added for {product}" in result.output 61 | assert "Added 4 Datasets, failed 0 Datasets, skipped 0 Datasets" in result.output 62 | 63 | # Running a second time should skip the datasets 64 | result = runner.invoke( 65 | cop_dem_to_dc_cli, 66 | [ 67 | "--add-product", 68 | "--bbox", 69 | bbox, 70 | "--product", 71 | product, 72 | "--env", 73 | env_name, 74 | ], 75 | ) 76 | assert result.exit_code == 0 77 | assert f"Product definition added for {product}" in result.output 78 | assert "Added 0 Datasets, failed 0 Datasets, skipped 4 Datasets" in result.output 79 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/test_esa_worldcover_to_dc.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | from click.testing import CliRunner 5 | 6 | from odc.apps.dc_tools.esa_worldcover_to_dc import ( 7 | _unpack_bbox, 8 | cli, 9 | get_tile_uris, 10 | URI_TEMPLATE, 11 | ) 12 | 13 | 14 | @pytest.fixture 15 | def bbox() -> str: 16 | return "5,5,7,7" 17 | 18 | 19 | @pytest.fixture 20 | def bbox_africa() -> str: 21 | return "-26.359944882003788,-47.96476498374171,64.4936701740102,38.34459242512347" 22 | 23 | 24 | def test_bboxes() -> None: 25 | bbox = "0,0,1,1" 26 | bounding_box = [float(x) for x in bbox.split(",")] 27 | assert _unpack_bbox(bounding_box) == (0, 0, 3, 3) 28 | assert len(list(get_tile_uris(bbox))) == 1 29 | 30 | bbox = "0,0,3,3" 31 | bounding_box = [float(x) for x in bbox.split(",")] 32 | assert _unpack_bbox(bounding_box) == (0, 0, 3, 3) 33 | assert len(list(get_tile_uris(bbox))) == 1 34 | 35 | bbox = "1,1,5,5" 36 | bounding_box = [float(x) for x in bbox.split(",")] 37 | assert _unpack_bbox(bounding_box) == (0, 0, 6, 6) 38 | 39 | bbox = "-98,15,-90,30" 40 | bounding_box = [float(x) for x in bbox.split(",")] 41 | assert _unpack_bbox(bounding_box) == (-99, 15, -90, 30) 42 | assert len(list(get_tile_uris(bbox))) == 15 43 | 44 | 45 | def test_get_dem_tile_uris(bbox) -> None: 46 | uris = list(get_tile_uris(bbox)) 47 | 48 | assert uris[0][0] == ( 49 | "https://esa-worldcover.s3.eu-central-1.amazonaws.com/" 50 | "v100/2020/map/ESA_WorldCover_10m_2020_v100_N03E003_Map.tif" 51 | ) 52 | 53 | print(uris) 54 | assert len(uris) == 4 55 | 56 | 57 | def test_complex_bbox(bbox_africa) -> None: 58 | uris = list(get_tile_uris(bbox_africa)) 59 | 60 | assert len(uris) == 899 61 | 62 | 63 | @pytest.fixture 64 | def mock_esa_worldcover_datasets(monkeypatch) -> None: 65 | """Replace the fetching of remote ESA WorldCover datasets with local downsampled versions""" 66 | fname_template = URI_TEMPLATE.split("/")[-1] 67 | local_template = ( 68 | "file://" 69 | + str(Path(__file__).parent.absolute()) 70 | + f"/data/esa_worldcover/{fname_template}" 71 | ) 72 | monkeypatch.setattr( 73 | "odc.apps.dc_tools.esa_worldcover_to_dc.URI_TEMPLATE", local_template 74 | ) 75 | 76 | 77 | def test_indexing_cli( 78 | bbox, odc_test_db_with_products, mock_esa_worldcover_datasets, env_name 79 | ) -> None: 80 | runner = CliRunner() 81 | result = runner.invoke( 82 | cli, 83 | [ 84 | "--bbox", 85 | bbox, 86 | "--env", 87 | env_name, 88 | ], 89 | ) 90 | assert result.exit_code == 0 91 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/test_fs_to_dc.py: -------------------------------------------------------------------------------- 1 | from click.testing import CliRunner 2 | from pathlib import Path 3 | 4 | from odc.apps.dc_tools.fs_to_dc import cli as fs_to_dc_cli 5 | 6 | TEST_DATA_FOLDER: Path = Path(__file__).parent.joinpath("data") 7 | 8 | 9 | def test_fs_to_fc_yaml(test_data_dir, env_name, odc_test_db_with_products) -> None: 10 | runner = CliRunner() 11 | result = runner.invoke( 12 | fs_to_dc_cli, 13 | [ 14 | test_data_dir, 15 | "--stac", 16 | "--glob=**/NASADEM_HGT_s56w072.stac-item.json", 17 | "--env", 18 | env_name, 19 | ], 20 | catch_exceptions=False, 21 | ) 22 | assert result.exit_code == 0 23 | 24 | 25 | def test_archive_less_mature( 26 | odc_db, env_name, test_data_dir, nrt_dsid, final_dsid 27 | ) -> None: 28 | dc = odc_db 29 | runner = CliRunner() 30 | 31 | # Index NRT dataset 32 | result = runner.invoke( 33 | fs_to_dc_cli, 34 | [ 35 | test_data_dir, 36 | "--glob=**/maturity-nrt.odc-metadata.yaml", 37 | "--archive-less-mature", 38 | "--env", 39 | env_name, 40 | ], 41 | ) 42 | assert result.exit_code == 0 43 | assert dc.index.datasets.get(final_dsid) is None 44 | assert dc.index.datasets.get(nrt_dsid).archived_time is None 45 | 46 | # Index Final dataset (autoarchiving NRT) 47 | result = runner.invoke( 48 | fs_to_dc_cli, 49 | [ 50 | test_data_dir, 51 | "--glob=**/maturity-final.odc-metadata.yaml", 52 | "--archive-less-mature", 53 | "--env", 54 | env_name, 55 | ], 56 | ) 57 | assert result.exit_code == 0 58 | assert dc.index.datasets.get(final_dsid).archived_time is None 59 | assert dc.index.datasets.get(nrt_dsid).archived_time is not None 60 | 61 | 62 | def test_dont_archive_less_mature( 63 | odc_db, env_name, test_data_dir, nrt_dsid, final_dsid 64 | ) -> None: 65 | # no archiving should be done if --archive-less-mature is not set 66 | dc = odc_db 67 | runner = CliRunner() 68 | 69 | # Index NRT dataset 70 | result = runner.invoke( 71 | fs_to_dc_cli, 72 | [ 73 | test_data_dir, 74 | "--glob=**/maturity-nrt.odc-metadata.yaml", 75 | "--env", 76 | env_name, 77 | ], 78 | ) 79 | assert result.exit_code == 0 80 | assert dc.index.datasets.get(final_dsid) is None 81 | assert dc.index.datasets.get(nrt_dsid).archived_time is None 82 | 83 | # Index Final dataset (autoarchiving NRT) 84 | result = runner.invoke( 85 | fs_to_dc_cli, 86 | [ 87 | test_data_dir, 88 | "--glob=**/maturity-final.odc-metadata.yaml", 89 | "--env", 90 | env_name, 91 | ], 92 | ) 93 | assert result.exit_code == 0 94 | assert dc.index.datasets.get(final_dsid).archived_time is None 95 | assert dc.index.datasets.get(nrt_dsid).archived_time is None 96 | 97 | 98 | def test_keep_more_mature( 99 | odc_db, env_name, test_data_dir, nrt_dsid, final_dsid 100 | ) -> None: 101 | dc = odc_db 102 | runner = CliRunner() 103 | 104 | # Index Final dataset 105 | result = runner.invoke( 106 | fs_to_dc_cli, 107 | [ 108 | test_data_dir, 109 | "--glob=**/maturity-final.odc-metadata.yaml", 110 | "--archive-less-mature", 111 | "--env", 112 | env_name, 113 | ], 114 | ) 115 | assert result.exit_code == 0 116 | assert dc.index.datasets.get(nrt_dsid) is None 117 | assert dc.index.datasets.get(final_dsid).archived_time is None 118 | 119 | # Index NRT dataset (less mature - should be skipped) 120 | result = runner.invoke( 121 | fs_to_dc_cli, 122 | [ 123 | test_data_dir, 124 | "--glob=**/maturity-nrt.odc-metadata.yaml", 125 | "--archive-less-mature", 126 | "--env", 127 | env_name, 128 | ], 129 | ) 130 | assert result.exit_code == 0 131 | assert dc.index.datasets.get(final_dsid).archived_time is None 132 | assert dc.index.datasets.get(nrt_dsid) is None 133 | -------------------------------------------------------------------------------- /apps/dc_tools/tests/test_stac_api_to_dc.py: -------------------------------------------------------------------------------- 1 | # Tests using the Click framework the stac_api-to-dc CLI tool 2 | import pytest 3 | from click.testing import CliRunner 4 | 5 | from odc.apps.dc_tools.stac_api_to_dc import cli 6 | from odc.apps.dc_tools.utils import MICROSOFT_PC_STAC_URI 7 | 8 | 9 | @pytest.mark.xfail(reason="Earth Search API has changed and now this is failing too") 10 | def test_stac_to_dc_earthsearch(odc_test_db_with_products) -> None: 11 | runner = CliRunner() 12 | result = runner.invoke( 13 | cli, 14 | [ 15 | "--catalog-href=https://earth-search.aws.element84.com/v0/", 16 | "--bbox=5,15,10,20", 17 | "--limit=10", 18 | "--collections=sentinel-s2-l2a-cogs", 19 | "--datetime=2020-08-01/2020-08-31", 20 | ], 21 | catch_exceptions=False, 22 | ) 23 | assert result.exit_code == 0 24 | assert "Added 10 Datasets, failed 0 Datasets, skipped 0 Datasets" in result.output 25 | 26 | 27 | @pytest.mark.xfail(reason="Currently failing because the USGS STAC is not up to spec") 28 | def test_stac_to_dc_usgs(odc_test_db_with_products) -> None: 29 | runner = CliRunner() 30 | result = runner.invoke( 31 | cli, 32 | [ 33 | "--catalog-href=https://ibhoyw8md9.execute-api.us-west-2.amazonaws.com/prod", 34 | "--bbox=5,15,10,20", 35 | "--limit=10", 36 | "--collections=landsat-c2l2-sr", 37 | "--datetime=2020-08-01/2020-08-31", 38 | ], 39 | catch_exceptions=False, 40 | ) 41 | assert result.exit_code == 0 42 | 43 | 44 | @pytest.mark.xfail( 45 | reason="Failing with error 'ConformanceClasses.ITEM_SEARCH not supported'" 46 | ) 47 | def test_stac_to_dc_planetarycomputer(odc_test_db_with_products) -> None: 48 | runner = CliRunner() 49 | result = runner.invoke( 50 | cli, 51 | [ 52 | f"--catalog-href={MICROSOFT_PC_STAC_URI}", 53 | "--limit=1", 54 | "--collections=nasadem", 55 | ], 56 | ) 57 | assert result.exit_code == 0 58 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: true 3 | 4 | coverage: 5 | precision: 2 6 | round: down 7 | range: "50...100" 8 | 9 | status: 10 | project: 11 | default: # This can be anything, but it needs to exist as the name 12 | # basic settings 13 | target: 50% 14 | threshold: 20% 15 | -------------------------------------------------------------------------------- /docker/.dockerignore: -------------------------------------------------------------------------------- 1 | .cache 2 | Makefile 3 | -------------------------------------------------------------------------------- /docker/.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | #syntax=docker/dockerfile:1.2 2 | ARG V_PG=16 3 | ARG V_PGIS=16-postgis-3 4 | 5 | FROM osgeo/gdal:ubuntu-small-3.6.3 6 | ENV LC_ALL=C.UTF-8 7 | ENV PATH="/env/bin:${PATH}" 8 | 9 | USER root 10 | RUN apt-get update -y \ 11 | && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ 12 | && DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y \ 13 | && DEBIAN_FRONTEND=noninteractive apt-get install -y --fix-missing --no-install-recommends --allow-change-held-packages \ 14 | # git is needed for sdist|bdist_wheel \ 15 | python3-dev python3-distutils python3-pip \ 16 | git \ 17 | # for docs 18 | build-essential \ 19 | graphviz \ 20 | # for integration tests \ 21 | libpq-dev libproj-dev \ 22 | postgresql \ 23 | postgresql-client-${V_PG} \ 24 | postgresql-${V_PG} \ 25 | postgresql-${V_PGIS} \ 26 | # for matching directory permissions when running tests as non-root user \ 27 | tini \ 28 | sudo \ 29 | && rm -rf /var/lib/apt/lists/* 30 | 31 | RUN groupadd --gid 1000 odc \ 32 | && useradd --gid 1000 \ 33 | --uid 1000 \ 34 | --create-home \ 35 | --shell /bin/bash -N odc \ 36 | && adduser odc users \ 37 | && adduser odc sudo \ 38 | && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \ 39 | && install -d -o odc -g odc /env \ 40 | && install -d -o odc -g odc /code \ 41 | && install -d -o odc -g odc -D /var/run/postgresql /srv/postgresql \ 42 | && install -d -o odc -g odc -D /home/odc/.cache/pip \ 43 | && chown -R odc:odc /home/odc/ \ 44 | && true 45 | 46 | COPY constraints.txt requirements.in /conf/ 47 | 48 | RUN python3 -m pip install -U pip cython 49 | 50 | RUN python3 -m pip install \ 51 | -r /conf/requirements.in \ 52 | -c /conf/constraints.txt 53 | 54 | 55 | # Bake in fresh empty datacube db into docker image (owned by odc user) 56 | # Need to run after environment setup as it needs datacube 57 | COPY --chown=0:0 assets/with-bootstrap /usr/local/bin/ 58 | COPY --chown=0:0 assets/with-test-db /usr/local/bin/ 59 | 60 | USER odc 61 | RUN with-bootstrap with-test-db prepare 62 | 63 | USER root 64 | 65 | WORKDIR /code 66 | ENTRYPOINT ["/bin/tini", "-s", "--", "/usr/local/bin/with-bootstrap"] 67 | -------------------------------------------------------------------------------- /docker/Makefile: -------------------------------------------------------------------------------- 1 | # Base image to use for constructing environment 2 | V_BASE ?= 3.3.0 3 | 4 | # Docker we are building 5 | DKR_IMG ?= opendatacube/odc-test-runner:latest 6 | 7 | # Absolute path for this directory 8 | WK := $(shell pwd) 9 | 10 | # Absolute path to code 11 | CODE := $(shell readlink -f ..) 12 | TTY := $(shell bash -c "tty -s && echo '-t' || true") 13 | 14 | dkr := docker run --rm -i $(TTY) \ 15 | -v $(CODE):/code \ 16 | -v $(WK):/wk \ 17 | -e TZ=Australia/Sydney \ 18 | $(BUILDER_IMG) 19 | 20 | all: dkr 21 | 22 | bash: 23 | @docker run --rm -ti \ 24 | -v $(CODE):/code \ 25 | -v $(WK):/wk \ 26 | $(DKR_IMG) bash 27 | 28 | constraints.txt: requirements.in 29 | @if [ -e /.dockerenv ]; then \ 30 | pip-compile -v \ 31 | --no-annotate \ 32 | --strip-extras \ 33 | --no-build-isolation \ 34 | --cache-dir .cache \ 35 | --output-file $@ $< ; \ 36 | else \ 37 | docker run --rm \ 38 | -v $(CODE):/code \ 39 | $(DKR_IMG) \ 40 | make -C docker $@; \ 41 | fi 42 | 43 | dkr: Dockerfile 44 | DOCKER_BUILDKIT=1 docker build \ 45 | --build-arg V_BASE=$(V_BASE) \ 46 | -t $(DKR_IMG) \ 47 | -f Dockerfile . 48 | 49 | dkr-no-deps: 50 | DOCKER_BUILDKIT=1 docker build \ 51 | --build-arg V_BASE=$(V_BASE) \ 52 | -t $(DKR_IMG) \ 53 | -f Dockerfile . 54 | 55 | run-test: 56 | @docker run --rm $(TTY) \ 57 | -v $(CODE):/code \ 58 | $(DKR_IMG) with-test-db env AWS_DEFAULT_REGION=us-west-2 DASK_TEMPORARY_DIRECTORY=/tmp/dask \ 59 | pytest --cov=. \ 60 | --cov-report=html \ 61 | --cov-report=xml:coverage.xml \ 62 | --timeout=30 \ 63 | libs apps 64 | 65 | clean: 66 | @echo "not implemented" 67 | 68 | .PHONY: dbg all clean dkr run-test dkr dkr-no-deps bash 69 | -------------------------------------------------------------------------------- /docker/assets/with-bootstrap: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Become `odc` user with UID/GID compatible to /code volume 4 | # If Running As root 5 | # If outside volume not owned by root 6 | # change `odc` to have compatible UID/GID 7 | # re-exec this script as odc user 8 | 9 | [[ $UID -ne 0 ]] || { 10 | target_uid=$(stat -c '%u' .) 11 | target_gid=$(stat -c '%g' .) 12 | 13 | [[ $target_uid -eq 0 ]] || { 14 | 15 | # unless gid already matches update gid 16 | [[ $(id -g odc) -eq ${target_gid} ]] || { 17 | groupmod --gid ${target_gid} odc 18 | usermod --gid ${target_gid} odc 19 | } 20 | 21 | # unless uid already matches: change it and update HOME and /env 22 | [[ $(id -u odc) -eq ${target_uid} ]] || { 23 | usermod --uid ${target_uid} odc 24 | chown -R odc:odc /home/odc /env /srv/postgresql /var/run/postgresql 25 | } 26 | 27 | exec sudo -u odc -E -H bash "$0" "$@" 28 | } 29 | } 30 | 31 | [[ $UID -ne 0 ]] || echo "WARNING: Running as root" 32 | 33 | cached_dev_install () { 34 | local cache_file=/code/.run/dev-install.tgz 35 | 36 | if [[ -e ${cache_file} ]]; then 37 | echo "Installing from cache" 38 | tar xvzf ${cache_file} -C / 39 | else 40 | echo "Fresh dev install" 41 | ./scripts/dev-install.sh --no-deps --no-build-isolation 42 | echo "Caching dev install to: ${cache_file}" 43 | mkdir -p $(dirname ${cache_file}) 44 | tar cvzf ${cache_file} $(find ${VIRTUAL_ENV} \( -name "*egg-link" -or -name "easy-install.pth" \)) 45 | fi 46 | } 47 | 48 | env="${PYENV:-/env}" 49 | 50 | if [ -e "${env}/bin/activate" ]; then 51 | [ -n "${VIRTUAL_ENV:-}" ] || { 52 | source "${env}/bin/activate" 53 | } 54 | fi 55 | 56 | if [[ -d /code/scripts ]]; then 57 | cached_dev_install 58 | fi 59 | 60 | [ -z "${1:-}" ] || { 61 | exec "$@" 62 | } 63 | -------------------------------------------------------------------------------- /docker/assets/with-test-db: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o pipefail 5 | set -o nounset 6 | 7 | _launch_db () { 8 | local pgdata="${1:-/srv/postgresql}" 9 | local bin=$(find /usr/lib/postgresql/ -type d -name bin) 10 | 11 | [ -e "${pgdata}/PG_VERSION" ] || { 12 | "${bin}/initdb" -D "${pgdata}" --auth-host=md5 --encoding=UTF8 13 | } 14 | 15 | "${bin}/pg_ctl" -D "${pgdata}" -l "${pgdata}/pg.log" start 16 | } 17 | 18 | _stop_db () { 19 | local pgdata="${1:-/srv/postgresql}" 20 | local bin=$(find /usr/lib/postgresql/ -type d -name bin) 21 | 22 | "${bin}/pg_ctl" -D "${pgdata}" stop 23 | } 24 | 25 | _prep_db () { 26 | psql -c '\conninfo' datacube 2> /dev/null > /dev/null || { 27 | echo "Launching DB" 28 | _launch_db 29 | } 30 | 31 | # Create `datacube` database if not present, then run `datacube system init` on it 32 | [[ $(psql -lqt | awk '$1 == "datacube" {print}' | wc -l) == 1 ]] || { 33 | echo "Initialising Test Database" 34 | createdb datacube 35 | datacube system init 36 | } 37 | } 38 | 39 | cat < $HOME/.odc_tools_datacube.conf 40 | [datacube] 41 | db_hostname: 42 | db_database: odc_tools_test 43 | index_driver: default 44 | 45 | [postgis] 46 | db_hostname: 47 | db_database: odc_tools_test 48 | index_driver: postgis 49 | 50 | EOL 51 | 52 | export ODC_CONFIG_PATH="${HOME}/.odc_tools_datacube.conf" 53 | 54 | [ -z "${1:-}" ] || { 55 | case "$1" in 56 | start) 57 | _launch_db 58 | ;; 59 | stop) 60 | _stop_db 61 | ;; 62 | prepare) 63 | # Start DB, prep it if needed then stop 64 | # used inside the docker 65 | _prep_db 66 | _stop_db 67 | ;; 68 | *) 69 | _prep_db 70 | exec "$@" 71 | ;; 72 | esac 73 | } 74 | -------------------------------------------------------------------------------- /docker/constraints.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.10 3 | # by the following command: 4 | # 5 | # pip-compile --no-annotate --no-build-isolation --output-file=constraints.txt --strip-extras requirements.in 6 | # 7 | affine==2.4.0 8 | aiobotocore==2.16.0 9 | aiohappyeyeballs==2.4.4 10 | aiohttp==3.11.11 11 | aioitertools==0.12.0 12 | aiosignal==1.3.2 13 | alembic==1.14.0 14 | antimeridian==0.4.0 15 | asttokens==3.0.0 16 | async-timeout==5.0.1 17 | attrs==24.3.0 18 | awscli==1.36.22 19 | azure-core==1.32.0 20 | azure-storage-blob==12.24.0 21 | blinker==1.9.0 22 | bokeh==3.6.2 23 | boltons==24.1.0 24 | boto3==1.35.81 25 | botocore==1.35.81 26 | bottleneck==1.4.2 27 | branca==0.8.1 28 | build==1.2.2.post1 29 | cachetools==5.5.0 30 | cattrs==24.1.2 31 | certifi==2024.12.14 32 | cffi==1.17.1 33 | charset-normalizer==3.4.0 34 | ciso8601==2.3.2 35 | click==8.1.8 36 | click-plugins==1.1.1 37 | cligj==0.7.2 38 | cloudpickle==3.1.0 39 | colorama==0.4.6 40 | comm==0.2.2 41 | contourpy==1.3.1 42 | cryptography==44.0.0 43 | dask==2024.10.0 44 | dask-expr==1.1.16 45 | dask-image==2024.5.3 46 | datacube==1.9.0 47 | datadog==0.50.2 48 | decorator==5.1.1 49 | deepdiff==8.1.1 50 | defusedxml==0.7.1 51 | deprecat==2.1.3 52 | distributed==2024.10.0 53 | docker==7.1.0 54 | docutils==0.16 55 | eodatasets3==1.9.0 56 | exceptiongroup==1.2.2 57 | executing==2.1.0 58 | fiona==1.10.1 59 | flask==3.1.0 60 | flask-cors==5.0.0 61 | frozenlist==1.5.0 62 | fsspec==2024.12.0 63 | geoalchemy2==0.16.0 64 | google-api-core==2.24.0 65 | google-auth==2.37.0 66 | google-cloud-core==2.4.1 67 | google-cloud-storage==2.19.0 68 | google-crc32c==1.6.0 69 | google-resumable-media==2.7.2 70 | googleapis-common-protos==1.66.0 71 | greenlet==3.1.1 72 | h5py==3.12.1 73 | hdstats==0.2.1 74 | idna==3.10 75 | imageio==2.36.1 76 | importlib-metadata==8.5.0 77 | iniconfig==2.0.0 78 | ipyleaflet==0.19.2 79 | ipython==8.31.0 80 | ipywidgets==8.1.5 81 | isodate==0.7.2 82 | itsdangerous==2.2.0 83 | jedi==0.19.2 84 | jinja2==3.1.5 85 | jmespath==1.0.1 86 | jsonschema==4.23.0 87 | jsonschema-specifications==2024.10.1 88 | jupyter-leaflet==0.19.2 89 | jupyter-ui-poll==1.0.0 90 | jupyterlab-widgets==3.0.13 91 | lark==1.2.2 92 | lazy-loader==0.4 93 | locket==1.0.0 94 | lxml==5.3.0 95 | lz4==4.3.3 96 | mako==1.3.8 97 | markupsafe==3.0.2 98 | matplotlib-inline==0.1.7 99 | moto==5.0.24 100 | msgpack==1.1.0 101 | multidict==6.1.0 102 | networkx==3.4.2 103 | numexpr==2.10.2 104 | numpy==1.26.4 105 | odc-geo==0.4.8 106 | orderly-set==5.2.3 107 | packaging==24.2 108 | pandas==2.2.3 109 | parso==0.8.4 110 | partd==1.4.2 111 | pexpect==4.9.0 112 | pillow==11.0.0 113 | pims==0.7 114 | pip-tools==7.4.1 115 | pluggy==1.5.0 116 | prompt-toolkit==3.0.48 117 | propcache==0.2.1 118 | proto-plus==1.25.0 119 | protobuf==5.29.2 120 | psutil==6.1.1 121 | ptyprocess==0.7.0 122 | pure-eval==0.2.3 123 | pyarrow==18.1.0 124 | pyasn1==0.6.1 125 | pyasn1-modules==0.4.1 126 | pycparser==2.22 127 | pygments==2.18.0 128 | pyparsing==3.2.0 129 | pyproj==3.7.0 130 | pyproject-hooks==1.2.0 131 | pystac==1.11.0 132 | pystac-client==0.8.5 133 | pytest==8.3.4 134 | python-dateutil==2.9.0.post0 135 | python-rapidjson==1.20 136 | pytz==2024.2 137 | pyyaml==6.0.2 138 | rasterio==1.4.3 139 | referencing==0.35.1 140 | requests==2.32.3 141 | responses==0.25.3 142 | rio-stac==0.10.1 143 | rpds-py==0.22.3 144 | rsa==4.7.2 145 | ruamel-yaml==0.18.6 146 | ruamel-yaml-clib==0.2.12 147 | s3transfer==0.10.4 148 | scikit-image==0.25.0 149 | scipy==1.14.1 150 | shapely==2.0.6 151 | six==1.17.0 152 | slicerator==1.1.0 153 | sortedcontainers==2.4.0 154 | sqlalchemy==2.0.36 155 | stack-data==0.6.3 156 | structlog==24.4.0 157 | tblib==3.0.0 158 | thredds-crawler==1.5.4 159 | tifffile==2024.12.12 160 | tomli==2.2.1 161 | toolz==1.0.0 162 | tornado==6.4.2 163 | tqdm==4.67.1 164 | traitlets==5.14.3 165 | traittypes==0.2.1 166 | typing-extensions==4.12.2 167 | tzdata==2024.2 168 | urllib3==2.3.0 169 | urlpath==1.2.0 170 | wcwidth==0.2.13 171 | werkzeug==3.1.3 172 | wheel==0.45.1 173 | widgetsnbextension==4.0.13 174 | wrapt==1.17.0 175 | xarray==2024.11.0 176 | xmltodict==0.14.2 177 | xyzservices==2024.9.0 178 | yarl==1.18.3 179 | zict==3.0.0 180 | zipp==3.21.0 181 | 182 | # The following packages are considered to be unsafe in a requirements file: 183 | # pip 184 | # setuptools 185 | -------------------------------------------------------------------------------- /docker/nobinary.txt: -------------------------------------------------------------------------------- 1 | # for now don't bother with compiling anything 2 | # but potential choices are 3 | # 4 | 5 | #aiohttp 6 | #cffi 7 | #cftime 8 | #lmdb 9 | #lxml 10 | #numexpr 11 | #numpy 12 | #pandas 13 | #pillow 14 | #pyyaml 15 | #pyzmq 16 | #sqlalchemy 17 | #tornado 18 | #zstandard 19 | -------------------------------------------------------------------------------- /docker/readme.md: -------------------------------------------------------------------------------- 1 | # Test Docker for odc-tools 2 | 3 | 1. Download all dependencies, wheels or sources into `/wheels` 4 | 2. Compile non-wheels into wheels (parallel compilation is enabled) 5 | 3. Build environment from wheels without network access 6 | 7 | ## Files 8 | 9 | - `requirements.in` 10 | - Test dependencies 11 | - Libs with constraints 12 | - Libs with extra feature flags 13 | - Optional libs 14 | 15 | - `constraints.txt` 16 | - Auto-generated do not edit manually, see below 17 | 18 | - `nobinary.txt` 19 | - Force compilation from source for selected libs (currently empty) 20 | 21 | ## Updating constraints.txt 22 | 23 | We use `pip-tools` to generate `constraints.txt` from `requirements.in` file. To get newer versions of libs run this: 24 | 25 | ``` 26 | cd docker 27 | make constraints.txt 28 | ``` 29 | 30 | Then verify that the new set of packages doesn't break things 31 | 32 | ``` 33 | make dkr 34 | make run-test 35 | ``` 36 | 37 | If all is fine check in new version of `constraints.txt`. 38 | 39 | 40 | ### Note on aibotocore 41 | 42 | - `pip` attempts to find compatible set of libs 43 | - AWS publishes new boto lib versions every few days, so there are a lot of versions 44 | - `aibotocore` depends on one specific version of `botocore, boto3` 45 | - `pip` has no chance of finding correct set without constraints 46 | 47 | This is why in `requirements.in` we specify `aiobotocore[boto3,awscli]==1.3.3`, 48 | as this limits the search space for `pip-tools` to just a specific version of 49 | `aiobotocore` and a compatible set of boto libs. 50 | -------------------------------------------------------------------------------- /docker/requirements.in: -------------------------------------------------------------------------------- 1 | # This file is used to download all dependencies 2 | # 3 | # Majority of dependencies will be pulled in via `-e /code/...`, but some 4 | # extras are needed: 5 | # - test dependencies that are not listed as direct dependencies 6 | # - libraries that need extra features enabled (dask[complete], datacube[dev]) 7 | # - optional dependencies that are needed to run tests more fully (hdstats) 8 | # - libraries that we need to constrain versions for to help pip resolutions 9 | # 10 | 11 | 12 | # other top level dependencies of odc. libraries/apps 13 | affine 14 | 15 | # need to constrain this one for pip-sake 16 | # Fixing this allows pip resolution without constraints.txt 17 | aiobotocore[boto3,awscli]>2.1 18 | aiohttp 19 | azure-storage-blob 20 | bottleneck>=1.3.5 21 | click 22 | 23 | # Make sure dask has all the features enabled 24 | dask[complete]>=2023.2.0,<2024.11.0 25 | dask_image>=2023.2.0,<2024.11.0 26 | 27 | datacube[dev]>=1.9.0 28 | datadog 29 | # things needed for tests that might not be referenced in setup.py 30 | deepdiff 31 | docker 32 | eodatasets3>=1.9.0 33 | fsspec 34 | google-cloud-storage 35 | hdstats>=0.1.7.post5 36 | ipyleaflet 37 | ipywidgets 38 | jinja2 39 | jupyter_ui_poll 40 | numexpr 41 | numpy>=1.24.0,<2 42 | pip-tools 43 | pyproj>=3.5 44 | pystac>=1.0.0 45 | pystac-client>=0.2.0 46 | pytest 47 | pyyaml 48 | rasterio>=1.3.2 49 | requests 50 | rio-stac>=0.3.1 51 | scikit-image 52 | thredds_crawler 53 | toolz 54 | tqdm 55 | urlpath 56 | xarray>=2023.9.0 57 | 58 | 59 | # Moto and Flask 60 | # Not using moto[server] because that causes json-schema conflicts 61 | moto>=4 62 | flask 63 | flask-cors 64 | -------------------------------------------------------------------------------- /docs/normalised-dataset.yaml: -------------------------------------------------------------------------------- 1 | # UUID of the dataset 2 | id: "37a8acf2-a7e9-4ba4-8412-aac6db8ae1ef" 3 | # Product name (what about auto-matching?) 4 | # Allow: "**auto**" ?? 5 | # Allow: "uuid:0410ac08-14f9-499c-b59d-c42631cf6a03" for cross-db compat ?? 6 | product: "ls5_nbar_albers" 7 | # Location is a List[URI]|URI, can be pointing to a file or a directory 8 | # - can be [] (archived datasets) 9 | location: 10 | - "file:///g/data/product/path/deep/metadata.yml" 11 | - "s3://bucket/data/path/" 12 | # DateTimeStr|Tuple[DateTimeStr,DateTimeStr] 13 | # Single timestamp or 2 element list for a range 14 | # datetime: ["2000-02-11T17:43:00", "2000-02-11T17:44:22"] 15 | datetime: "2000-02-11T17:43:00" 16 | # Projection used by all bands within this dataset 17 | # - Prefer EPSG: syntax 18 | # - Use WKT otherwise 19 | crs: "EPSG:32654" 20 | # *optional* 21 | # Polygon in native CRS: no valid pixels outside of this 22 | # polygon can be found for any band. GeoJSON polygon. 23 | # (what currently is called `valid_data`) 24 | geometry: {} #..GeoJSON.. polygon 25 | # Grids 26 | # - Must include 'default' 27 | # - All grids are expected to be defined in one projection 28 | # - If only footprint data is available use shape: [1,1] and 29 | # compute corresponding transform from the footrprint. 30 | # - shape: [Height, Width] of image plane 31 | # - transform: Affine matrix in row-major order, mapping from 32 | # pixel plane coordinates to world 33 | grids: 34 | default: 35 | shape: [7731, 7621] 36 | transform: [30.0, 0.0, 306285.0, 0.0, -30.0, -1802085.0, 0, 0, 1] 37 | ir: 38 | shape: [3865, 3810] 39 | transform: [60.0, 0.0, 306285.0, 0.0, -60.0, -1802085.0, 0, 0, 1] 40 | # Information about measurements 41 | # - path: str 42 | # - Path relative to .location 43 | # - Or fully qualified URI (for absolute file paths use file:///) 44 | # - band: int (1 based index into a file) 45 | # - layer: str|None 46 | # - variable name for netcdf/hdf4 variables 47 | # - grid: str 48 | # - Key into `grids` defined above 49 | # - Allow "unknown" if no location available? 50 | measurements: 51 | a: 52 | path: "a.tiff" 53 | band: 1 54 | layer: null 55 | grid: "default" 56 | a2: 57 | path: "a.tiff" 58 | band: 2 59 | layer: null 60 | grid: "default" 61 | b: 62 | path: "b.nc" 63 | band: 1 64 | layer: "var_b" 65 | grid: "ir" 66 | # File format for all bands? 67 | file_format: "GeoTIFF" 68 | # Lineage: Dict[str, List[UUID]] 69 | # - Lineage is for immediate parents only 70 | # - Order of UUIDs within a list is not significant 71 | # - Allows multiple parents to have same label unlike current format 72 | # This requires changing constraint on dataset_source:classifier 73 | # - Constraints interop for OLD code + new DB 74 | # - Possibly change to just str:UUID mapping for backwards compatibility if too hard 75 | lineage: 76 | nbar: [8f7ff9c7-1f8c-4c67-823b-f9f4d7d751b1] 77 | pq: [11cc9455-7ee2-47ca-9de5-e3ccc663a1ca] 78 | # *optional*, future enhancement 79 | # UUID of a previous dataset this one replaces 80 | replaces: 6f7d925e-3c93-4eaa-bf25-6bf0374fd1db 81 | # STAC compatible names for Platform/Sensor/provider/etc 82 | # Datacube aware set of properties, expected to add more in the future, 83 | # so code should skip over unknown ones without error 84 | # - All should be optional 85 | # - Prefer STAC names when available 86 | # - Prefix with `odc` otherwise, i.e. `odc:gqa` 87 | properties: 88 | eo:platform: landsat-8 89 | odc:gqa: 0.8 90 | odc:region_code: "albers:-10+20" 91 | # *optional* 92 | # Information to store in the DB on behalf of IO driver and to pass 93 | # it on to IO driver when loading data. 94 | # Example: CRS+transform Override per band 95 | io_driver_data: {} 96 | # everything else: 97 | # - user supplied, user interpreted 98 | # - can be queried via usual metadata indirection mechanism 99 | user_data: 100 | algorithm: "jumpy2" 101 | opts: [3, 2, 3, 4] 102 | -------------------------------------------------------------------------------- /docs/s3-tar.md: -------------------------------------------------------------------------------- 1 | Collate Objects from S3 2 | ======================= 3 | 4 | 1. Uses async http to have many concurrent S3 object get requests from a few python threads 5 | 2. Example tool `s3-to-tar` turns a list of urls read from `stdin` into a tar archive on `stdout` or on disk 6 | 7 | Using WOfS yamls as a sample dataset I get following performance: 8 | 9 | - Instance type `r4.xlarge`, 4 cores, 32Gb of memory 10 | - Reading 100K documents completes in about 4 minutes when running 2 concurrent tasks 11 | - That's 240s, ~420 datasets per second per worker 12 | - 840 datasets per second per instance 13 | - Output is pumped into `gzip -3` then dumped to file 14 | 15 | In comparison a simple fetch one object at a time using `boto3` is about 30 16 | small objects per second per thread. 17 | 18 | Processing a single chunk looks like this: 19 | 20 | ```bash 21 | #!/bin/bash 22 | 23 | chunk="$1" 24 | time_file="${chunk}-time.txt" 25 | tar_file="${chunk}.tgz" 26 | 27 | echo "${chunk} -> ${time_file}, ${tar_file}" 28 | exec /usr/bin/time -vv -o "${time_file}" s3-to-tar < "${chunk}" | gzip -3 > "${tar_file}" 29 | ``` 30 | 31 | Chunks were generated with this: 32 | 33 | ```bash 34 | #!/bin/bash 35 | split -d --lines 100000 ../urls.txt wofs- 36 | ``` 37 | 38 | Then finally: 39 | 40 | ```bash 41 | #!/bin/bash 42 | find . -name 'wofs-??' | sort | xargs -n 1 -P 2 ./process-chunk.sh 43 | ``` 44 | 45 | Downloading all 2.6 million WOfS meatadata documents from S3 took less than 46 | hour, this is higher throughput than the yaml parser can parse. 47 | 48 | Limitations 49 | ----------- 50 | 51 | 1. Assumes small documents 52 | - reads whole object into RAM 53 | - large internal queues 54 | -------------------------------------------------------------------------------- /libs/cloud/README.md: -------------------------------------------------------------------------------- 1 | odc.cloud 2 | ========= 3 | 4 | Various cloud access helper methods. 5 | 6 | This package ships several namespaces: 7 | 8 | - `odc.aws.` S3 and other AWS resources 9 | - `odc.aio.` S3 but async, depends (needs `[ASYNC]` feature flag, pulls in `aiobotocore`) 10 | - `odc.azure.` Azure blob storage traversal helpers (needs `[AZURE]` feature flag) 11 | - `odc.thredds.` THREDDS traversal helper method (needs `[THREDDS]` feature flag) 12 | 13 | 14 | Installation 15 | ------------ 16 | 17 | ``` 18 | pip install odc-cloud 19 | ``` 20 | 21 | Usage 22 | ----- 23 | 24 | TODO 25 | -------------------------------------------------------------------------------- /libs/cloud/odc/aws/_find.py: -------------------------------------------------------------------------------- 1 | from fnmatch import fnmatch 2 | from itertools import takewhile 3 | from types import SimpleNamespace 4 | 5 | 6 | # TODO: Document these SimpleNamespace objects that are returned! 7 | # Probably by turning them into dataclasses 8 | def s3_file_info(f, bucket): 9 | url = f"s3://{bucket}/{f.get('Key')}" 10 | return SimpleNamespace( 11 | url=url, 12 | size=f.get("Size"), 13 | last_modified=f.get("LastModified"), 14 | etag=f.get("ETag"), 15 | ) 16 | 17 | 18 | def norm_predicate(pred=None, glob=None): 19 | def glob_predicate(glob, pred): 20 | if pred is None: 21 | return lambda f: fnmatch(f.url, glob) 22 | else: 23 | return lambda f: fnmatch(f.url, glob) and pred(f) 24 | 25 | if glob is not None: 26 | return glob_predicate(glob, pred) 27 | 28 | return pred 29 | 30 | 31 | def parse_query(url_query): 32 | """ 33 | - s3://bucket/some/path/ 34 | - s3://bucket/some/path/something 35 | - s3://bucket/some/path/*/*/ 36 | - s3://bucket/some/path/*/*/file.yaml 37 | - s3://bucket/some/path/*/*/*.yaml 38 | - s3://bucket/some/path/**/file.yaml 39 | """ 40 | 41 | glob_set = set("*[]?") 42 | 43 | def is_glob(s): 44 | return bool(glob_set.intersection(set(s))) 45 | 46 | pp = url_query.split("/") 47 | base = list(takewhile(lambda s: not is_glob(s), pp)) 48 | 49 | qq = pp[len(base) :] 50 | 51 | glob, _file, depth = None, None, None 52 | 53 | if len(qq) > 0: 54 | last = qq.pop() 55 | 56 | if is_glob(last): 57 | glob = last 58 | elif last != "": 59 | _file = last 60 | 61 | qq_set = set(qq) 62 | if len(qq) == 0: 63 | depth = 0 64 | elif qq_set == {"**"}: 65 | depth = -1 66 | elif "**" not in qq_set: 67 | depth = len(qq) 68 | else: 69 | raise ValueError(f"Bad query: {url_query}") 70 | 71 | base = "/".join(base) 72 | base = base.rstrip("/") + "/" 73 | 74 | return SimpleNamespace(base=base, depth=depth, file=_file, glob=glob) 75 | -------------------------------------------------------------------------------- /libs/cloud/odc/aws/dns.py: -------------------------------------------------------------------------------- 1 | """Tools for interacting with route53""" 2 | 3 | import sys 4 | 5 | from . import _fetch_text, ec2_tags, mk_boto_session 6 | 7 | 8 | def public_ip(): 9 | return _fetch_text("http://instance-data/latest/meta-data/public-ipv4") 10 | 11 | 12 | def _find_zone_id(domain, route53): 13 | zone_name = ".".join(domain.split(".")[1:]) 14 | zone_name = zone_name.rstrip(".") + "." 15 | rr = route53.list_hosted_zones() 16 | for z in rr["HostedZones"]: 17 | if z["Name"] == zone_name: 18 | return z["Id"] 19 | 20 | return None 21 | 22 | 23 | def dns_update(domain, ip=None, route53=None, ttl=300): 24 | if route53 is None: 25 | route53 = mk_boto_session().create_client("route53") 26 | 27 | domain = domain.rstrip(".") + "." 28 | zone_id = _find_zone_id(domain, route53) 29 | 30 | if zone_id is None: 31 | return False 32 | 33 | if ip is None: 34 | ip = public_ip() 35 | 36 | update = { 37 | "Name": domain, 38 | "Type": "A", 39 | "TTL": ttl, 40 | "ResourceRecords": [{"Value": ip}], 41 | } 42 | changes = {"Changes": [{"Action": "UPSERT", "ResourceRecordSet": update}]} 43 | 44 | rr = route53.change_resource_record_sets(HostedZoneId=zone_id, ChangeBatch=changes) 45 | 46 | return rr["ResponseMetadata"]["HTTPStatusCode"] == 200 47 | 48 | 49 | def dns_delete(domain, route53=None): 50 | if route53 is None: 51 | route53 = mk_boto_session().create_client("route53") 52 | 53 | domain = domain.rstrip(".") + "." 54 | zone_id = _find_zone_id(domain, route53) 55 | 56 | if zone_id is None: 57 | return False 58 | 59 | rr = route53.list_resource_record_sets( 60 | HostedZoneId=zone_id, StartRecordType="A", StartRecordName=domain 61 | ).get("ResourceRecordSets", []) 62 | if len(rr) < 1: 63 | return False 64 | rec = rr[0] 65 | if rec.get("Name", "") != domain: 66 | return False 67 | 68 | changes = {"Changes": [{"Action": "DELETE", "ResourceRecordSet": rec}]} 69 | rr = route53.change_resource_record_sets(HostedZoneId=zone_id, ChangeBatch=changes) 70 | return rr["ResponseMetadata"]["HTTPStatusCode"] == 200 71 | 72 | 73 | def cli(args): 74 | def error(msg): 75 | print(msg, file=sys.stderr) 76 | 77 | def display_help(): 78 | print( 79 | """Modify DNS record of EC2 instance: 80 | 81 | arguments: domain_name|tag/ [auto|delete|ip] 82 | 83 | Examples: 84 | test.devbox.dea.ga.gov.au 85 | test.devbox.dea.ga.gov.au auto 86 | test.devbox.dea.ga.gov.au 3.44.10.22 87 | tag/domain auto 88 | tag/domain delete 89 | """ 90 | ) 91 | 92 | n = len(args) 93 | if n == 0: 94 | display_help() 95 | return 0 96 | elif n == 1: 97 | if args[0] in ("help", "--help"): 98 | display_help() 99 | return 0 100 | args = (args[0], "auto") 101 | elif n > 2: 102 | error("Too many arguments, expect: domain [ip|auto|delete]") 103 | return 1 104 | 105 | domain, ip = args 106 | 107 | if domain.startswith("tag/"): 108 | tag = domain[4:] 109 | tags = ec2_tags() 110 | if tags is None: 111 | error("Unable to query tags") 112 | return 2 113 | domain = tags.get(tag) 114 | if domain is None: 115 | error('No such tag: "{}"'.format(tag)) 116 | return 3 117 | 118 | if ip == "auto": 119 | ip = public_ip() 120 | if ip is None: 121 | error("Unable to find public IP of this EC2 instance") 122 | return 4 123 | if ip == "delete": 124 | print("Deleting record for domain: {}".format(domain)) 125 | ok = dns_delete(domain) 126 | else: 127 | print("Updating record for {} to {}".format(domain, ip)) 128 | ok = dns_update(domain, ip) 129 | 130 | if not ok: 131 | error("FAILED") 132 | return 1 133 | 134 | return 0 135 | 136 | 137 | if __name__ == "__main__": 138 | sys.exit(cli(sys.argv[1:])) 139 | -------------------------------------------------------------------------------- /libs/cloud/odc/aws/inventory.py: -------------------------------------------------------------------------------- 1 | from concurrent.futures import ThreadPoolExecutor, as_completed 2 | 3 | import csv 4 | import json 5 | from gzip import GzipFile 6 | from io import BytesIO 7 | from types import SimpleNamespace 8 | 9 | from . import s3_client, s3_fetch, s3_ls_dir 10 | 11 | 12 | def find_latest_manifest(prefix, s3, **kw) -> str: 13 | """ 14 | Find latest manifest 15 | """ 16 | manifest_dirs = sorted(s3_ls_dir(prefix, s3=s3, **kw), reverse=True) 17 | 18 | for d in manifest_dirs: 19 | if d.endswith("/"): 20 | leaf = d.split("/")[-2] 21 | if leaf.endswith("Z"): 22 | return d + "manifest.json" 23 | return "" 24 | 25 | 26 | def retrieve_manifest_files(key: str, s3, schema, **kw): 27 | """ 28 | Retrieve manifest file and return a namespace 29 | 30 | namespace( 31 | Bucket=, 32 | Key=, 33 | LastModifiedDate=, 34 | Size= 35 | ) 36 | """ 37 | bb = s3_fetch(key, s3=s3, **kw) 38 | gz = GzipFile(fileobj=BytesIO(bb), mode="r") 39 | csv_rdr = csv.reader(line.decode("utf8") for line in gz) 40 | for rec in csv_rdr: 41 | yield SimpleNamespace(**dict(zip(schema, rec))) 42 | 43 | 44 | def list_inventory( 45 | manifest, 46 | s3=None, 47 | prefix: str = "", 48 | suffix: str = "", 49 | contains: str = "", 50 | n_threads: int = None, 51 | **kw, 52 | ): 53 | """ 54 | Returns a generator of inventory records 55 | 56 | manifest -- s3:// url to manifest.json or a folder in which case latest one is chosen. 57 | 58 | :param manifest: (str) 59 | :param s3: (aws client) 60 | :param prefix: (str) 61 | :param suffix: (str) 62 | :param contains: (str) 63 | :param n_threads: (int) number of threads, if not sent does not use threads 64 | :return: SimpleNamespace 65 | """ 66 | # TODO: refactor parallel execution part out of this function 67 | # pylint: disable=too-many-locals 68 | s3 = s3 or s3_client() 69 | 70 | if manifest.endswith("/"): 71 | manifest = find_latest_manifest(manifest, s3, **kw) 72 | 73 | info = s3_fetch(manifest, s3=s3, **kw) 74 | info = json.loads(info) 75 | 76 | must_have_keys = {"fileFormat", "fileSchema", "files", "destinationBucket"} 77 | missing_keys = must_have_keys - set(info) 78 | if missing_keys: 79 | raise ValueError("Manifest file haven't parsed correctly") 80 | 81 | if info["fileFormat"].upper() != "CSV": 82 | raise ValueError("Data is not in CSV format") 83 | 84 | s3_prefix = "s3://" + info["destinationBucket"].split(":")[-1] + "/" 85 | data_urls = [s3_prefix + f["key"] for f in info["files"]] 86 | schema = tuple(info["fileSchema"].split(", ")) 87 | 88 | if n_threads: 89 | with ThreadPoolExecutor(max_workers=1000) as executor: 90 | tasks = [ 91 | executor.submit(retrieve_manifest_files, key, s3, schema) 92 | for key in data_urls 93 | ] 94 | 95 | for future in as_completed(tasks): 96 | for namespace in future.result(): 97 | key = namespace.Key 98 | if ( 99 | key.startswith(prefix) 100 | and key.endswith(suffix) 101 | and contains in key 102 | ): 103 | yield namespace 104 | else: 105 | for u in data_urls: 106 | for namespace in retrieve_manifest_files(u, s3, schema): 107 | key = namespace.Key 108 | if key.startswith(prefix) and key.endswith(suffix) and contains in key: 109 | yield namespace 110 | -------------------------------------------------------------------------------- /libs/cloud/odc/aws/misc.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from botocore.auth import S3SigV4Auth 3 | from botocore.awsrequest import AWSRequest 4 | from botocore.session import get_session 5 | from urllib.request import Request 6 | 7 | from . import auto_find_region, s3_url_parse 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | 12 | def s3_get_object_request_maker(region_name=None, credentials=None, ssl=True): 13 | session = get_session() 14 | 15 | if region_name is None: 16 | region_name = auto_find_region(session) 17 | 18 | if credentials is None: 19 | managed_credentials = session.get_credentials() 20 | credentials = managed_credentials.get_frozen_credentials() 21 | else: 22 | managed_credentials = None 23 | 24 | protocol = "https" if ssl else "http" 25 | auth = S3SigV4Auth(credentials, "s3", region_name) 26 | 27 | def maybe_refresh_credentials(): 28 | nonlocal credentials 29 | nonlocal auth 30 | 31 | if not managed_credentials: 32 | return 33 | 34 | creds = managed_credentials.get_frozen_credentials() 35 | if creds is credentials: 36 | return 37 | 38 | log.debug("Refreshed credentials (s3_get_object_request_maker)") 39 | 40 | credentials = creds 41 | auth = S3SigV4Auth(credentials, "s3", region_name) 42 | 43 | def build_request( 44 | bucket=None, key=None, url=None, range=None 45 | ): # pylint: disable=redefined-builtin 46 | if key is None and url is None: 47 | if bucket is None: 48 | raise ValueError("Have to supply bucket,key or url") 49 | # assume bucket is url 50 | url = bucket 51 | 52 | if url is not None: 53 | bucket, key = s3_url_parse(url) 54 | 55 | if isinstance(range, (tuple, list)): 56 | range = "bytes={}-{}".format(range[0], range[1] - 1) 57 | 58 | maybe_refresh_credentials() 59 | 60 | headers = {} 61 | if range is not None: 62 | headers["Range"] = range 63 | 64 | req = AWSRequest( 65 | method="GET", 66 | url="{}://s3.{}.amazonaws.com/{}/{}".format( 67 | protocol, region_name, bucket, key 68 | ), 69 | headers=headers, 70 | ) 71 | 72 | auth.add_auth(req) 73 | 74 | return Request(req.url, headers={**req.headers}, method="GET") 75 | 76 | return build_request 77 | -------------------------------------------------------------------------------- /libs/cloud/odc/azure.py: -------------------------------------------------------------------------------- 1 | from multiprocessing.dummy import Pool as ThreadPool 2 | 3 | from azure.storage.blob import BlobClient, ContainerClient 4 | from functools import partial 5 | from typing import List, Optional, Tuple 6 | 7 | 8 | def find_blobs( 9 | container_name: str, 10 | credential: str, 11 | prefix: str, 12 | suffix: str, 13 | account_url: Optional[str] = None, 14 | ): 15 | if account_url is not None: 16 | container = ContainerClient( 17 | account_url=account_url, 18 | container_name=container_name, 19 | credential=credential, 20 | ) 21 | else: 22 | container = ContainerClient.from_connection_string( 23 | conn_str=credential, container_name=container_name 24 | ) 25 | for blob_record in container.list_blobs(name_starts_with=prefix): 26 | blob_name = blob_record["name"] 27 | if blob_name.endswith(suffix): 28 | yield blob_name 29 | 30 | 31 | def download_yamls( 32 | account_url: str, 33 | container_name: str, 34 | credential: str, 35 | yaml_urls: List[str], 36 | workers: int = 31, 37 | ) -> List[Tuple[Optional[bytes], str, Optional[str]]]: 38 | """Download all YAML's in a list of blob names and generate content 39 | Arguments: 40 | account_url {str} -- Azure account url 41 | container_name {str} -- Azure container name 42 | credential {str} -- Azure credential token 43 | 44 | yaml_urls {list} -- List of URL's to download YAML's from 45 | workers {int} -- Number of workers to use for Thredds Downloading 46 | Returns: 47 | list -- tuples of contents and filenames 48 | """ 49 | # use a threadpool to download from Azure blobstore 50 | 51 | pool = ThreadPool(workers) 52 | yamls = pool.map( 53 | partial(download_blob, account_url, container_name, credential), yaml_urls 54 | ) 55 | pool.close() 56 | pool.join() 57 | return yamls 58 | 59 | 60 | def download_blob( 61 | account_url: Optional[str], container_name: str, credential: str, blob_name: str 62 | ) -> Tuple[Optional[bytes], str, Optional[str]]: 63 | """Internal method to download YAML's from Azure via BlobClient 64 | Arguments: 65 | account_url {str} -- Azure account url 66 | container_name {str} -- Azure container name 67 | credential {str} -- Azure credential token 68 | blob_name {str} -- Blob name to download 69 | Returns: 70 | tuple -- URL content, target file and placeholder for error 71 | """ 72 | if account_url is not None: 73 | blob = BlobClient( 74 | account_url=account_url, 75 | container_name=container_name, 76 | credential=credential, 77 | blob_name=blob_name, 78 | ) 79 | else: 80 | blob = BlobClient.from_connection_string( 81 | conn_str=credential, container_name=container_name, blob_name=blob_name 82 | ) 83 | 84 | return blob.download_blob().readall(), blob.url, None 85 | -------------------------------------------------------------------------------- /libs/cloud/odc/cloud/__init__.py: -------------------------------------------------------------------------------- 1 | from ._version import __version__ # noqa: F401 2 | -------------------------------------------------------------------------------- /libs/cloud/odc/cloud/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.2.5" 2 | -------------------------------------------------------------------------------- /libs/cloud/odc/thredds.py: -------------------------------------------------------------------------------- 1 | """Thredds crawling and YAML fetching utilities""" 2 | 3 | from multiprocessing.dummy import Pool as ThreadPool 4 | 5 | import requests 6 | from thredds_crawler.crawl import Crawl 7 | from typing import List, Optional, Tuple 8 | from urllib.parse import urlparse 9 | 10 | 11 | def thredds_find_glob( 12 | base_catalog: str, skips: List[str], select: List[str], workers: int = 8 13 | ) -> List[str]: 14 | """Glob YAML's from base Thredds Catalog recursively 15 | Arguments: 16 | base_catalog {str} -- Base of the catlog to crawl from 17 | user_skips {list} -- Paths to skip in addition to NCI specific defaults 18 | select {list} -- Paths to select (useful YAML's) 19 | workers {int} -- Number of workers to use for Thredds Crawling 20 | Returns: 21 | list -- List of Thredds hosted dataset YAML url's to Index 22 | """ 23 | user_skips = Crawl.SKIPS 24 | user_skips = user_skips.extend(skips) 25 | 26 | results = Crawl( 27 | base_catalog + "/catalog.xml", select=select, skip=user_skips, workers=workers 28 | ).datasets 29 | 30 | urls = [ 31 | service["url"] 32 | for dataset in results 33 | for service in dataset.services 34 | if service["service"].lower() == "httpserver" 35 | ] 36 | 37 | return urls 38 | 39 | 40 | def download_yamls( 41 | yaml_urls: List[str], workers: int = 8 42 | ) -> List[Tuple[Optional[bytes], str, Optional[str]]]: 43 | """Download all YAML's in a list of URL's and generate content 44 | Arguments: 45 | yaml_urls {list} -- List of URL's to download YAML's from 46 | workers {int} -- Number of workers to use for Thredds Downloading 47 | Returns: 48 | list -- tuples of contents and filenames 49 | """ 50 | # use a threadpool to download from thredds 51 | pool = ThreadPool(workers) 52 | yamls = pool.map(_download, yaml_urls) 53 | pool.close() 54 | pool.join() 55 | 56 | return yamls 57 | 58 | 59 | def _download(url: str) -> Tuple[Optional[bytes], str, Optional[str]]: 60 | """Internal method to download YAML's from thredds via requests 61 | Arguments: 62 | url {str} -- URL on thredds to download YAML for 63 | Returns: 64 | tuple -- URL content, target file and placeholder for error 65 | """ 66 | parsed_uri = urlparse(url) 67 | target_filename = url[len(parsed_uri.scheme + "://") :] 68 | try: 69 | resp = requests.get(url) 70 | if resp.status_code == 200: 71 | return resp.content, target_filename, None 72 | else: 73 | return None, target_filename, "Yaml not found" 74 | except Exception: # pylint: disable=broad-except 75 | return None, target_filename, "Thredds Failed" 76 | -------------------------------------------------------------------------------- /libs/cloud/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=51.0.0", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | -------------------------------------------------------------------------------- /libs/cloud/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = odc-cloud 3 | description = Various AWS helper methods 4 | version = attr: odc.cloud.__version__ 5 | author = Open Data Cube 6 | author_email = 7 | maintainer = Open Data Cube 8 | maintainer_email = 9 | long_description_content_type = text/markdown 10 | long_description = file: README.md 11 | platforms = any 12 | license = Apache License 2.0 13 | url = https://github.com/opendatacube/odc-tools/ 14 | 15 | [options] 16 | include_package_data = true 17 | zip_safe = false 18 | packages = find_namespace: 19 | python_requires = >=3.9 20 | tests_require = pytest 21 | install_requires = 22 | botocore 23 | boto3 24 | 25 | [options.extras_require] 26 | THREDDS = 27 | thredds_crawler 28 | requests 29 | 30 | AZURE = azure-storage-blob 31 | ASYNC = aiobotocore[boto3]>=1.0 32 | 33 | 34 | [options.packages.find] 35 | include = 36 | odc* 37 | -------------------------------------------------------------------------------- /libs/cloud/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /libs/cloud/tests/test_azure.py: -------------------------------------------------------------------------------- 1 | """Test thredds downloader code""" 2 | 3 | import pytest 4 | from odc.azure import download_yamls, find_blobs 5 | 6 | 7 | @pytest.mark.xfail(reason="Libcloud azure tests are broken") 8 | def test_find_blobs(): 9 | """Find blobs in a sample Azure account, this will fail if the blob store changes or is removed""" 10 | 11 | account_name = "geoau" 12 | account_url = "https://" + account_name + ".blob.core.windows.net" 13 | container_name = "ey-gsa" 14 | credential = "sv=2019-10-10&si=ey-gsa-ro&sr=c&sig=4T2mncGZ2v%2FqVsjjyHp%2Fv7BZytih8b251pSW0QelT98%3D" 15 | suffix = "odc-metadata.yaml" 16 | prefix = "baseline/ga_ls7e_ard_3/092/087/2018/05/25" 17 | 18 | blob_names = list( 19 | find_blobs(container_name, credential, prefix, suffix, account_url=account_url) 20 | ) 21 | assert blob_names 22 | assert len(blob_names) == 1 23 | 24 | 25 | @pytest.mark.xfail(reason="Libcloud azure tests are broken") 26 | def test_download_yamls(): 27 | """Test pass/fail arms of YAML download from Azure blobstore""" 28 | 29 | account_name = "geoau" 30 | account_url = "https://" + account_name + ".blob.core.windows.net" 31 | container_name = "ey-gsa" 32 | credential = "sv=2019-10-10&si=ey-gsa-ro&sr=c&sig=4T2mncGZ2v%2FqVsjjyHp%2Fv7BZytih8b251pSW0QelT98%3D" 33 | 34 | test_blob_names = [ 35 | "baseline/ga_ls7e_ard_3/092/087/2018/05/25/ga_ls7e_ard_3-0-0_092087_2018-05-25_final.odc-metadata.yaml" 36 | ] 37 | 38 | results = download_yamls(account_url, container_name, credential, test_blob_names) 39 | assert results 40 | assert len(results) == 1 41 | print(results) 42 | assert results[0][0] is not None 43 | -------------------------------------------------------------------------------- /libs/cloud/tests/test_thredds.py: -------------------------------------------------------------------------------- 1 | """Test thredds downloader code""" 2 | 3 | import pytest 4 | from odc.thredds import download_yamls, thredds_find_glob 5 | 6 | 7 | # It's too slow to fail, disabling this for now 8 | @pytest.mark.skip 9 | @pytest.mark.xfail 10 | def test_thredds_crawl(): 11 | """Crawl a sample Thredds URL, this will fail if NCI loses this data 12 | or Thredds is down 13 | """ 14 | thredds_catalog = "http://dapds00.nci.org.au/thredds/catalog/if87/2018-11-29/" 15 | select = [".*ARD-METADATA.yaml"] 16 | skips = [".*NBAR.*", ".*SUPPLEMENTARY.*", ".*NBART.*", ".*/QA/.*"] 17 | urls = thredds_find_glob(thredds_catalog, skips, select) 18 | assert urls 19 | assert len(urls) == 490 20 | 21 | 22 | @pytest.mark.skip 23 | @pytest.mark.xfail 24 | def test_download_yaml(): 25 | """Test pass/fail arms of YAML download from Thredds""" 26 | test_urls = [ 27 | "http://dapds00.nci.org.au/thredds/fileServer/if87/2018-11-29/S2A_OPER_" 28 | "MSI_ARD_TL_EPAE_20181129T012952_A017945_T56LLM_N02.07/ARD-METADATA.yaml", 29 | "http://dapds00.nci.org.au/thredds/fileServer/if87/2028-11-29/S2A_OPER_MSI_" 30 | "ARD_TL_EPAE_20281129T012952_A017945_T56LLM_N02.07/ARD-METADATA.yaml", 31 | "http://downtime00.nci.org.au/thredds/fileServer/if87/2018-11-29/S2A_OPER_" 32 | "MSI_ARD_TL_EPAE_20181129T012952_A017945_T56LLM_N02.07/ARD-METADATA.yaml", 33 | ] 34 | results = download_yamls(test_urls) 35 | assert results 36 | assert len(results) == 3 37 | print(results) 38 | assert results[0][0] is not None 39 | assert results[1][0] is None 40 | assert results[1][2] == "Yaml not found" 41 | assert results[2][2] == "Thredds Failed" 42 | -------------------------------------------------------------------------------- /libs/io/README.md: -------------------------------------------------------------------------------- 1 | odc.io 2 | ====== 3 | 4 | Various file IO tools. 5 | 6 | Installation 7 | ------------ 8 | 9 | ``` 10 | pip install odc-io 11 | ``` 12 | 13 | Usage 14 | ----- 15 | 16 | TODO 17 | -------------------------------------------------------------------------------- /libs/io/odc/io/__init__.py: -------------------------------------------------------------------------------- 1 | """Various file io helpers""" 2 | 3 | from .tar import tar_doc_stream # pylint: disable=W0406 4 | from .text import ( 5 | parse_mtl, # pylint: disable=W0406 6 | parse_yaml, # pylint: disable=W0406 7 | read_stdin_lines, # pylint: disable=W0406 8 | slurp, # pylint: disable=W0406 9 | slurp_lines, # pylint: disable=W0406 10 | ) 11 | from .timer import RateEstimator # pylint: disable=W0406 12 | 13 | from ._version import __version__ # pylint: disable=W0406 14 | 15 | __all__ = ( 16 | "parse_yaml", 17 | "read_stdin_lines", 18 | "slurp", 19 | "slurp_lines", 20 | "parse_mtl", 21 | "tar_doc_stream", 22 | "RateEstimator", 23 | "__version__", 24 | ) 25 | -------------------------------------------------------------------------------- /libs/io/odc/io/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.2.2" 2 | -------------------------------------------------------------------------------- /libs/io/odc/io/cgroups.py: -------------------------------------------------------------------------------- 1 | """ 2 | Query Linux cgroup fs for various info 3 | """ 4 | 5 | from typing import Optional 6 | 7 | from .text import read_int 8 | 9 | 10 | def get_cpu_quota() -> Optional[float]: 11 | """ 12 | :returns: ``None`` if unconstrained or there is an error 13 | :returns: maximum amount of CPU this pod is allowed to use 14 | """ 15 | quota = read_int("/sys/fs/cgroup/cpu/cpu.cfs_quota_us") 16 | if quota is None: 17 | return None 18 | period = read_int("/sys/fs/cgroup/cpu/cpu.cfs_period_us") 19 | if period is None: 20 | return None 21 | return quota / period 22 | 23 | 24 | def get_mem_quota() -> Optional[int]: 25 | """ 26 | :returns: ``None`` if there was some error 27 | :returns: maximum RAM, in bytes, this pod can use according to Linux cgroups 28 | 29 | Note that number returned can be larger than total available memory. 30 | """ 31 | return read_int("/sys/fs/cgroup/memory/memory.limit_in_bytes") 32 | -------------------------------------------------------------------------------- /libs/io/odc/io/tar.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from io import BytesIO 3 | import itertools 4 | import tarfile 5 | import time 6 | from pathlib import Path 7 | 8 | 9 | def tar_mode(gzip=None, xz=None, is_pipe=None): 10 | """Return tarfile.open compatible mode from boolean flags""" 11 | if gzip: 12 | return ":gz" 13 | if xz: 14 | return ":xz" 15 | if is_pipe: 16 | return "|" 17 | return "" 18 | 19 | 20 | def tar_doc_stream(fname, mode=None, predicate=None): 21 | """Read small documents (whole doc must fit into memory) from tar file. 22 | 23 | predicate : entry_info -> Bool 24 | return True for those entries that need to be read and False for those that need to be skipped. 25 | 26 | where `entry_info` is a tar entry info dictionary with keys like: 27 | name -- internal path 28 | size -- size in bytes 29 | mtime -- timestamp as and integer 30 | 31 | mode: passed on to tarfile.open(..), things like 'r:gz' 32 | 33 | 34 | Function returns iterator of tuples (name:str, data:bytes) 35 | """ 36 | if predicate: 37 | 38 | def should_skip(entry): 39 | if not entry.isfile(): 40 | return True 41 | return not predicate(entry.get_info()) 42 | 43 | else: 44 | 45 | def should_skip(entry): 46 | return not entry.isfile() 47 | 48 | def tar_open(fname, mode): 49 | if isinstance(fname, (str, Path)): 50 | open_args = [mode] if mode is not None else [] 51 | return tarfile.open(fname, *open_args) 52 | 53 | return tarfile.open(mode=mode, fileobj=fname) 54 | 55 | with tar_open(fname, mode) as tar: 56 | ee_stream = itertools.filterfalse(should_skip, tar) 57 | 58 | for entry in ee_stream: 59 | with tar.extractfile(entry) as f: 60 | buf = f.read() 61 | yield entry.name, buf 62 | 63 | 64 | def add_txt_file(tar, fname, content, mode=0o644, last_modified=None): 65 | """Add file to tar from RAM (string or bytes) + name 66 | 67 | :param tar: tar file object opened for writing 68 | :param fname: path within tar file 69 | :param content: string or bytes, content or the file to write 70 | :param mode: file permissions octet 71 | :param last_modified: file modification timestamp 72 | """ 73 | if last_modified is None: 74 | last_modified = time.time() 75 | 76 | if isinstance(last_modified, datetime.datetime): 77 | last_modified = last_modified.timestamp() 78 | 79 | info = tarfile.TarInfo(name=fname) 80 | if isinstance(content, str): 81 | content = content.encode("utf-8") 82 | info.size = len(content) 83 | info.mtime = last_modified 84 | info.mode = mode 85 | tar.addfile(tarinfo=info, fileobj=BytesIO(content)) 86 | -------------------------------------------------------------------------------- /libs/io/odc/io/timer.py: -------------------------------------------------------------------------------- 1 | from timeit import default_timer as t_now 2 | from types import SimpleNamespace 3 | 4 | 5 | class RateEstimator: 6 | def __init__(self): 7 | self.t0 = t_now() 8 | self.t_last = self.t0 9 | self.n = 0 10 | 11 | def _compute(self): 12 | dt = self.t_last - self.t0 13 | fps = 0 if self.n == 0 else self.n / dt 14 | return SimpleNamespace(elapsed=dt, n=self.n, fps=fps) 15 | 16 | def stats(self): 17 | return self._compute() 18 | 19 | def every(self, N): 20 | return (self.n % N) == 0 21 | 22 | def __call__(self, n=1): 23 | self.t_last = t_now() 24 | self.n += n 25 | 26 | def __str__(self): 27 | state = self._compute() 28 | return "N: {s.n:6,d} T: {s.elapsed:6.1f}s FPS: {s.fps:4.1f}".format(s=state) 29 | 30 | def __repr__(self): 31 | return "".format(self.t0, self.t_last, self.n) 32 | -------------------------------------------------------------------------------- /libs/io/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=51.0.0", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | -------------------------------------------------------------------------------- /libs/io/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = odc-io 3 | description = Miscellaneous file IO helper methods 4 | version = attr: odc.io.__version__ 5 | author = Open Data Cube 6 | author_email = 7 | maintainer = Open Data Cube 8 | maintainer_email = 9 | long_description_content_type = text/markdown 10 | long_description = file: README.md 11 | platforms = any 12 | license = Apache License 2.0 13 | url = https://github.com/opendatacube/odc-tools/ 14 | 15 | [options] 16 | include_package_data = true 17 | zip_safe = false 18 | packages = find_namespace: 19 | python_requires = >=3.9 20 | tests_require = pytest 21 | install_requires = 22 | 23 | [options.packages.find] 24 | include = 25 | odc* 26 | -------------------------------------------------------------------------------- /libs/io/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /libs/io/tests/test_text.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from odc.io.text import parse_mtl, parse_slice, parse_yaml, split_and_check 3 | 4 | 5 | def test_mtl(): 6 | txt = """ 7 | GROUP = a 8 | a_int = 10 9 | GROUP = b 10 | b_string = "string with spaces" 11 | GROUP = c 12 | c_float = 1.34 13 | c_int = 3 14 | END_GROUP = c 15 | b_more = 2e-4 16 | END_GROUP = b 17 | a_date = 2018-09-23 18 | END_GROUP = a 19 | END 20 | """ 21 | 22 | expect = { 23 | "a": { 24 | "a_int": 10, 25 | "a_date": "2018-09-23", 26 | "b": { 27 | "b_string": "string with spaces", 28 | "b_more": 2e-4, 29 | "c": {"c_float": 1.34, "c_int": 3}, 30 | }, 31 | } 32 | } 33 | 34 | doc = parse_mtl(txt) 35 | assert doc == expect 36 | 37 | with pytest.raises(ValueError): 38 | parse_mtl( 39 | """ 40 | GROUP = a 41 | END_GROUP = b 42 | """ 43 | ) 44 | 45 | with pytest.raises(ValueError): 46 | parse_mtl( 47 | """ 48 | GROUP = a 49 | GROUP = b 50 | END_GROUP = b 51 | END_GROUP = a 52 | END_GROUP = a 53 | """ 54 | ) 55 | 56 | # test duplicate keys: values 57 | with pytest.raises(ValueError): 58 | parse_mtl( 59 | """ 60 | a = 10 61 | a = 3 62 | """ 63 | ) 64 | 65 | # test duplicate keys: values/subtrees 66 | with pytest.raises(ValueError): 67 | parse_mtl( 68 | """ 69 | GROUP = a 70 | b = 10 71 | GROUP = b 72 | END_GROUP = b 73 | END_GROUP = a 74 | """ 75 | ) 76 | 77 | assert parse_mtl("") == {} 78 | assert parse_mtl("END") == {} 79 | 80 | 81 | def test_parse_yaml(): 82 | o = parse_yaml( 83 | """ 84 | a: 3 85 | b: foo 86 | """ 87 | ) 88 | 89 | assert o["a"] == 3 and o["b"] == "foo" 90 | assert set(o) == {"a", "b"} 91 | 92 | 93 | def test_split_check(): 94 | assert split_and_check("one/two/three", "/", 3) == ("one", "two", "three") 95 | assert split_and_check("one/two/three", "/", (3, 4)) == ("one", "two", "three") 96 | 97 | with pytest.raises(ValueError): 98 | split_and_check("a:b", ":", 3) 99 | 100 | 101 | def test_parse_slice(): 102 | from numpy import s_ 103 | 104 | assert parse_slice("::2") == s_[::2] 105 | assert parse_slice("1:") == s_[1:] 106 | assert parse_slice("1:4") == s_[1:4] 107 | assert parse_slice("1:4:2") == s_[1:4:2] 108 | assert parse_slice("1::2") == s_[1::2] 109 | -------------------------------------------------------------------------------- /libs/ui/README.md: -------------------------------------------------------------------------------- 1 | odc.ui 2 | ====== 3 | 4 | Notebook display helper methods 5 | 6 | Installation 7 | ------------ 8 | 9 | ``` 10 | pip install odc-ui 11 | ``` 12 | 13 | Make sure `ipywidgets` and `ipyleaflet` are installed and enabled: 14 | 15 | ``` 16 | jupyter nbextension enable --py widgetsnbextension 17 | jupyter nbextension enable --py ipyleaflet 18 | ``` 19 | 20 | If using jupyter lab do this as well: 21 | 22 | ``` 23 | jupyter labextension install --no-build @jupyter-widgets/jupyterlab-manager 24 | jupyter labextension install --no-build jupyter-leaflet 25 | jupyter lab build 26 | ``` 27 | 28 | Usage 29 | ----- 30 | 31 | TODO 32 | -------------------------------------------------------------------------------- /libs/ui/odc/ui/__init__.py: -------------------------------------------------------------------------------- 1 | """Notebook display helper methods.""" 2 | 3 | from ._dc_explore import ( 4 | DcViewer, 5 | ) 6 | from ._images import ( 7 | to_rgba, 8 | image_shape, 9 | image_aspect, 10 | mk_data_uri, 11 | to_png_data, 12 | to_jpeg_data, 13 | mk_image_overlay, 14 | ) 15 | from ._map import ( 16 | dss_to_geojson, 17 | gridspec_to_geojson, 18 | zoom_from_bbox, 19 | show_datasets, 20 | mk_map_region_selector, 21 | select_on_a_map, 22 | ) 23 | from ._ui import ( 24 | ui_poll, 25 | with_ui_cbk, 26 | simple_progress_cbk, 27 | ) 28 | from ._version import __version__ 29 | 30 | __all__ = ( 31 | "ui_poll", 32 | "with_ui_cbk", 33 | "simple_progress_cbk", 34 | "dss_to_geojson", 35 | "gridspec_to_geojson", 36 | "zoom_from_bbox", 37 | "show_datasets", 38 | "mk_map_region_selector", 39 | "select_on_a_map", 40 | "to_rgba", 41 | "image_shape", 42 | "image_aspect", 43 | "mk_data_uri", 44 | "to_png_data", 45 | "to_jpeg_data", 46 | "mk_image_overlay", 47 | "DcViewer", 48 | "__version__", 49 | ) 50 | -------------------------------------------------------------------------------- /libs/ui/odc/ui/_cmaps.py: -------------------------------------------------------------------------------- 1 | """ 2 | Some color map data 3 | """ 4 | 5 | # flake8: noqa 6 | import numpy as np 7 | 8 | scl_colormap = np.array( 9 | [ 10 | [255, 0, 255, 255], # 0 - NODATA 11 | [255, 0, 4, 255], # 1 - Saturated or Defective 12 | [0, 0, 0, 255], # 2 - Dark Areas 13 | [97, 97, 97, 255], # 3 - Cloud Shadow 14 | [3, 139, 80, 255], # 4 - Vegetation 15 | [192, 132, 12, 255], # 5 - Bare Ground 16 | [21, 103, 141, 255], # 6 - Water 17 | [117, 0, 27, 255], # 7 - Unclassified 18 | [208, 208, 208, 255], # 8 - Cloud 19 | [244, 244, 244, 255], # 9 - Definitely Cloud 20 | [195, 231, 240, 255], # 10 - Thin Cloud 21 | [222, 157, 204, 255], # 11 - Snow or Ice 22 | ], 23 | dtype="uint8", 24 | ) 25 | -------------------------------------------------------------------------------- /libs/ui/odc/ui/_ui.py: -------------------------------------------------------------------------------- 1 | """Notebook display helper methods.""" 2 | 3 | from IPython.display import display 4 | from ipywidgets import HBox, IntProgress, Label, Layout, VBox 5 | from jupyter_ui_poll import run_ui_poll_loop 6 | from timeit import default_timer as t_now 7 | 8 | 9 | def mk_cbk_ui(width="100%"): 10 | """Create ipywidget and a callback to pass to `dc.load(progress_cbk=..)` 11 | 12 | :param width: Width of the UI, for example: '80%' '200px' '30em' 13 | """ 14 | 15 | pbar = IntProgress(min=0, max=100, value=0, layout=Layout(width="100%")) 16 | lbl_right = Label("") 17 | lbl_left = Label("") 18 | info = HBox([lbl_left, lbl_right], layout=Layout(justify_content="space-between")) 19 | 20 | ui = VBox([info, HBox([pbar])], layout=Layout(width=width)) 21 | 22 | t0 = t_now() 23 | 24 | def cbk(n, ntotal): 25 | elapsed = t_now() - t0 26 | 27 | pbar.max = ntotal 28 | pbar.value = n 29 | 30 | lbl_right.value = "{:d} of {:d}".format(n, ntotal) 31 | lbl_left.value = "FPS: {:.1f} ({:0.1f} s remaining)".format( 32 | n / elapsed, elapsed / n * (ntotal - n) 33 | ) 34 | 35 | return ui, cbk 36 | 37 | 38 | def with_ui_cbk(width="100%", **kwargs): 39 | """Use this inside notebook like so: 40 | 41 | dc.load(..., progress_cbk=with_ui_cbk()) 42 | 43 | :param width: Width of the UI, for example: '80%' '200px' '30em' 44 | """ 45 | ui, cbk = mk_cbk_ui(width=width, **kwargs) 46 | display(ui) 47 | return cbk 48 | 49 | 50 | def simple_progress_cbk(n, total): 51 | print("\r{:4d} of {:4d}".format(n, total), end="", flush=True) 52 | 53 | 54 | def ui_poll(f, sleep=0.02, n=1): 55 | return run_ui_poll_loop(f, sleep, n=n) 56 | -------------------------------------------------------------------------------- /libs/ui/odc/ui/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.2.1" 2 | -------------------------------------------------------------------------------- /libs/ui/odc/ui/plt_tools.py: -------------------------------------------------------------------------------- 1 | """ 2 | Various data visualisation helper methods 3 | """ 4 | 5 | from matplotlib import pyplot as plt 6 | 7 | from ._cmaps import scl_colormap 8 | 9 | __all__ = ( 10 | "scl_colormap", 11 | "compare_masks", 12 | ) 13 | 14 | 15 | def compare_masks( 16 | a, b, names=("A", "B"), figsize=None, cmap="bone", interpolation="nearest", **kw 17 | ): 18 | """ 19 | Plot two similar mask images on one figure, typically B is a transformed 20 | version of A. 21 | 22 | [ A | B ] 23 | [added | removed] 24 | 25 | Where 26 | - ``added`` are pixels that turned True from A->B 27 | - ``removed`` are pixels that turned False from A->B 28 | """ 29 | fig, axs = plt.subplots(2, 2, figsize=figsize) 30 | opts = {"interpolation": interpolation, "cmap": cmap, **kw} 31 | 32 | axs[0][0].set_title(names[0]) 33 | axs[0][0].imshow(a, **opts) 34 | 35 | axs[0][1].set_title(names[1]) 36 | axs[0][1].imshow(b, **opts) 37 | 38 | axs[1][0].set_title("Added") 39 | axs[1][0].imshow((~a) * b, **opts) 40 | 41 | axs[1][1].set_title("Removed") 42 | axs[1][1].imshow(a * (~b), **opts) 43 | 44 | axs[0][0].xaxis.set_visible(False) 45 | axs[0][1].xaxis.set_visible(False) 46 | axs[0][1].yaxis.set_visible(False) 47 | axs[1][1].yaxis.set_visible(False) 48 | 49 | fig.tight_layout(pad=0) 50 | return fig, axs 51 | -------------------------------------------------------------------------------- /libs/ui/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=51.0.0", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | -------------------------------------------------------------------------------- /libs/ui/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = odc-ui 3 | description = Notebook display helper methods 4 | version = attr: odc.ui._version.__version__ 5 | author = Open Data Cube 6 | author_email = 7 | maintainer = Open Data Cube 8 | maintainer_email = 9 | long_description_content_type = text/markdown 10 | long_description = file: README.md 11 | platforms = any 12 | license = Apache License 2.0 13 | url = https://github.com/opendatacube/odc-tools/ 14 | 15 | [options] 16 | include_package_data = true 17 | zip_safe = false 18 | packages = find_namespace: 19 | python_requires = >=3.9 20 | tests_require = pytest 21 | install_requires = 22 | datacube>=1.9.0-rc9 23 | ipyleaflet 24 | ipython 25 | ipywidgets>=8.0 26 | jupyter_ui_poll 27 | matplotlib 28 | numpy 29 | odc-algo 30 | odc-geo 31 | pandas 32 | rasterio 33 | xarray 34 | 35 | [options.packages.find] 36 | include = 37 | odc* 38 | -------------------------------------------------------------------------------- /libs/ui/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | python_version = 3.8 3 | ignore_missing_imports = True 4 | allow_redefinition = True 5 | -------------------------------------------------------------------------------- /notebooks/README.md: -------------------------------------------------------------------------------- 1 | Rendered versions of some of these are available: 2 | 3 | 4 | - Displaying datacube loaded data as an image on a map 5 | - [show-image-on-a-map](https://nbviewer.jupyter.org/urls/s3-ap-southeast-2.amazonaws.com/ga-aws-dea-dev-users/kk/show-image-on-a-map.ipynb) 6 | - Inspect Datasets in the Cube 7 | - [dc-explore](https://nbviewer.jupyter.org/urls/ga-aws-dea-dev-users.s3.amazonaws.com/kk/dc-explore.ipynb) 8 | - [![Video of notebook in action](https://ga-aws-dea-dev-users.s3.amazonaws.com/kk/dc-explore-demo.gif)](https://ga-aws-dea-dev-users.s3.amazonaws.com/kk/dc-explore-demo.mp4) 9 | - Configuring Dask cluster for efficient S3 access 10 | - [dask-dc-load](https://nbviewer.jupyter.org/urls/s3-ap-southeast-2.amazonaws.com/ga-aws-dea-dev-users/kk/dask-dc-load.ipynb) 11 | - Includes some neat data visualisations as well 12 | 13 | -------------------------------------------------------------------------------- /notebooks/dc-explore.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from odc.ui import DcViewer\n", 10 | "from datacube import Datacube\n", 11 | "\n", 12 | "dc = Datacube(env='s2')\n", 13 | "\n", 14 | "dcv = DcViewer(\n", 15 | " dc, \n", 16 | " time='2019-04-24',\n", 17 | " zoom=3,\n", 18 | " center=(-24, 135),\n", 19 | " height='500px', width='800px',\n", 20 | " products='non-empty',\n", 21 | " style={'fillOpacity': 0.05,\n", 22 | " 'color': 'teal',\n", 23 | " 'weight': 0.7})\n", 24 | "dcv" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "-------------------------------------------------------------------------" 32 | ] 33 | } 34 | ], 35 | "metadata": { 36 | "kernelspec": { 37 | "display_name": "Python 3", 38 | "language": "python", 39 | "name": "python3" 40 | }, 41 | "language_info": { 42 | "codemirror_mode": { 43 | "name": "ipython", 44 | "version": 3 45 | }, 46 | "file_extension": ".py", 47 | "mimetype": "text/x-python", 48 | "name": "python", 49 | "nbconvert_exporter": "python", 50 | "pygments_lexer": "ipython3", 51 | "version": "3.6.7" 52 | } 53 | }, 54 | "nbformat": 4, 55 | "nbformat_minor": 2 56 | } 57 | -------------------------------------------------------------------------------- /notebooks/dc-load-progress-bar.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "%matplotlib inline\n", 10 | "from matplotlib import pyplot as plt\n", 11 | "import numpy as np\n", 12 | "\n", 13 | "from odc.ui import with_ui_cbk, show_datasets, to_rgba, image_aspect\n", 14 | "import datacube\n", 15 | "from datacube.utils.rio import set_default_rio_config\n", 16 | "\n", 17 | "set_default_rio_config(aws={'region_name': 'auto'},\n", 18 | " cloud_defaults=True,\n", 19 | " GDAL_INGESTED_BYTES_AT_OPEN=16*1024)" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "PRODUCT = 'ls8_nbart_geomedian_annual'\n", 29 | "NATIVE_RES = (-25, 25)\n", 30 | "CRS = 'EPSG:3577'\n", 31 | "\n", 32 | "query = dict(lat=(-38, -26),\n", 33 | " lon=(134, 141),\n", 34 | " time=('2017-01-01', '2017-12-31'))\n", 35 | "\n", 36 | "dc = datacube.Datacube(env='gm')" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "dss = dc.find_datasets(product=PRODUCT, **query)\n", 46 | "\n", 47 | "show_datasets(dss)" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "%%time\n", 57 | "xx = dc.load(product=PRODUCT,\n", 58 | " **query,\n", 59 | " output_crs=CRS,\n", 60 | " measurements=['red', 'green', 'blue'],\n", 61 | " resolution=tuple(32*n for n in NATIVE_RES),\n", 62 | " progress_cbk=with_ui_cbk())" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "to_rgba(xx, clamp=3000).isel(time=0).plot.imshow(\n", 72 | " size=8, \n", 73 | " aspect=image_aspect(xx),\n", 74 | " add_colorbar=False,\n", 75 | " add_labels=False,\n", 76 | " interpolation='bilinear');" 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "-----------------------------------------------------------------------" 84 | ] 85 | } 86 | ], 87 | "metadata": { 88 | "kernelspec": { 89 | "display_name": "Python 3", 90 | "language": "python", 91 | "name": "python3" 92 | }, 93 | "language_info": { 94 | "codemirror_mode": { 95 | "name": "ipython", 96 | "version": 3 97 | }, 98 | "file_extension": ".py", 99 | "mimetype": "text/x-python", 100 | "name": "python", 101 | "nbconvert_exporter": "python", 102 | "pygments_lexer": "ipython3", 103 | "version": "3.6.7" 104 | } 105 | }, 106 | "nbformat": 4, 107 | "nbformat_minor": 2 108 | } 109 | -------------------------------------------------------------------------------- /notebooks/s3-fetch-example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import itertools\n", 10 | "from odc.aio import S3Fetcher" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": {}, 16 | "source": [ 17 | "## Construct fetcher object" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 2, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "s3 = S3Fetcher()" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "`s3` is a callable that accepts a sequence of urls and generates a sequence of result objects with fields\n", 34 | "\n", 35 | "- `url` -- requested url\n", 36 | "- `data` -- bytes\n", 37 | "- `last_modified` -- timestamp of the object\n", 38 | "- `range=None` -- optional, range of bytes if requested partial read\n", 39 | "- `error=None` -- on error this contains an exception object\n", 40 | "\n", 41 | "\n", 42 | "Note that output order will not be the same as input order, you can not assume one to one correspondence between input and output sequences." 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "## Get some urls to fetch\n", 50 | "\n", 51 | "Get 100 urls pointing to yaml documents for S2A/B NRT." 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 3, 57 | "metadata": {}, 58 | "outputs": [ 59 | { 60 | "name": "stdout", 61 | "output_type": "stream", 62 | "text": [ 63 | "CPU times: user 2.68 s, sys: 348 ms, total: 3.03 s\n", 64 | "Wall time: 2.88 s\n" 65 | ] 66 | } 67 | ], 68 | "source": [ 69 | "%%time\n", 70 | "urls = (o.url for o in s3.find('s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/', glob='*yaml'))\n", 71 | "urls = itertools.islice(urls, 100)\n", 72 | "urls = list(urls)" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 4, 78 | "metadata": {}, 79 | "outputs": [ 80 | { 81 | "data": { 82 | "text/plain": [ 83 | "(100,\n", 84 | " ['s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2018-08-12/S2B_OPER_MSI_ARD_TL_EPAE_20180813T012421_A007492_T56LPM_N02.06/ARD-METADATA.yaml',\n", 85 | " 's3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2018-08-12/S2B_OPER_MSI_ARD_TL_EPAE_20180813T012421_A007492_T56LPN_N02.06/ARD-METADATA.yaml',\n", 86 | " 's3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2018-08-12/S2B_OPER_MSI_ARD_TL_EPAE_20180813T012421_A007492_T56LPP_N02.06/ARD-METADATA.yaml'])" 87 | ] 88 | }, 89 | "execution_count": 4, 90 | "metadata": {}, 91 | "output_type": "execute_result" 92 | } 93 | ], 94 | "source": [ 95 | "len(urls), urls[:3]" 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "We didn't need to wait for `s3.find` to finish, as fetcher accepts an iterator, we could just pass in the sequence coming out of `s3.find` directly to the fetcher. This was mainly done to understand relative costs of the two operations." 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": 5, 108 | "metadata": {}, 109 | "outputs": [ 110 | { 111 | "name": "stdout", 112 | "output_type": "stream", 113 | "text": [ 114 | "CPU times: user 459 ms, sys: 40.8 ms, total: 500 ms\n", 115 | "Wall time: 790 ms\n" 116 | ] 117 | } 118 | ], 119 | "source": [ 120 | "%%time\n", 121 | "rr = list(s3(urls))" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": 6, 127 | "metadata": {}, 128 | "outputs": [ 129 | { 130 | "data": { 131 | "text/plain": [ 132 | "(100, datetime.datetime(2018, 8, 13, 7, 12, 11, tzinfo=tzutc()), 23877, bytes)" 133 | ] 134 | }, 135 | "execution_count": 6, 136 | "metadata": {}, 137 | "output_type": "execute_result" 138 | } 139 | ], 140 | "source": [ 141 | "r = rr[0]\n", 142 | "len(rr), r.last_modified, len(r.data), type(r.data)" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": 7, 148 | "metadata": {}, 149 | "outputs": [ 150 | { 151 | "name": "stdout", 152 | "output_type": "stream", 153 | "text": [ 154 | "algorithm_information:\n", 155 | " algorithm_version: 2.0\n", 156 | " arg25_doi: http://dx.doi.org/10.4225/25/5487CC0D4F40B\n", 157 | " nbar_doi: http://dx.doi.org/10.1109/JSTARS.2010.2042281\n", 158 | " nbar_terrain_corrected_doi: http://dx.doi.org/10.1016/j.rse.2012.06.018\n", 159 | "extent:\n", 160 | " center_dt: '2018-08-12T23:57:34.459Z'\n", 161 | " coord:\n", 162 | " ll:\n", 163 | " lat: -12.750842155509027 \n", 164 | "...\n" 165 | ] 166 | } 167 | ], 168 | "source": [ 169 | "txt = r.data.decode('utf8')\n", 170 | "txt = '\\n'.join(txt.splitlines()[:10])\n", 171 | "print(txt, '\\n...')" 172 | ] 173 | } 174 | ], 175 | "metadata": { 176 | "kernelspec": { 177 | "display_name": "Python 3", 178 | "language": "python", 179 | "name": "python3" 180 | }, 181 | "language_info": { 182 | "codemirror_mode": { 183 | "name": "ipython", 184 | "version": 3 185 | }, 186 | "file_extension": ".py", 187 | "mimetype": "text/x-python", 188 | "name": "python", 189 | "nbconvert_exporter": "python", 190 | "pygments_lexer": "ipython3", 191 | "version": "3.6.5" 192 | } 193 | }, 194 | "nbformat": 4, 195 | "nbformat_minor": 2 196 | } 197 | -------------------------------------------------------------------------------- /notebooks/s3-inventory-example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from odc.aws import s3_client\n", 10 | "from odc.aws.inventory import list_inventory" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "manifest = 's3://dea-public-data-inventory/dea-public-data/dea-public-data-csv-inventory/'\n", 20 | "manifest += '2018-10-13T08-00Z/manifest.json' ## force for now, because of dev account permissions" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "s3 = s3_client()\n", 30 | "\n", 31 | "full_inventory = list_inventory(manifest, s3=s3)" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": null, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "%%time\n", 41 | "total_sz, n = 0,0\n", 42 | "\n", 43 | "for i in full_inventory:\n", 44 | " n += 1\n", 45 | " total_sz += int(i.Size)\n", 46 | " if (n%100_000) == 0:\n", 47 | " print('.', end='', flush=True)" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "print('Found {:,d} objects, total sz: {:,.1f} GiB'.format(n, total_sz/(1<<30)))" 57 | ] 58 | } 59 | ], 60 | "metadata": { 61 | "kernelspec": { 62 | "display_name": "Python 3", 63 | "language": "python", 64 | "name": "python3" 65 | }, 66 | "language_info": { 67 | "codemirror_mode": { 68 | "name": "ipython", 69 | "version": 3 70 | }, 71 | "file_extension": ".py", 72 | "mimetype": "text/x-python", 73 | "name": "python", 74 | "nbconvert_exporter": "python", 75 | "pygments_lexer": "ipython3", 76 | "version": "3.6.6" 77 | } 78 | }, 79 | "nbformat": 4, 80 | "nbformat_minor": 2 81 | } 82 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.pytest.ini_options] 2 | filterwarnings = [ 3 | "ignore:datetime.datetime.utcnow*:DeprecationWarning:botocore.*" 4 | ] 5 | -------------------------------------------------------------------------------- /scripts/bootstrap-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # ppa:ubuntugis/ppa 4 | # to build needs: 5 | # apt install libgdal-dev libudunits2-0 6 | # to run: 7 | # apt install gdal-data libgdal20 libudunits2-0 8 | 9 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 10 | 11 | set -eux 12 | 13 | exit_with_error () { 14 | echo "$1" 15 | exit 1 16 | } 17 | 18 | env_dir=${1:-"./test_env"} 19 | 20 | [ -d "${env_dir}" ] || python3 -m venv "${env_dir}" 21 | [ -f "${env_dir}/bin/activate" ] || exit_with_error "Not a valid python environment: ${env_dir}" 22 | 23 | source "${env_dir}/bin/activate" 24 | echo "Using python: $(which python)" 25 | python -m pip install --upgrade pip 26 | python -m pip install wheel 27 | python -m pip install 'aiobotocore[boto3]' 28 | python -m pip install GDAL=="$(gdal-config --version)" 29 | python -m pip install datacube 30 | 31 | if [ "${2:-}" == "dev" ]; then 32 | "${SCRIPT_DIR}/dev-install.sh" 33 | else 34 | wheel_dir="./wheels" 35 | "${SCRIPT_DIR}/build-wheels.sh" "${wheel_dir}" 36 | for w in $(find "${wheel_dir}" -type f -name "*whl"); do 37 | pip install --find-links "${wheel_dir}" "$w" 38 | done 39 | fi 40 | -------------------------------------------------------------------------------- /scripts/build-wheels.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eux 4 | 5 | WHEEL_DIR=${1:-"$(pwd)/wheels"} 6 | WHEEL_DIR=$(readlink -f ${WHEEL_DIR}) 7 | mkdir -p "${WHEEL_DIR}" 8 | # find all folders under apps and libs that have `setup.py` file in them 9 | PP=$(find libs apps -type f -name setup.py -exec dirname '{}' ';') 10 | 11 | for p in $PP; do 12 | echo "Building in ${p}" 13 | (cd "${p}" && \ 14 | python setup.py bdist_wheel --dist-dir "${WHEEL_DIR}" && \ 15 | python setup.py sdist --dist-dir "${WHEEL_DIR}" 16 | ) 17 | done 18 | 19 | echo "Wheels are in: ${WHEEL_DIR}" 20 | -------------------------------------------------------------------------------- /scripts/dev-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Perform a development (or editable) install of all the libraries and apps contained 3 | # within the odc-tools repository. 4 | 5 | set -eu 6 | 7 | mk_edit_requirements () { 8 | for d in $(find $(pwd)/libs $(pwd)/apps -name "setup.py" -type f -exec dirname '{}' ';'); do 9 | echo "-e $d" 10 | done 11 | } 12 | 13 | install_all_in_edit_mode () { 14 | local reqs=$(mktemp /tmp/requirements-XXXX.txt) 15 | 16 | # List all libs in -e /path/to/lib mode 17 | # this should let pip find all the odc- dependencies locally 18 | mk_edit_requirements >> "${reqs}" 19 | python3 -m pip install -r "${reqs}" $@ 20 | 21 | rm "${reqs}" 22 | } 23 | 24 | install_all_in_edit_mode $@ 25 | -------------------------------------------------------------------------------- /scripts/mk-pip-tree.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | create_pip_tree() { 6 | local src="${1:-.}" 7 | local dst="${2:-.}" 8 | dst=$(readlink -f "${dst}") 9 | 10 | for w in $(cd "$src" && find . -name "*.whl" | awk -F - '{sub("^./", ""); print $1}') 11 | do 12 | local base="${w//_/-}" 13 | local out="${dst}/${base}" 14 | mkdir -p "${out}" 15 | 16 | echo "${src}/${w}"*.whl "-> ${out}/" 17 | echo "${src}/${w}"*tar.gz "-> ${out}/" 18 | cp "${src}/${w}"*.whl "${out}/" 19 | cp "${src}/${w}"*.tar.gz "${out}/" 20 | done 21 | } 22 | 23 | create_pip_tree "$@" 24 | -------------------------------------------------------------------------------- /scripts/patch_version.py: -------------------------------------------------------------------------------- 1 | import re 2 | import sys 3 | 4 | from packaging import version 5 | 6 | version_rgx = re.compile(r"^\s*__version__\s*=\s*['\"]([^'\"]*)['\"]") 7 | 8 | 9 | def match_version(line): 10 | mm = version_rgx.match(line) 11 | if mm is None: 12 | return None 13 | (version_str,) = mm.groups() 14 | return version_str 15 | 16 | 17 | def mk_dev_version(v, build_number): 18 | *fixed, last = version.parse(v).release 19 | next_version = (*fixed, f"{last+1:d}-dev{build_number:d}") 20 | return ".".join(map(str, next_version)) 21 | 22 | 23 | def patch_version_lines(lines, build_number): 24 | for line in lines: 25 | v_prev = match_version(line) 26 | if v_prev is not None: 27 | v_next = mk_dev_version(v_prev, build_number) 28 | line = line.replace(v_prev, v_next) 29 | yield line 30 | 31 | 32 | def patch_file(fname, build_number): 33 | with open(fname, "rt", encoding="utf8") as src: 34 | lines = list(patch_version_lines(src, build_number)) 35 | with open(fname, "wt", encoding="utf8") as dst: 36 | dst.writelines(lines) 37 | 38 | 39 | if __name__ == "__main__": 40 | args = sys.argv[1:] 41 | if len(args) < 2: 42 | print(f"Usage: {sys.argv[0]} build-number [FILE]...") 43 | 44 | build_number, *files = args 45 | build_number = int(build_number) 46 | for f in files: 47 | patch_file(f, build_number) 48 | -------------------------------------------------------------------------------- /scripts/setup-test-db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | function start_db() { 6 | { 7 | pg_ctl -D ${pgdata} -l "${pgdata}/pg.log" start 8 | } || { 9 | # a common reason for failure is that there is already something running on port 5432 10 | # if that's the case, kill that process and retry 11 | # if not or it still fails, print the log 12 | if [[ "$(<${pgdata}/pg.log)" =~ .*"Address already in use".* ]]; then 13 | sudo kill -9 $(sudo lsof -i:5432 | awk 'NR==2 {print $2}') 14 | pg_ctl -D ${pgdata} -l "${pgdata}/pg.log" start || cat "${pgdata}/pg.log" 15 | else 16 | cat "${pgdata}/pg.log" 17 | exit 1 18 | fi 19 | } 20 | } 21 | 22 | if [ -d "$(pwd)/.dbdata" ]; then 23 | rm -rf "$(pwd)/.dbdata" 24 | fi 25 | 26 | export ODC_DATACUBE_DB_URL=postgresql:///datacube 27 | pgdata=$(pwd)/.dbdata 28 | initdb -D ${pgdata} --auth-host=md5 --encoding=UTF8 29 | start_db 30 | createdb datacube 31 | datacube system init 32 | # add any new metadata types 33 | # datacube metadata add "https://raw.githubusercontent.com/GeoscienceAustralia/dea-config/master/product_metadata/eo3_sentinel_ard.odc-type.yaml" 34 | datacube metadata add apps/dc_tools/tests/data/eo3_sentinel_ard.odc-type.yaml 35 | -------------------------------------------------------------------------------- /scripts/sync-publish-branch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o noclobber # Avoid overlay files (echo "hi" > foo) 4 | set -o errexit # Used to exit upon error, avoiding cascading errors 5 | set -o pipefail # Unveils hidden failures 6 | set -o nounset # Exposes unset variables 7 | 8 | B=pypi/publish 9 | 10 | _workdir=$(mktemp -d /tmp/odc-tools-XXXXX) 11 | echo "Clone to $_workdir" 12 | git clone --branch $B git@github.com:opendatacube/odc-tools.git $_workdir 13 | 14 | echo "Fast forward '$B' to match 'develop' branch" 15 | cd $_workdir 16 | git pull --ff-only origin develop 17 | 18 | read -p "Push to GitHub? [y/n]" -n1 ok 19 | echo "" 20 | case $ok in 21 | y | Y) 22 | echo " ok, pushing" 23 | git push origin pypi/publish 24 | ;; 25 | *) 26 | echo " ok, won't push to GitHub" 27 | ;; 28 | esac 29 | 30 | read -p "Clean up $_workdir [y/n]" -n1 ok 31 | echo "" 32 | case $ok in 33 | y | Y) 34 | echo " deleting checkout: ${_workdir} in 5 seconds" 35 | sleep 5 36 | rm -rfv "${_workdir}" 37 | ;; 38 | *) 39 | echo " ok, leaving files in: ${_workdir}" 40 | ;; 41 | esac 42 | -------------------------------------------------------------------------------- /tests/test-env.yml: -------------------------------------------------------------------------------- 1 | # Conda environment for running tests in odc-tools 2 | # mamba env create -f test-env.yml 3 | # conda activate odc-tools-tests 4 | 5 | name: odc-tools-tests 6 | channels: 7 | - conda-forge 8 | 9 | dependencies: 10 | - python=3.12 11 | 12 | # Datacube 13 | # - datacube[postgres]>=1.9.0 14 | - sqlalchemy>=2.0 15 | 16 | # odc.ui 17 | - ipywidgets>=8.0 18 | - ipyleaflet 19 | - tqdm 20 | 21 | # odc-apps-dc-tools 22 | - pystac>=1.2.0 23 | - pystac-client>=0.4.0 24 | - azure-storage-blob 25 | - fsspec 26 | - lxml # needed for thredds-crawler 27 | - urlpath 28 | - datadog 29 | - docker-py 30 | # - eodatasets3>=1.9.0 31 | - odc-geo 32 | - odc-stac 33 | 34 | # odc.{aws,aio}: aiobotocore/boto3 35 | # pin aiobotocore for easier resolution of dependencies 36 | - aiobotocore 37 | - boto3 38 | 39 | # For tests 40 | - pytest 41 | - pytest-httpserver 42 | - pytest-cov 43 | - pytest-timeout 44 | - moto 45 | - deepdiff 46 | 47 | # for docs 48 | - sphinx 49 | - sphinx_rtd_theme 50 | - sphinx-autodoc-typehints 51 | - nbsphinx 52 | 53 | - pip=24 54 | - pip: 55 | - datacube[postgres]>=1.9.0 56 | # odc.apps.dc-tools 57 | - thredds-crawler 58 | - rio-stac 59 | # - psycopg2-binary 60 | - eodatasets3>=1.9.0 61 | - docker 62 | 63 | # odc.ui 64 | - jupyter-ui-poll>=0.2.0a 65 | --------------------------------------------------------------------------------