├── eof ├── tests │ ├── __init__.py │ ├── test_asf_s3.py │ ├── test_dataspace_client.py │ ├── test_asf_client.py │ ├── test_products.py │ ├── test_eof.py │ └── cassettes │ │ └── test_dataspace_client │ │ ├── test_scihub_query_orbit_by_dt.yaml │ │ └── test_query_resorb_s1_reader_issue68.yaml ├── __main__.py ├── __init__.py ├── log.py ├── _types.py ├── _select_orbit.py ├── _asf_s3.py ├── parsing.py ├── _auth.py ├── cli.py ├── asf_client.py ├── download.py ├── products.py └── dataspace_client.py ├── requirements.txt ├── requirements-dev.txt ├── .gitignore ├── .github └── workflows │ ├── python-publish.yml │ └── ci.yml ├── .pre-commit-config.yaml ├── LICENSE.txt ├── setup.py └── README.md /eof/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | click==6.7 2 | python-dateutil==2.5.1 3 | requests>=2.20.0 4 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-recording 3 | cython 4 | twine 5 | coveralls 6 | -------------------------------------------------------------------------------- /eof/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | 4 | from eof.cli import cli 5 | 6 | sys.exit(cli()) 7 | -------------------------------------------------------------------------------- /eof/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib.metadata 2 | 3 | from . import download, parsing # noqa 4 | 5 | __version__ = importlib.metadata.version("sentineleof") 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | *.vscode 4 | .DS_Store 5 | 6 | # Don't add huge DEM data files 7 | *.hgt 8 | *.dem 9 | *.dem.rsc 10 | 11 | # Directories for setup.py 12 | build 13 | dist 14 | *.egg-info 15 | .eggs 16 | 17 | *.log 18 | *.sql 19 | 20 | docs/_build 21 | .coverage 22 | 23 | # cython build product 24 | *cpython*.so 25 | -------------------------------------------------------------------------------- /eof/log.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | def _set_logger_handler(level="INFO"): 5 | logger.setLevel(level) 6 | h = logging.StreamHandler() 7 | h.setLevel(level) 8 | format_ = "[%(asctime)s] [%(levelname)s %(filename)s] %(message)s" 9 | fmt = logging.Formatter(format_, datefmt="%m/%d %H:%M:%S") 10 | h.setFormatter(fmt) 11 | logger.addHandler(h) 12 | 13 | 14 | logger = logging.Logger("sentineleof") 15 | logger.addHandler(logging.NullHandler()) 16 | # _set_logger_handler() 17 | -------------------------------------------------------------------------------- /eof/_types.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from os import PathLike 4 | from typing import TYPE_CHECKING, Tuple, Union 5 | 6 | # Some classes are declared as generic in stubs, but not at runtime. 7 | # In Python 3.9 and earlier, os.PathLike is not subscriptable, results in a runtime error 8 | # https://stackoverflow.com/questions/71077499/typeerror-abcmeta-object-is-not-subscriptable 9 | if TYPE_CHECKING: 10 | PathLikeStr = PathLike[str] 11 | else: 12 | PathLikeStr = PathLike 13 | 14 | Filename = Union[str, PathLikeStr] 15 | 16 | # left, bottom, right, top 17 | Bbox = Tuple[float, float, float, float] 18 | -------------------------------------------------------------------------------- /eof/tests/test_asf_s3.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from eof._asf_s3 import get_orbit_files 3 | 4 | 5 | @pytest.mark.vcr() 6 | def test_get_orbit_files(): 7 | """ 8 | Test the get_orbit_files function using pytest and vcr. 9 | """ 10 | precise_orbits = get_orbit_files("precise") 11 | restituted_orbits = get_orbit_files("restituted") 12 | 13 | assert len(precise_orbits) > 0, "No precise orbit files found" 14 | assert len(restituted_orbits) > 0, "No restituted orbit files found" 15 | assert all( 16 | orbit.startswith("AUX_POEORB") for orbit in precise_orbits 17 | ), "Invalid precise orbit file name" 18 | assert all( 19 | orbit.startswith("AUX_RESORB") for orbit in restituted_orbits 20 | ), "Invalid restituted orbit file name" 21 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | name: Upload Python Package 5 | 6 | on: 7 | release: 8 | types: [created] 9 | 10 | jobs: 11 | deploy: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Python 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: '3.x' 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install setuptools wheel twine 25 | - name: Build and publish 26 | env: 27 | TWINE_USERNAME: __token__ 28 | TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} 29 | run: | 30 | python setup.py sdist bdist_wheel 31 | twine upload dist/* 32 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autofix_prs: false 3 | 4 | repos: 5 | - repo: https://github.com/pre-commit/pre-commit-hooks 6 | rev: "v4.4.0" 7 | hooks: 8 | # https://github.com/pre-commit/pre-commit-hooks/issues/718 9 | # - id: check-added-large-files # Fails with git v1.8.3 10 | - id: check-case-conflict 11 | - id: check-merge-conflict 12 | - id: check-yaml 13 | args: [--allow-multiple-documents] 14 | - id: debug-statements 15 | - id: end-of-file-fixer 16 | - id: file-contents-sorter 17 | files: (requirements.txt)$ 18 | - id: mixed-line-ending 19 | - id: trailing-whitespace 20 | 21 | - repo: https://github.com/astral-sh/ruff-pre-commit 22 | rev: v0.1.2 23 | hooks: 24 | - id: ruff 25 | args: [--fix, --exit-non-zero-on-fix] 26 | types_or: [python, jupyter] 27 | - id: ruff-format 28 | 29 | - repo: https://github.com/pre-commit/mirrors-mypy 30 | rev: "v1.4.1" 31 | hooks: 32 | - id: mypy 33 | additional_dependencies: 34 | - types-requests 35 | - types-python-dateutil 36 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018-2020 Scott Staniewicz 4 | Copyright (c) 2024 Luc Hermitte, CS Group, refactor authentication on CDSE 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | name: Build ${{ matrix.os }} ${{ matrix.python-version }} 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [ubuntu-latest] 12 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 13 | include: 14 | - os: windows-latest 15 | python-version: "3.12" 16 | - os: macos-latest 17 | python-version: "3.12" 18 | 19 | steps: 20 | - uses: actions/checkout@v2 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v2 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | python -m pip install requests click python-dateutil pytest pytest-recording 29 | python -m pip install . 30 | - name: Setup Dummy ~/.netrc file 31 | run: | 32 | echo "machine dataspace.copernicus.eu" >> ~/.netrc 33 | echo " login asdf" >> ~/.netrc 34 | echo " password asdf" >> ~/.netrc 35 | chmod 600 ~/.netrc 36 | - name: Test with pytest 37 | run: | 38 | python -m pytest -v --doctest-modules --record-mode none --ignore=eof/__main__.py 39 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="sentineleof", 8 | version="0.11.1", 9 | author="Scott Staniewicz", 10 | author_email="scott.stanie@gmail.com", 11 | description="Download precise orbit files for Sentinel 1 products", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | url="https://github.com/scottstanie/sentineleof", 15 | packages=setuptools.find_packages(), 16 | include_package_data=True, 17 | classifiers=( 18 | "Programming Language :: Python", 19 | "Programming Language :: Python :: 3.9", 20 | "Programming Language :: Python :: 3.10", 21 | "Programming Language :: Python :: 3.11", 22 | "Programming Language :: Python :: 3.12", 23 | "Programming Language :: Python :: 3.13", 24 | "License :: OSI Approved :: MIT License", 25 | "Topic :: Scientific/Engineering", 26 | "Intended Audience :: Science/Research", 27 | ), 28 | install_requires=[ 29 | "requests", 30 | "click", 31 | "python-dateutil", 32 | ], 33 | entry_points={ 34 | "console_scripts": [ 35 | "eof=eof.cli:cli", 36 | ], 37 | }, 38 | zip_safe=False, 39 | ) 40 | -------------------------------------------------------------------------------- /eof/_select_orbit.py: -------------------------------------------------------------------------------- 1 | """Module for filtering/selecting from orbit query""" 2 | 3 | from __future__ import annotations 4 | 5 | import operator 6 | from datetime import datetime, timedelta 7 | from typing import Sequence 8 | 9 | from .products import SentinelOrbit 10 | 11 | T_ORBIT = (12 * 86400.0) / 175.0 12 | """Orbital period of Sentinel-1 in seconds""" 13 | 14 | DEFAULT_MARGIN = timedelta(seconds=60) 15 | 16 | 17 | class OrbitSelectionError(RuntimeError): 18 | pass 19 | 20 | 21 | class ValidityError(ValueError): 22 | pass 23 | 24 | 25 | def last_valid_orbit( 26 | t0: datetime, 27 | t1: datetime, 28 | data: Sequence[SentinelOrbit], 29 | margin0: timedelta | None = DEFAULT_MARGIN, 30 | margin1: timedelta | None = DEFAULT_MARGIN, 31 | ) -> str: 32 | if margin0 is None: 33 | margin0 = DEFAULT_MARGIN 34 | if margin1 is None: 35 | margin1 = DEFAULT_MARGIN 36 | # Orbit files must cover the acquisition time with a small margin 37 | candidates = [ 38 | item 39 | for item in data 40 | if item.start_time <= (t0 - margin0) and item.stop_time >= (t1 + margin1) 41 | ] 42 | if not candidates: 43 | raise ValidityError( 44 | "none of the input products completely covers the requested " 45 | "time interval: [t0={}, t1={}]".format(t0, t1) 46 | ) 47 | 48 | candidates.sort(key=operator.attrgetter("created_time"), reverse=True) 49 | 50 | return candidates[0].filename 51 | -------------------------------------------------------------------------------- /eof/tests/test_dataspace_client.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import pytest 4 | from dateutil.parser import parse 5 | 6 | from eof.dataspace_client import DataspaceClient 7 | from eof.products import Sentinel 8 | 9 | 10 | @pytest.mark.vcr 11 | def test_scihub_query_orbit_by_dt(): 12 | dt = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc) 13 | mission = "S1A" 14 | c = DataspaceClient() 15 | # Restituted seems to fail for old dates... 16 | # Need to look into sentinelsat, or if ESA has just stopped allowing it 17 | results = c.query_orbit_by_dt([dt], [mission], orbit_type="precise") 18 | assert len(results) == 1 19 | r = results[0] 20 | assert r["Id"] == "21db46df-3991-4700-a454-dd91b6f2217a" 21 | assert parse(r["ContentDate"]["End"]) > dt 22 | assert parse(r["ContentDate"]["Start"]) < dt 23 | 24 | 25 | @pytest.mark.skip("Dataspace stopped carrying resorbs older than 3 months") 26 | def test_query_resorb_edge_case(): 27 | p = Sentinel( 28 | "S1A_IW_SLC__1SDV_20230823T154908_20230823T154935_050004_060418_521B.zip" 29 | ) 30 | 31 | client = DataspaceClient() 32 | 33 | results = client.query_orbit_by_dt( 34 | [p.start_time], [p.mission], orbit_type="restituted" 35 | ) 36 | assert "702fa0e1-22db-4d20-ab26-0499f262d550" in results 37 | r = results["702fa0e1-22db-4d20-ab26-0499f262d550"] 38 | assert ( 39 | r["title"] 40 | == "S1A_OPER_AUX_RESORB_OPOD_20230823T174849_V20230823T141024_20230823T172754" 41 | ) 42 | 43 | 44 | @pytest.mark.vcr 45 | def test_query_resorb_s1_reader_issue68(): 46 | f = "S1A_IW_SLC__1SDV_20250310T204228_20250310T204253_058247_0732D8_1AA3" 47 | 48 | client = DataspaceClient() 49 | query = client.query_orbit_for_product(f, orbit_type="restituted") 50 | assert len(query) == 1 51 | expected = ( 52 | "S1A_OPER_AUX_RESORB_OPOD_20250310T220905_V20250310T180852_20250310T212622.EOF" 53 | ) 54 | assert query[0]["Name"] == expected 55 | -------------------------------------------------------------------------------- /eof/tests/test_asf_client.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import pytest 4 | 5 | from eof.asf_client import ASFClient 6 | from eof.products import Sentinel 7 | from eof._asf_s3 import ASF_BUCKET_NAME, list_public_bucket 8 | 9 | 10 | @pytest.mark.vcr 11 | def test_asf_client(): 12 | ASFClient() 13 | 14 | 15 | @pytest.mark.vcr 16 | def test_asf_full_url_list(tmp_path): 17 | cache_dir = tmp_path / "sentineleof1" 18 | cache_dir.mkdir() 19 | asfclient = ASFClient(cache_dir=cache_dir) 20 | 21 | urls = asfclient.get_full_eof_list() 22 | assert len(urls) > 0 23 | # Should be quick second time 24 | assert len(asfclient.get_full_eof_list()) 25 | 26 | 27 | @pytest.mark.vcr 28 | def test_asf_client_download(tmp_path): 29 | cache_dir = tmp_path / "sentineleof2" 30 | cache_dir.mkdir() 31 | asfclient = ASFClient(cache_dir=cache_dir) 32 | 33 | dt = datetime.datetime(2020, 1, 1) 34 | mission = "S1A" 35 | urls = asfclient.get_download_urls([dt], [mission], orbit_type="precise") 36 | expected = "https://s1-orbits.s3.amazonaws.com/AUX_POEORB/S1A_OPER_AUX_POEORB_OPOD_20210316T161714_V20191231T225942_20200102T005942.EOF" 37 | assert urls == [expected] 38 | 39 | 40 | @pytest.mark.vcr 41 | def test_list_public_bucket_resorb(): 42 | resorbs = list_public_bucket(ASF_BUCKET_NAME, prefix="AUX_RESORB") 43 | assert ( 44 | resorbs[0] 45 | == "AUX_RESORB/S1A_OPER_AUX_RESORB_OPOD_20231002T140558_V20231002T102001_20231002T133731.EOF" 46 | ) 47 | 48 | 49 | @pytest.mark.vcr 50 | def test_list_public_bucket_poeorb(): 51 | precise = list_public_bucket(ASF_BUCKET_NAME, prefix="AUX_POEORB") 52 | assert ( 53 | precise[0] 54 | == "AUX_POEORB/S1A_OPER_AUX_POEORB_OPOD_20210203T122423_V20210113T225942_20210115T005942.EOF" 55 | ) 56 | 57 | 58 | @pytest.mark.vcr 59 | def test_query_resorb_s1_reader_issue68(): 60 | f = "S1A_IW_SLC__1SDV_20250310T204228_20250310T204253_058247_0732D8_1AA3" 61 | sent = Sentinel(f) 62 | orbit_dts, missions = [sent.start_time], [sent.mission] 63 | 64 | client = ASFClient() 65 | 66 | urls = client.get_download_urls(orbit_dts, missions, orbit_type="restituted") 67 | assert len(urls) == 1 68 | expected = ( 69 | "S1A_OPER_AUX_RESORB_OPOD_20250310T234452_V20250310T194736_20250310T230506.EOF" 70 | ) 71 | assert urls[0].split("/")[-1] == expected 72 | -------------------------------------------------------------------------------- /eof/tests/test_products.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from pathlib import Path 3 | 4 | from eof.products import Sentinel, SentinelOrbit 5 | 6 | 7 | def test_sentinel(): 8 | p = Sentinel( 9 | "S1A_IW_SLC__1SDV_20230823T154908_20230823T154935_050004_060418_521B.zip" 10 | ) 11 | assert ( 12 | p.filename 13 | == "S1A_IW_SLC__1SDV_20230823T154908_20230823T154935_050004_060418_521B.zip" 14 | ) 15 | assert p.start_time == datetime(2023, 8, 23, 15, 49, 8) 16 | assert p.stop_time == datetime(2023, 8, 23, 15, 49, 35) 17 | assert p.relative_orbit == 57 18 | assert p.polarization == "DV" 19 | assert p.mission == "S1A" 20 | 21 | 22 | def test_sentinel_orbit(): 23 | p = SentinelOrbit( 24 | "S1A_OPER_AUX_RESORB_OPOD_20230823T174849_V20230823T141024_20230823T172754" 25 | ) 26 | assert ( 27 | p.filename 28 | == "S1A_OPER_AUX_RESORB_OPOD_20230823T174849_V20230823T141024_20230823T172754" 29 | ) 30 | assert p.orbit_type == "restituted" 31 | assert p.start_time == datetime(2023, 8, 23, 14, 10, 24) 32 | assert p.stop_time == datetime(2023, 8, 23, 17, 27, 54) 33 | 34 | 35 | def test_pathlib(): 36 | p1 = SentinelOrbit( 37 | "S1A_OPER_AUX_RESORB_OPOD_20230823T174849_V20230823T141024_20230823T172754" 38 | ) 39 | p2 = SentinelOrbit( 40 | Path( 41 | "S1A_OPER_AUX_RESORB_OPOD_20230823T174849_V20230823T141024_20230823T172754" 42 | ) 43 | ) 44 | assert p1 == p2 45 | 46 | p1 = Sentinel( 47 | "S1A_IW_SLC__1SDV_20230823T154908_20230823T154935_050004_060418_521B.zip" 48 | ) 49 | p2 = Sentinel( 50 | Path("S1A_IW_SLC__1SDV_20230823T154908_20230823T154935_050004_060418_521B.zip") 51 | ) 52 | assert p1 == p2 53 | 54 | 55 | def test_s1c(): 56 | fname = "S1C_IW_SLC__1SDV_20250331T060116_20250331T060143_001681_002CD0_8D44" 57 | p = Sentinel(fname) 58 | 59 | assert p.filename == fname 60 | assert p.start_time == datetime(2025, 3, 31, 6, 1, 16) 61 | assert p.stop_time == datetime(2025, 3, 31, 6, 1, 43) 62 | assert p.relative_orbit == 110 63 | assert p.polarization == "DV" 64 | assert p.mission == "S1C" 65 | 66 | 67 | def test_s1d_fake_orbit_issue_74(): 68 | """Tests ability to handle S1D: https://github.com/scottstanie/sentineleof/issues/74 .""" 69 | fname = ( 70 | "S1D_OPER_AUX_RESORB_OPOD_20250703T092541_V20250703T045039_20250703T080809.EOF" 71 | ) 72 | p = SentinelOrbit(fname) 73 | 74 | assert p.filename == fname 75 | assert p.start_time == datetime(2025, 7, 3, 4, 50, 39) 76 | assert p.stop_time == datetime(2025, 7, 3, 8, 8, 9) 77 | assert p.mission == "S1D" 78 | -------------------------------------------------------------------------------- /eof/_asf_s3.py: -------------------------------------------------------------------------------- 1 | from functools import cache 2 | from typing import Optional, Literal 3 | 4 | import requests 5 | import xml.etree.ElementTree as ET 6 | 7 | from .log import logger 8 | 9 | ASF_BUCKET_NAME = "s1-orbits" 10 | 11 | 12 | @cache 13 | def list_public_bucket(bucket_name: str, prefix: str = "") -> list[str]: 14 | """List all objects in a public S3 bucket. 15 | 16 | Parameters 17 | ---------- 18 | bucket_name : str 19 | Name of the S3 bucket. 20 | prefix : str, optional 21 | Prefix to filter objects, by default "". 22 | 23 | Returns 24 | ------- 25 | list[str] 26 | list of object keys in the bucket. 27 | 28 | Raises 29 | ------ 30 | requests.RequestException 31 | If there's an error in the HTTP request. 32 | """ 33 | endpoint = f"https://{bucket_name}.s3.amazonaws.com" 34 | marker: Optional[str] = None 35 | keys: list[str] = [] 36 | 37 | while True: 38 | params = {"prefix": prefix} 39 | if marker: 40 | params["marker"] = marker 41 | 42 | try: 43 | response = requests.get(endpoint, params=params) 44 | response.raise_for_status() 45 | except requests.RequestException as e: 46 | logger.error(f"Error fetching bucket contents: {e}") 47 | raise 48 | 49 | root = ET.fromstring(response.content) 50 | for contents in root.findall( 51 | "{http://s3.amazonaws.com/doc/2006-03-01/}Contents" 52 | ): 53 | key = contents.find("{http://s3.amazonaws.com/doc/2006-03-01/}Key") 54 | if key is not None: 55 | keys.append(key.text or "") 56 | logger.debug(f"Found key: {key}") 57 | 58 | is_truncated = root.find("{http://s3.amazonaws.com/doc/2006-03-01/}IsTruncated") 59 | if ( 60 | is_truncated is not None 61 | and is_truncated.text 62 | and is_truncated.text.lower() == "true" 63 | ): 64 | next_marker = root.find( 65 | "{http://s3.amazonaws.com/doc/2006-03-01/}NextMarker" 66 | ) 67 | if next_marker is not None: 68 | marker = next_marker.text 69 | else: 70 | found_keys = root.findall( 71 | "{http://s3.amazonaws.com/doc/2006-03-01/}Contents/{http://s3.amazonaws.com/doc/2006-03-01/}Key" 72 | ) 73 | if found_keys: 74 | marker = found_keys[-1].text 75 | else: 76 | break 77 | else: 78 | break 79 | 80 | return keys 81 | 82 | 83 | def get_orbit_files(orbit_type: Literal["precise", "restituted"]) -> list[str]: 84 | """Get a list of precise or restituted orbit files. 85 | 86 | Parameters 87 | ---------- 88 | orbit_type : Literal["precise", "restituted"] 89 | Type of orbit files to retrieve. 90 | 91 | Returns 92 | ------- 93 | list[str] 94 | list of orbit file keys. 95 | 96 | Raises 97 | ------ 98 | ValueError 99 | If an invalid orbit_type is provided. 100 | """ 101 | if orbit_type not in ("precise", "restituted"): 102 | raise ValueError("orbit_type must be either 'precise' or 'restituted'") 103 | prefix = "AUX_POEORB" if orbit_type == "precise" else "AUX_RESORB" 104 | 105 | orbit_files = list_public_bucket(ASF_BUCKET_NAME, prefix=prefix) 106 | 107 | logger.info(f"Found {len(orbit_files)} {orbit_type} orbit files") 108 | return orbit_files 109 | -------------------------------------------------------------------------------- /eof/parsing.py: -------------------------------------------------------------------------------- 1 | """Module for parsing the orbit state vectors (OSVs) from the .EOF file""" 2 | 3 | from __future__ import annotations 4 | 5 | from datetime import datetime, timezone 6 | from xml.etree import ElementTree 7 | 8 | from .log import logger 9 | 10 | 11 | def parse_utc_string(timestring): 12 | # dt = datetime.strptime(timestring, 'TAI=%Y-%m-%dT%H:%M:%S.%f') 13 | # dt = datetime.strptime(timestring, 'UT1=%Y-%m-%dT%H:%M:%S.%f') 14 | return datetime.strptime(timestring, "UTC=%Y-%m-%dT%H:%M:%S.%f") 15 | 16 | 17 | def secs_since_midnight(dt): 18 | return dt.hour * 3600 + dt.minute * 60 + dt.second + dt.microsecond / 1000000.0 19 | 20 | 21 | def _convert_osv_field(osv, field, converter=float): 22 | # osv is a xml.etree.ElementTree.Element 23 | field_str = osv.find(field).text 24 | return converter(field_str) 25 | 26 | 27 | def parse_orbit( 28 | eof_filename, 29 | min_time=datetime(1900, 1, 1), 30 | max_time=datetime(2100, 1, 1), 31 | extra_osvs=1, 32 | ): 33 | min_time = to_datetime(min_time) 34 | max_time = to_datetime(max_time) 35 | logger.info( 36 | "parsing OSVs from %s between %s and %s", 37 | eof_filename, 38 | min_time, 39 | max_time, 40 | ) 41 | tree = ElementTree.parse(eof_filename) 42 | root = tree.getroot() 43 | all_osvs = [] 44 | idxs_in_range = [] 45 | for idx, osv in enumerate(root.findall("./Data_Block/List_of_OSVs/OSV")): 46 | all_osvs.append(osv) 47 | utc_dt = to_datetime(_convert_osv_field(osv, "UTC", parse_utc_string)) 48 | if utc_dt >= min_time and utc_dt <= max_time: 49 | idxs_in_range.append(idx) 50 | 51 | if not idxs_in_range: 52 | return [] 53 | 54 | min_idx = min(idxs_in_range) 55 | for ii in range(extra_osvs): 56 | idxs_in_range.append(min_idx - 1 - ii) 57 | max_idx = max(idxs_in_range) 58 | for ii in range(extra_osvs): 59 | idxs_in_range.append(max_idx + 1 + ii) 60 | idxs_in_range.sort() 61 | 62 | osvs_in_range = [] 63 | for idx in idxs_in_range: 64 | cur_osv = all_osvs[idx] 65 | utc_dt = _convert_osv_field(cur_osv, "UTC", parse_utc_string) 66 | utc_secs = secs_since_midnight(utc_dt) 67 | cur_line = [utc_secs] 68 | for field in ("X", "Y", "Z", "VX", "VY", "VZ"): 69 | # Note: the 'unit' would be elem.attrib['unit'] 70 | cur_line.append(_convert_osv_field(cur_osv, field, float)) 71 | osvs_in_range.append(cur_line) 72 | 73 | return osvs_in_range 74 | 75 | 76 | def write_orbinfo(orbit_tuples, outname="out.orbtiming"): 77 | """Write file with orbit states parsed into simpler format 78 | 79 | seconds x y z vx vy vz ax ay az 80 | """ 81 | with open(outname, "w") as f: 82 | f.write("0\n") 83 | f.write("0\n") 84 | f.write("0\n") 85 | f.write("%s\n" % len(orbit_tuples)) 86 | for tup in orbit_tuples: 87 | # final 0.0 0.0 0.0 is ax, ax, az accelerations 88 | f.write(" ".join(map(str, tup)) + " 0.0 0.0 0.0\n") 89 | 90 | 91 | def to_datetime(dates, tzinfo=timezone.utc): 92 | """Convert a single (or list of) `datetime.date` to timezone-aware `datetime.datetime`""" 93 | if isinstance(dates, datetime): 94 | return datetime(*dates.timetuple()[:6], tzinfo=tzinfo) 95 | try: 96 | iter(dates) 97 | if len(dates) == 0: 98 | return dates 99 | try: # Check if its a list of tuples (an ifglist) 100 | iter(dates[0]) 101 | return [to_datetime(tup) for tup in dates] 102 | except TypeError: 103 | return [datetime(*d.timetuple()[:6], tzinfo=tzinfo) for d in dates] 104 | # Or if it's just one sigle date 105 | except TypeError: 106 | return datetime(*dates.timetuple()[:6], tzinfo=tzinfo) 107 | -------------------------------------------------------------------------------- /eof/_auth.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import getpass 4 | import netrc 5 | import os 6 | from pathlib import Path 7 | from typing import Optional 8 | 9 | from ._types import Filename 10 | from .log import logger as _logger 11 | 12 | NASA_HOST = "urs.earthdata.nasa.gov" 13 | DATASPACE_HOST = "dataspace.copernicus.eu" 14 | 15 | 16 | def check_netrc(netrc_file: Filename = "~/.netrc"): 17 | """Chech that the netrc file exists and has the proper permissions.""" 18 | if not _file_is_0600(netrc_file): 19 | # User has a netrc file, but it's not set up correctly 20 | _logger.warning( 21 | f"Your netrc file ('{netrc_file}') does not have the " 22 | f"correct permissions: 0600* (read/write for user only).", 23 | ) 24 | 25 | 26 | def setup_netrc( 27 | netrc_file: Optional[Filename] = None, 28 | host: str = NASA_HOST, 29 | dryrun: bool = False, 30 | ): 31 | """Prompt user for NASA/Dataspace username/password, store as attribute of ~/.netrc.""" 32 | netrc_file = netrc_file or "~/.netrc" 33 | netrc_file = Path(netrc_file).expanduser() 34 | try: 35 | n = netrc.netrc(netrc_file) 36 | has_correct_permission = _file_is_0600(netrc_file) 37 | if not has_correct_permission: 38 | # User has a netrc file, but it's not set up correctly 39 | if dryrun: 40 | _logger.warning( 41 | f"Your netrc file ('{netrc_file}') does not have the " 42 | f"correct permissions: 0600* (read/write for user only).", 43 | ) 44 | else: 45 | _logger.warning( 46 | "Your ~/.netrc file does not have the correct" 47 | " permissions.\n*Changing permissions to 0600*" 48 | " (read/write for user only).", 49 | ) 50 | os.chmod(netrc_file, 0o600) 51 | # Check account exists, as well is having username and password 52 | authenticator = n.authenticators(host) 53 | if authenticator is not None: 54 | username, _, password = authenticator 55 | 56 | _has_existing_entry = ( 57 | host in n.hosts 58 | and username # type: ignore 59 | and password # type: ignore 60 | ) 61 | if _has_existing_entry: 62 | return username, password 63 | except FileNotFoundError: 64 | if not dryrun: 65 | # User doesn't have a netrc file, make one 66 | print("No ~/.netrc file found, creating one.") 67 | Path(netrc_file).write_text("") 68 | n = netrc.netrc(netrc_file) 69 | 70 | username, password = _get_username_pass(host) 71 | if not dryrun: 72 | # Add account to netrc file 73 | n.hosts[host] = (username, "", password) 74 | print(f"Saving credentials to {netrc_file} (machine={host}).") 75 | with open(netrc_file, "w") as f: 76 | f.write(str(n)) 77 | # Set permissions to 0600 (read/write for user only) 78 | # https://www.ibm.com/docs/en/aix/7.1?topic=formats-netrc-file-format-tcpip 79 | os.chmod(netrc_file, 0o600) 80 | 81 | return username, password 82 | 83 | 84 | def _file_is_0600(filename: Filename): 85 | """Check that a file has 0600 permissions (read/write for user only).""" 86 | return oct(Path(filename).stat().st_mode)[-4:] == "0600" 87 | 88 | 89 | def get_netrc_credentials( 90 | host: str, netrc_file: Optional[Filename] = None 91 | ) -> tuple[str, str]: 92 | """ 93 | Get username and password from netrc file for a given host. 94 | 95 | :return: username and password found for host in netrc_file 96 | :postcondition: username and password are non empty strings. 97 | """ 98 | netrc_file = netrc_file or "~/.netrc" 99 | netrc_file = Path(netrc_file).expanduser() 100 | _logger.debug(f"Using {netrc_file=!r}") 101 | n = netrc.netrc(netrc_file) 102 | auth = n.authenticators(host) 103 | if auth is None: 104 | raise ValueError(f"No username/password found for {host} in ~/.netrc") 105 | username, _, password = auth 106 | if username is None or password is None: 107 | raise ValueError(f"No username/password found for {host} in ~/.netrc") 108 | if not username or not password: 109 | raise ValueError(f"Empty username/password found for {host} in ~/.netrc") 110 | return username, password 111 | 112 | 113 | def _get_username_pass(host: str): 114 | """If netrc is not set up, get username/password via command line input.""" 115 | if host == NASA_HOST: 116 | # TODO: refactor the clients such that this wouldn't get called for NASA/ASF 117 | return ("", "") 118 | elif host == DATASPACE_HOST: 119 | from .dataspace_client import SIGNUP_URL as signup_url 120 | 121 | print(f"Please enter credentials for {host} to download data.") 122 | print(f"See the {signup_url} for signup info") 123 | 124 | username = input("Username: ") 125 | 126 | password = getpass.getpass("Password (will not be displayed): ") 127 | return username, password 128 | -------------------------------------------------------------------------------- /eof/tests/test_eof.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from pathlib import Path 3 | 4 | import pytest 5 | 6 | from eof import download, products 7 | 8 | 9 | @pytest.mark.vcr 10 | def test_find_scenes_to_download(tmpdir): 11 | with tmpdir.as_cwd(): 12 | name1 = ( 13 | "S1A_IW_SLC__1SDV_20180420T043026_20180420T043054_021546_025211_81BE.zip" 14 | ) 15 | name2 = ( 16 | "S1B_IW_SLC__1SDV_20180502T043026_20180502T043054_021721_025793_5C18.zip" 17 | ) 18 | name3 = "S1C_IW_SLC__1SDV_20250331T060116_20250331T060143_001681_002CD0_8D44" 19 | # Fake S1D name 20 | name4 = "S1D_IW_SLC__1SDV_20250731T060116_20250731T060143_001681_1234D0_1234" 21 | open(name1, "w").close() 22 | open(name2, "w").close() 23 | open(name3, "w").close() 24 | open(name4, "w").close() 25 | orbit_dates, missions = download.find_scenes_to_download(search_path=".") 26 | 27 | assert sorted(missions) == ["S1A", "S1B", "S1C", "S1D"] 28 | assert sorted(orbit_dates) == [ 29 | datetime.datetime(2018, 4, 20, 4, 30, 26), 30 | datetime.datetime(2018, 5, 2, 4, 30, 26), 31 | datetime.datetime(2025, 3, 31, 6, 1, 16), 32 | datetime.datetime(2025, 7, 31, 6, 1, 16), 33 | ] 34 | 35 | 36 | @pytest.mark.vcr 37 | def test_download_eofs_errors(): 38 | orbit_dates = [datetime.datetime(2018, 5, 2, 4, 30, 26)] 39 | with pytest.raises(ValueError): 40 | download.download_eofs(orbit_dates, missions=["BadMissionStr"]) 41 | # 1 date, 2 missions -> 42 | # ValueError: missions arg must be same length as orbit_dts 43 | with pytest.raises(ValueError): 44 | download.download_eofs(orbit_dates, missions=["S1A", "S1B"]) 45 | 46 | 47 | def test_main_nothing_found(): 48 | # Test "no sentinel products found" 49 | assert download.main(search_path="/notreal") == [] 50 | 51 | 52 | def test_main_error_args(): 53 | with pytest.raises(ValueError): 54 | download.main(search_path="/notreal", mission="S1A") 55 | 56 | 57 | @pytest.mark.vcr 58 | def test_download_mission_date(tmpdir): 59 | with tmpdir.as_cwd(): 60 | filenames = download.main(mission="S1A", date="20200101") 61 | assert len(filenames) == 1 62 | product = products.SentinelOrbit(filenames[0]) 63 | assert product.start_time < datetime.datetime(2020, 1, 1) 64 | assert product.stop_time > datetime.datetime(2020, 1, 1, 23, 59) 65 | 66 | 67 | @pytest.mark.vcr 68 | def test_edge_issue45(tmpdir): 69 | date = "2023-10-13 11:15:11" 70 | with tmpdir.as_cwd(): 71 | filenames = download.main(mission="S1A", date=date) 72 | assert len(filenames) == 1 73 | 74 | 75 | @pytest.mark.vcr 76 | @pytest.mark.parametrize("force_asf", [True, False]) 77 | def test_download_multiple(tmpdir, force_asf): 78 | granules = [ 79 | "S1A_IW_SLC__1SDV_20180420T043026_20180420T043054_021546_025211_81BE.zip", 80 | "S1B_IW_SLC__1SDV_20180502T043026_20180502T043054_021721_025793_5C18.zip", 81 | ] 82 | with tmpdir.as_cwd(): 83 | # Make empty files 84 | for g in granules: 85 | Path(g).write_text("") 86 | 87 | out_paths = download.main(search_path=".", force_asf=force_asf, max_workers=1) 88 | # should find two .EOF files 89 | expected_eofs = [ 90 | "S1A_OPER_AUX_POEORB_OPOD_20210307T053325_V20180419T225942_20180421T005942.EOF", 91 | "S1B_OPER_AUX_POEORB_OPOD_20210313T012515_V20180501T225942_20180503T005942.EOF", 92 | ] 93 | assert len(out_paths) == 2 94 | assert sorted((p.name for p in out_paths)) == expected_eofs 95 | 96 | 97 | def test_edge_issue78(): 98 | """Test orbit selection with looser margins for issue 78. 99 | 100 | This tests the edge case where a RESORB file starts slightly less than 101 | one full orbit before the acquisition start time. With the new looser 102 | defaults (60 second margins), this should work. With the old strict margins 103 | (T_ORBIT + 60), it would fail. 104 | """ 105 | from datetime import datetime, timedelta 106 | from eof._select_orbit import last_valid_orbit, ValidityError, T_ORBIT 107 | 108 | # Scene from issue 78 109 | scene_start = datetime(2025, 7, 15, 11, 25, 31) 110 | scene_stop = datetime(2025, 7, 15, 11, 25, 58) 111 | 112 | # Three candidate RESORB files 113 | orbit_files = [ 114 | products.SentinelOrbit( 115 | "AUX_RESORB/S1A_OPER_AUX_RESORB_OPOD_20250715T120248_V20250715T080808_20250715T112538.EOF" 116 | ), 117 | products.SentinelOrbit( 118 | "AUX_RESORB/S1A_OPER_AUX_RESORB_OPOD_20250715T133833_V20250715T094652_20250715T130422.EOF" 119 | ), 120 | products.SentinelOrbit( 121 | "AUX_RESORB/S1A_OPER_AUX_RESORB_OPOD_20250715T150939_V20250715T112537_20250715T144307.EOF" 122 | ), 123 | ] 124 | 125 | # With new looser margins (60 seconds), should select the second file 126 | result = last_valid_orbit( 127 | scene_start, 128 | scene_stop, 129 | orbit_files, 130 | margin0=timedelta(seconds=60), 131 | margin1=timedelta(seconds=60), 132 | ) 133 | assert ( 134 | "S1A_OPER_AUX_RESORB_OPOD_20250715T133833_V20250715T094652_20250715T130422.EOF" 135 | in result 136 | ) 137 | 138 | # With strict margins (T_ORBIT + 60), none should be valid 139 | with pytest.raises(ValidityError): 140 | last_valid_orbit( 141 | scene_start, 142 | scene_stop, 143 | orbit_files, 144 | margin0=timedelta(seconds=T_ORBIT + 60), 145 | margin1=timedelta(seconds=60), 146 | ) 147 | -------------------------------------------------------------------------------- /eof/cli.py: -------------------------------------------------------------------------------- 1 | """ 2 | CLI tool for downloading Sentinel 1 EOF files 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | import logging 8 | from typing import Optional 9 | 10 | import click 11 | from ._types import Filename 12 | 13 | from eof import download, log 14 | from eof._auth import DATASPACE_HOST, setup_netrc 15 | 16 | 17 | @click.command() 18 | @click.option( 19 | "--search-path", 20 | "-p", 21 | type=click.Path(exists=False, file_okay=False, writable=True), 22 | default=".", 23 | help="Path of interest for finding Sentinel products. ", 24 | show_default=True, 25 | ) 26 | @click.option( 27 | "--save-dir", 28 | type=click.Path(exists=False, file_okay=False, writable=True), 29 | default=".", 30 | help="Directory to save output .EOF files into", 31 | show_default=True, 32 | ) 33 | @click.option( 34 | "--sentinel-file", 35 | type=click.Path(exists=False, file_okay=True, dir_okay=True), 36 | help="Specify path to download only 1 .EOF for a Sentinel-1 file/folder", 37 | show_default=True, 38 | ) 39 | @click.option( 40 | "--date", 41 | "-d", 42 | help="Alternative to specifying Sentinel products: choose date to download for.", 43 | ) 44 | @click.option( 45 | "--mission", 46 | "-m", 47 | type=click.Choice(["S1A", "S1B", "S1C"]), 48 | help=( 49 | "If using `--date`, optionally specify Sentinel satellite to download" 50 | " (default: gets S1A, S1B, and S1C)" 51 | ), 52 | ) 53 | @click.option( 54 | "--orbit-type", 55 | type=click.Choice(["precise", "restituted"]), 56 | default="precise", 57 | help="Optionally specify the type of orbit file to get " 58 | "(default: precise (POEORB), but fallback to restituted (RESORB))", 59 | ) 60 | @click.option( 61 | "--force-asf", 62 | is_flag=True, 63 | help="Force the downloader to search ASF instead of ESA.", 64 | ) 65 | @click.option( 66 | "--debug", 67 | is_flag=True, 68 | help="Set logging level to DEBUG", 69 | ) 70 | @click.option( 71 | "--cdse-access-token", 72 | help="Copernicus Data Space Ecosystem access-token. " 73 | "The access token can be generated beforehand. See https://documentation.dataspace.copernicus.eu/APIs/Token.html", 74 | ) 75 | @click.option( 76 | "--cdse-user", 77 | help="Copernicus Data Space Ecosystem username. " 78 | "If not provided the program asks for it", 79 | ) 80 | @click.option( 81 | "--cdse-password", 82 | help="Copernicus Data Space Ecosystem password. " 83 | "If not provided the program asks for it", 84 | ) 85 | @click.option( 86 | "--cdse-2fa-token", 87 | help="Copernicus Data Space Ecosystem Two-Factor Token. " 88 | "Optional, unless 2FA Authentification has been enabled in user profile.", 89 | ) 90 | @click.option( 91 | "--asf-user", 92 | help="(Deprecated) ASF username. ASF orbits are now publicly available", 93 | ) 94 | @click.option( 95 | "--asf-password", 96 | help="(Deprecated) ASF password. ASF orbits are now publicly available", 97 | ) 98 | @click.option( 99 | "--ask-password", 100 | is_flag=True, 101 | help="ask for passwords interactively if needed", 102 | ) 103 | @click.option( 104 | "--update-netrc", 105 | is_flag=True, 106 | help="save credentials provided interactively in the ~/.netrc file if necessary", 107 | ) 108 | @click.option( 109 | "--netrc-file", 110 | help="Path to .netrc file. Default: ~/.netrc", 111 | ) 112 | @click.option( 113 | "--max-workers", 114 | type=int, 115 | default=3, 116 | help="Number of parallel downloads to run. Note that CDSE has a limit of 4", 117 | ) 118 | @click.option( 119 | "--s1reader-compat", 120 | is_flag=True, 121 | help="Use strict orbit margins (>1 orbit before start) for OPERA s1-reader compatibility", 122 | ) 123 | def cli( 124 | search_path: str, 125 | save_dir: str, 126 | sentinel_file: str, 127 | date: str, 128 | mission: str, 129 | orbit_type: str, 130 | force_asf: bool, 131 | debug: bool, 132 | asf_user: str = "", 133 | asf_password: str = "", 134 | cdse_access_token: Optional[str] = None, 135 | cdse_user: str = "", 136 | cdse_password: str = "", 137 | cdse_2fa_token: str = "", 138 | ask_password: bool = False, 139 | update_netrc: bool = False, 140 | netrc_file: Optional[Filename] = None, 141 | max_workers: int = 3, 142 | s1reader_compat: bool = False, 143 | ): 144 | """Download Sentinel precise orbit files. 145 | 146 | Saves files to `save-dir` (default = current directory) 147 | 148 | Download EOFs for specific date, or searches for Sentinel files in --path. 149 | Will find both ".SAFE" and ".zip" files matching Sentinel-1 naming convention. 150 | With no arguments, searches current directory for Sentinel 1 products 151 | """ 152 | log._set_logger_handler(level=logging.DEBUG if debug else logging.INFO) 153 | if ask_password: 154 | dryrun = not update_netrc 155 | if not force_asf and not (cdse_user and cdse_password): 156 | cdse_user, cdse_password = setup_netrc( 157 | netrc_file=netrc_file, host=DATASPACE_HOST, dryrun=dryrun 158 | ) 159 | 160 | download.main( 161 | search_path=search_path, 162 | save_dir=save_dir, 163 | sentinel_file=sentinel_file, 164 | mission=mission, 165 | date=date, 166 | orbit_type=orbit_type, 167 | force_asf=force_asf, 168 | cdse_access_token=cdse_access_token, 169 | cdse_user=cdse_user, 170 | cdse_password=cdse_password, 171 | cdse_2fa_token=cdse_2fa_token, 172 | netrc_file=netrc_file, 173 | max_workers=max_workers, 174 | s1reader_compat=s1reader_compat, 175 | ) 176 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build](https://github.com/scottstanie/sentineleof/actions/workflows/ci.yml/badge.svg)](https://github.com/scottstanie/sentineleof/actions/workflows/ci.yml) 2 | [![PyPI version][pypi-version]][pypi-link] 3 | [![Conda-Forge][conda-badge]][conda-link] 4 | [![PyPI platforms][pypi-platforms]][pypi-link] 5 | 6 | 7 | [conda-badge]: https://img.shields.io/conda/vn/conda-forge/sentineleof 8 | [conda-link]: https://github.com/conda-forge/sentineleof-feedstock 9 | [pypi-link]: https://pypi.org/project/sentineleof/ 10 | [pypi-platforms]: https://img.shields.io/pypi/pyversions/sentineleof 11 | [pypi-version]: https://img.shields.io/pypi/v/sentineleof 12 | 13 | 14 | # Sentinel EOF 15 | 16 | Tool to download Sentinel 1 precise/restituted orbit files (.EOF files) for processing SLCs 17 | 18 | ## Changes to Sentinel-1 orbit files source 19 | 20 | The source for orbit files provided by ASF has switched to their [public S3 bucket](https://registry.opendata.aws/s1-orbits/). 21 | Since the S3 bucket is public, no Earthdata credentials are required. 22 | 23 | To use this directly bypass the default of using the Copernicus Data Space Ecosystem (CDSE), you can pass the `--force-asf` flag[^1] to the command line tool. 24 | 25 | ## Changes to Copernicus Data Space Ecosystem (since October, 2023) 26 | 27 | The [Copernicus Scihub client has discontinued service](https://scihub.copernicus.eu/) in favor of [the new Copernicus Data Space Ecosystem](https://dataspace.copernicus.eu/). The new service no longer allows anonymous public downloads (using the `gnssuser`), which means you must register for either a Dataspace account (to use the CDSE data). 28 | 29 | *Changes required by you to continue using CDSE-provided orbits with this tool:* 30 | 31 | Register for CDSE 32 | 33 | 1. Register for an account with Copernicus Data Space account at https://dataspace.copernicus.eu/ (using the Loging button, which will have the option for a "Register" page) 34 | 2. After creating the username and confirming your email, store your username/password in a `~/.netrc` file (or, on Windows, `~_netrc`) with the hostname `dataspace.copernicus.eu`: 35 | ``` 36 | machine dataspace.copernicus.eu 37 | login MYUSERNAME 38 | password MYPASSWORD 39 | ``` 40 | 41 | 42 | ## Setup and installation 43 | 44 | ```bash 45 | pip install sentineleof 46 | ``` 47 | 48 | or through conda: 49 | 50 | ```bash 51 | conda install -c conda-forge sentineleof 52 | ``` 53 | 54 | This will put the executable `eof` on your path 55 | 56 | ## Usage 57 | 58 | After setting up your `~/.netrc` (see above), if you have a bunch of Sentinel 1 zip files (or unzipped SAFE folders), you can simply run 59 | 60 | ```bash 61 | eof 62 | ``` 63 | and download either the precise orbit files, or, if the POEORB files have not been released, the restituted RESORB files. 64 | 65 | Running 66 | ```bash 67 | eof --search-path /path/to/safe_files/ --save-dir ./orbits/ 68 | ``` 69 | will search `/path/to/safe_files/` for Sentinel-1 scenes, and save the `.EOF` files to `./orbits/` (creating it if it does not exist) 70 | 71 | 72 | ## Command Line Interface Reference 73 | 74 | Full options available for the command line tool are: 75 | 76 | ``` 77 | $ eof --help 78 | Usage: eof [OPTIONS] 79 | 80 | Download Sentinel precise orbit files. 81 | 82 | Saves files to `save-dir` (default = current directory) 83 | 84 | Download EOFs for specific date, or searches for Sentinel files in --path. 85 | Will find both ".SAFE" and ".zip" files matching Sentinel-1 naming 86 | convention. With no arguments, searches current directory for Sentinel 1 87 | products 88 | 89 | Options: 90 | -p, --search-path DIRECTORY Path of interest for finding Sentinel 91 | products. [default: .] 92 | --save-dir DIRECTORY Directory to save output .EOF files into 93 | [default: .] 94 | --sentinel-file PATH Specify path to download only 1 .EOF for a 95 | Sentinel-1 file/folder 96 | -d, --date TEXT Alternative to specifying Sentinel products: 97 | choose date to download for. 98 | -m, --mission [S1A|S1B|S1C] If using `--date`, optionally specify 99 | Sentinel satellite to download (default: 100 | gets S1A, S1B, and S1C) 101 | --orbit-type [precise|restituted] 102 | Optionally specify the type of orbit file to 103 | get (default: precise (POEORB), but fallback 104 | to restituted (RESORB)) 105 | --force-asf Force the downloader to search ASF instead 106 | of ESA. 107 | --debug Set logging level to DEBUG 108 | --cdse-access-token TEXT Copernicus Data Space Ecosystem access- 109 | token. The access token can be generated 110 | beforehand. See https://documentation.datasp 111 | ace.copernicus.eu/APIs/Token.html 112 | --cdse-user TEXT Copernicus Data Space Ecosystem username. If 113 | not provided the program asks for it 114 | --cdse-password TEXT Copernicus Data Space Ecosystem password. If 115 | not provided the program asks for it 116 | --cdse-2fa-token TEXT Copernicus Data Space Ecosystem Two-Factor 117 | Token. Optional, unless 2FA Authentification 118 | has been enabled in user profile. 119 | --ask-password ask for passwords interactively if needed 120 | --update-netrc save credentials provided interactively in 121 | the ~/.netrc file if necessary 122 | --netrc-file TEXT Path to .netrc file. Default: ~/.netrc 123 | --max-workers INTEGER Number of parallel downloads to run. Note 124 | that CDSE has a limit of 4 125 | --help Show this message and exit. 126 | ``` 127 | 128 | To use the function from python, you can pass a list of dates: 129 | 130 | ```python 131 | from eof.download import download_eofs 132 | 133 | download_eofs([datetime.datetime(2018, 5, 3, 0, 0, 0)]) 134 | download_eofs(['20180503', '20180507'], ['S1A', 'S1B', 'S1C']) 135 | ``` 136 | 137 | [^1]: This will be the default in a future version of this package. It is currently still an optional a flag to keep backward compatibility. 138 | -------------------------------------------------------------------------------- /eof/tests/cassettes/test_dataspace_client/test_scihub_query_orbit_by_dt.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: client_id=cdse-public&username=scott.stanie%40gmail.com&password=1Z8Xmfl2KbSx%21&grant_type=password 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate, br, zstd 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '100' 13 | Content-Type: 14 | - application/x-www-form-urlencoded 15 | User-Agent: 16 | - python-requests/2.32.3 17 | method: POST 18 | uri: https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token 19 | response: 20 | body: 21 | string: '{"access_token":"eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJYVUh3VWZKaHVDVWo0X3k4ZF8xM0hxWXBYMFdwdDd2anhob2FPLUxzREZFIn0.eyJleHAiOjE3NjAzNzAzNDgsImlhdCI6MTc2MDM2OTc0OCwianRpIjoiN2VkOWQ3ZDItMDY4ZS00YjhmLThiNjQtODgyYWNkNDc4OTZjIiwiaXNzIjoiaHR0cHM6Ly9pZGVudGl0eS5kYXRhc3BhY2UuY29wZXJuaWN1cy5ldS9hdXRoL3JlYWxtcy9DRFNFIiwiYXVkIjpbIkNMT1VERkVSUk9fUFVCTElDIiwiYWNjb3VudCJdLCJzdWIiOiI2YjUyOTQyZS0yMzEwLTRkZGItYjhkZS0wNjU1MjEyOTQwNjEiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJjZHNlLXB1YmxpYyIsInNlc3Npb25fc3RhdGUiOiIzOTA4OGNhNi02NTZjLTQzMzMtODlmMy0yODY1YzYxODVmNTQiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cHM6Ly9sb2NhbGhvc3Q6NDIwMCIsIioiLCJodHRwczovL3dvcmtzcGFjZS5zdGFnaW5nLWNkc2UtZGF0YS1leHBsb3Jlci5hcHBzLnN0YWdpbmcuaW50cmEuY2xvdWRmZXJyby5jb20iXSwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbImNkc2UtanVweXRlcmxhYiIsImNvcGVybmljdXMtZ2VuZXJhbC1xdW90YSIsIm9mZmxpbmVfYWNjZXNzIiwidW1hX2F1dGhvcml6YXRpb24iLCJkZWZhdWx0LXJvbGVzLWNkYXMiLCJjb3Blcm5pY3VzLWdlbmVyYWwiXX0sInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6IkFVRElFTkNFX1BVQkxJQyBvcGVuaWQgZW1haWwgcHJvZmlsZSBvbmRlbWFuZF9wcm9jZXNzaW5nIHVzZXItY29udGV4dCIsInNpZCI6IjM5MDg4Y2E2LTY1NmMtNDMzMy04OWYzLTI4NjVjNjE4NWY1NCIsImdyb3VwX21lbWJlcnNoaXAiOlsiL2FjY2Vzc19ncm91cHMvdXNlcl90eXBvbG9neS9jb3Blcm5pY3VzX2dlbmVyYWwiLCIvb3JnYW5pemF0aW9ucy9kZWZhdWx0LTZiNTI5NDJlLTIzMTAtNGRkYi1iOGRlLTA2NTUyMTI5NDA2MS9yZWd1bGFyX3VzZXIiXSwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJTY290dCBTdGFuaWV3aWN6Iiwib3JnYW5pemF0aW9ucyI6WyJkZWZhdWx0LTZiNTI5NDJlLTIzMTAtNGRkYi1iOGRlLTA2NTUyMTI5NDA2MSJdLCJ1c2VyX2NvbnRleHRfaWQiOiJjN2JlMjE3Yi04Mzc4LTRkOTQtYmFmNi0yN2RmNWYxMGY3NmMiLCJjb250ZXh0X3JvbGVzIjp7fSwiY29udGV4dF9ncm91cHMiOlsiL2FjY2Vzc19ncm91cHMvdXNlcl90eXBvbG9neS9jb3Blcm5pY3VzX2dlbmVyYWwvIiwiL29yZ2FuaXphdGlvbnMvZGVmYXVsdC02YjUyOTQyZS0yMzEwLTRkZGItYjhkZS0wNjU1MjEyOTQwNjEvcmVndWxhcl91c2VyLyJdLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJzY290dC5zdGFuaWVAZ21haWwuY29tIiwiZ2l2ZW5fbmFtZSI6IlNjb3R0IiwiZmFtaWx5X25hbWUiOiJTdGFuaWV3aWN6IiwidXNlcl9jb250ZXh0IjoiZGVmYXVsdC02YjUyOTQyZS0yMzEwLTRkZGItYjhkZS0wNjU1MjEyOTQwNjEiLCJlbWFpbCI6InNjb3R0LnN0YW5pZUBnbWFpbC5jb20ifQ.btli7lOuuRyH_qbPuR1x9YtP19JacdV39Bq3a68ApeDUsCv9hwVkxvoW43bW6zIt3qf4Xxo-7SwllbXSUmFzu5kI3bqF_IG5nJibpVqPboIAo6pu3JDWZy1C_Y5AbXR9c8jmgwH-64P073klgw0dlZEICugKo73l_nx6kb-NNhimrVacyF30d_SW7Sg3namPKjtvQiITtGfs0TV2oX6IrHLTxS0hdedy4yhr81CxI83AKuq8VXZQ1uDAb3XoRE0E16dVlQVfLGUGB6pSpPqSaiFBkBZ89l3LpTON1UnbsJT41nVEp25_x0HUjDF5_OJNkRrbIb7Pn1XQpwVcO1Gsnw","expires_in":600,"refresh_expires_in":3600,"refresh_token":"eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhZmFlZTU2Zi1iNWZiLTRiMzMtODRlYS0zMWY2NzMyMzNhNzgifQ.eyJleHAiOjE3NjAzNzMzNDgsImlhdCI6MTc2MDM2OTc0OCwianRpIjoiYmMwNTAwZDAtYzAxNC00YjM0LTlkYzktMjFjYWM4OTZlMDZlIiwiaXNzIjoiaHR0cHM6Ly9pZGVudGl0eS5kYXRhc3BhY2UuY29wZXJuaWN1cy5ldS9hdXRoL3JlYWxtcy9DRFNFIiwiYXVkIjoiaHR0cHM6Ly9pZGVudGl0eS5kYXRhc3BhY2UuY29wZXJuaWN1cy5ldS9hdXRoL3JlYWxtcy9DRFNFIiwic3ViIjoiNmI1Mjk0MmUtMjMxMC00ZGRiLWI4ZGUtMDY1NTIxMjk0MDYxIiwidHlwIjoiUmVmcmVzaCIsImF6cCI6ImNkc2UtcHVibGljIiwic2Vzc2lvbl9zdGF0ZSI6IjM5MDg4Y2E2LTY1NmMtNDMzMy04OWYzLTI4NjVjNjE4NWY1NCIsInNjb3BlIjoiQVVESUVOQ0VfUFVCTElDIG9wZW5pZCBlbWFpbCBwcm9maWxlIG9uZGVtYW5kX3Byb2Nlc3NpbmcgdXNlci1jb250ZXh0Iiwic2lkIjoiMzkwODhjYTYtNjU2Yy00MzMzLTg5ZjMtMjg2NWM2MTg1ZjU0In0.FBzLyJgDazLmPKy-UqyHN4u4Ev0XzmtKtmvNCVMiG6Q","token_type":"Bearer","not-before-policy":0,"session_state":"39088ca6-656c-4333-89f3-2865c6185f54","scope":"AUDIENCE_PUBLIC 22 | openid email profile ondemand_processing user-context"}' 23 | headers: 24 | cache-control: 25 | - no-store 26 | content-length: 27 | - '3483' 28 | content-type: 29 | - application/json 30 | pragma: 31 | - no-cache 32 | referrer-policy: 33 | - no-referrer 34 | set-cookie: 35 | - KEYCLOAK_LOCALE=; Version=1; Comment=Expiring cookie; Expires=Thu, 01-Jan-1970 36 | 00:00:10 GMT; Max-Age=0; Path=/auth/realms/CDSE/; Secure; HttpOnly 37 | - KC_RESTART=; Version=1; Expires=Thu, 01-Jan-1970 00:00:10 GMT; Max-Age=0; 38 | Path=/auth/realms/CDSE/; Secure; HttpOnly 39 | - KC_AUTH_STATE=; Version=1; Expires=Thu, 01-Jan-1970 00:00:10 GMT; Max-Age=0; 40 | Path=/auth/realms/CDSE/; Secure 41 | - SERVERID=keycloak002.waw2; path=/ 42 | strict-transport-security: 43 | - max-age=31536000; includeSubDomains 44 | x-content-type-options: 45 | - nosniff 46 | x-frame-options: 47 | - ALLOW-FROM https://www.google.com 48 | x-xss-protection: 49 | - 1; mode=block 50 | status: 51 | code: 200 52 | message: OK 53 | - request: 54 | body: null 55 | headers: 56 | Accept: 57 | - '*/*' 58 | Accept-Encoding: 59 | - gzip, deflate, br, zstd 60 | Connection: 61 | - keep-alive 62 | User-Agent: 63 | - python-requests/2.32.3 64 | method: GET 65 | uri: https://catalogue.dataspace.copernicus.eu/odata/v1/Products?%24filter=startswith%28Name%2C%27S1A%27%29+and+contains%28Name%2C%27AUX_POEORB%27%29+and+ContentDate%2FStart+lt+%272019-12-31T23%3A59%3A00.000000Z%27+and+ContentDate%2FEnd+gt+%272020-01-01T00%3A01%3A00.000000Z%27&%24orderby=ContentDate%2FStart+asc&%24top=1 66 | response: 67 | body: 68 | string: '{"@odata.context":"$metadata#Products","value":[{"@odata.mediaContentType":"application/octet-stream","Id":"21db46df-3991-4700-a454-dd91b6f2217a","Name":"S1A_OPER_AUX_POEORB_OPOD_20210315T155112_V20191230T225942_20200101T005942.EOF","ContentType":"application/octet-stream","ContentLength":4409905,"OriginDate":"2021-04-17T10:54:33.561000Z","PublicationDate":"2023-10-25T14:34:15.278386Z","ModificationDate":"2023-10-25T14:49:02.413958Z","Online":true,"EvictionDate":"9999-12-31T23:59:59.999999Z","S3Path":"/eodata/Sentinel-1/AUX/AUX_POEORB/2019/12/30/S1A_OPER_AUX_POEORB_OPOD_20210315T155112_V20191230T225942_20200101T005942.EOF","Checksum":[{"Value":"e0345368421e1b65127adc6ea103ab53","Algorithm":"MD5","ChecksumDate":"2023-10-25T14:49:02.002746Z"},{"Value":"b8fcec2aca5a68702f4887652b17994e4f23fb2392501dfba4106bee5ad7ce94","Algorithm":"BLAKE3","ChecksumDate":"2023-10-25T14:49:02.020456Z"}],"ContentDate":{"Start":"2019-12-30T22:59:42.000000Z","End":"2020-01-01T00:59:42.000000Z"},"Footprint":null,"GeoFootprint":null}],"@odata.nextLink":"https://catalogue.dataspace.copernicus.eu/odata/v1/Products?%24filter=startswith%28Name%2C%27S1A%27%29+and+contains%28Name%2C%27AUX_POEORB%27%29+and+ContentDate%2FStart+lt+%272019-12-31T23%3A59%3A00.000000Z%27+and+ContentDate%2FEnd+gt+%272020-01-01T00%3A01%3A00.000000Z%27&%24orderby=ContentDate%2FStart+asc&%24top=1&%24skip=1"}' 69 | headers: 70 | Access-Control-Allow-Credentials: 71 | - 'true' 72 | Access-Control-Allow-Headers: 73 | - DNT,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization 74 | Access-Control-Allow-Methods: 75 | - GET, PUT, POST, DELETE, PATCH, OPTIONS 76 | Access-Control-Allow-Origin: 77 | - '*' 78 | Access-Control-Max-Age: 79 | - '1728000' 80 | Connection: 81 | - keep-alive 82 | Content-Length: 83 | - '1371' 84 | Content-Type: 85 | - application/json 86 | Date: 87 | - Mon, 13 Oct 2025 15:35:49 GMT 88 | Strict-Transport-Security: 89 | - max-age=15724800; includeSubDomains 90 | request-id: 91 | - e4aee827-516d-4856-a1a3-66fd014fb67e 92 | x-envoy-upstream-service-time: 93 | - '237' 94 | status: 95 | code: 200 96 | message: OK 97 | version: 1 98 | -------------------------------------------------------------------------------- /eof/tests/cassettes/test_dataspace_client/test_query_resorb_s1_reader_issue68.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: client_id=cdse-public&username=scott.stanie%40gmail.com&password=1Z8Xmfl2KbSx%21&grant_type=password 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate, br, zstd 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '100' 13 | Content-Type: 14 | - application/x-www-form-urlencoded 15 | User-Agent: 16 | - python-requests/2.32.3 17 | method: POST 18 | uri: https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token 19 | response: 20 | body: 21 | string: '{"access_token":"eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJYVUh3VWZKaHVDVWo0X3k4ZF8xM0hxWXBYMFdwdDd2anhob2FPLUxzREZFIn0.eyJleHAiOjE3NjAzNzAzNTYsImlhdCI6MTc2MDM2OTc1NiwianRpIjoiNjUzNjQ4MmQtZjNjYi00Y2JjLWFhOTctZWExZTVhYzk4ODNjIiwiaXNzIjoiaHR0cHM6Ly9pZGVudGl0eS5kYXRhc3BhY2UuY29wZXJuaWN1cy5ldS9hdXRoL3JlYWxtcy9DRFNFIiwiYXVkIjpbIkNMT1VERkVSUk9fUFVCTElDIiwiYWNjb3VudCJdLCJzdWIiOiI2YjUyOTQyZS0yMzEwLTRkZGItYjhkZS0wNjU1MjEyOTQwNjEiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJjZHNlLXB1YmxpYyIsInNlc3Npb25fc3RhdGUiOiJhYmMyYmU2My1hMzViLTQ3ZTItODQ3MS1mZjIwOTJmYTljYzkiLCJhbGxvd2VkLW9yaWdpbnMiOlsiaHR0cHM6Ly9sb2NhbGhvc3Q6NDIwMCIsIioiLCJodHRwczovL3dvcmtzcGFjZS5zdGFnaW5nLWNkc2UtZGF0YS1leHBsb3Jlci5hcHBzLnN0YWdpbmcuaW50cmEuY2xvdWRmZXJyby5jb20iXSwicmVhbG1fYWNjZXNzIjp7InJvbGVzIjpbImNkc2UtanVweXRlcmxhYiIsImNvcGVybmljdXMtZ2VuZXJhbC1xdW90YSIsIm9mZmxpbmVfYWNjZXNzIiwidW1hX2F1dGhvcml6YXRpb24iLCJkZWZhdWx0LXJvbGVzLWNkYXMiLCJjb3Blcm5pY3VzLWdlbmVyYWwiXX0sInJlc291cmNlX2FjY2VzcyI6eyJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX19LCJzY29wZSI6IkFVRElFTkNFX1BVQkxJQyBvcGVuaWQgZW1haWwgcHJvZmlsZSBvbmRlbWFuZF9wcm9jZXNzaW5nIHVzZXItY29udGV4dCIsInNpZCI6ImFiYzJiZTYzLWEzNWItNDdlMi04NDcxLWZmMjA5MmZhOWNjOSIsImdyb3VwX21lbWJlcnNoaXAiOlsiL2FjY2Vzc19ncm91cHMvdXNlcl90eXBvbG9neS9jb3Blcm5pY3VzX2dlbmVyYWwiLCIvb3JnYW5pemF0aW9ucy9kZWZhdWx0LTZiNTI5NDJlLTIzMTAtNGRkYi1iOGRlLTA2NTUyMTI5NDA2MS9yZWd1bGFyX3VzZXIiXSwiZW1haWxfdmVyaWZpZWQiOnRydWUsIm5hbWUiOiJTY290dCBTdGFuaWV3aWN6Iiwib3JnYW5pemF0aW9ucyI6WyJkZWZhdWx0LTZiNTI5NDJlLTIzMTAtNGRkYi1iOGRlLTA2NTUyMTI5NDA2MSJdLCJ1c2VyX2NvbnRleHRfaWQiOiJjN2JlMjE3Yi04Mzc4LTRkOTQtYmFmNi0yN2RmNWYxMGY3NmMiLCJjb250ZXh0X3JvbGVzIjp7fSwiY29udGV4dF9ncm91cHMiOlsiL2FjY2Vzc19ncm91cHMvdXNlcl90eXBvbG9neS9jb3Blcm5pY3VzX2dlbmVyYWwvIiwiL29yZ2FuaXphdGlvbnMvZGVmYXVsdC02YjUyOTQyZS0yMzEwLTRkZGItYjhkZS0wNjU1MjEyOTQwNjEvcmVndWxhcl91c2VyLyJdLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJzY290dC5zdGFuaWVAZ21haWwuY29tIiwiZ2l2ZW5fbmFtZSI6IlNjb3R0IiwiZmFtaWx5X25hbWUiOiJTdGFuaWV3aWN6IiwidXNlcl9jb250ZXh0IjoiZGVmYXVsdC02YjUyOTQyZS0yMzEwLTRkZGItYjhkZS0wNjU1MjEyOTQwNjEiLCJlbWFpbCI6InNjb3R0LnN0YW5pZUBnbWFpbC5jb20ifQ.gDiUOWqTdSoab1gCIwyxBiE1vTekCd1WPkiqZ9fX5hdm233E4rcgJZDDGBYcTdY4xUd0yIuVbaDbodIDx96tJT6Q9KWkmX6RtTMOXu0GzUCKkgY3rmlMhWTP676Cv4jGTw_hMuGIzhliD137OrnDG-R4FR-zDKX3pOzrze7MoQpl2tRcNNCe-whmULWfoA6PwS77kGVKCi79YwBlwJVZTT2gMR-Ys0bSkUsVPflLeRwbulOYy7QIt14RfwvGAopXAjwRgSryPZvkox-Dpq5am0RNedoXKrBKiwu7O8nQoAMqNhE12-Q1gW0by7cf9tfIkq9rF9Ev0zxU76Syort2gw","expires_in":600,"refresh_expires_in":3600,"refresh_token":"eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhZmFlZTU2Zi1iNWZiLTRiMzMtODRlYS0zMWY2NzMyMzNhNzgifQ.eyJleHAiOjE3NjAzNzMzNTYsImlhdCI6MTc2MDM2OTc1NiwianRpIjoiNTNkOGUyY2UtN2Q5OS00Yzc4LTllZTgtMGQ1ZTJmZjU2OTA3IiwiaXNzIjoiaHR0cHM6Ly9pZGVudGl0eS5kYXRhc3BhY2UuY29wZXJuaWN1cy5ldS9hdXRoL3JlYWxtcy9DRFNFIiwiYXVkIjoiaHR0cHM6Ly9pZGVudGl0eS5kYXRhc3BhY2UuY29wZXJuaWN1cy5ldS9hdXRoL3JlYWxtcy9DRFNFIiwic3ViIjoiNmI1Mjk0MmUtMjMxMC00ZGRiLWI4ZGUtMDY1NTIxMjk0MDYxIiwidHlwIjoiUmVmcmVzaCIsImF6cCI6ImNkc2UtcHVibGljIiwic2Vzc2lvbl9zdGF0ZSI6ImFiYzJiZTYzLWEzNWItNDdlMi04NDcxLWZmMjA5MmZhOWNjOSIsInNjb3BlIjoiQVVESUVOQ0VfUFVCTElDIG9wZW5pZCBlbWFpbCBwcm9maWxlIG9uZGVtYW5kX3Byb2Nlc3NpbmcgdXNlci1jb250ZXh0Iiwic2lkIjoiYWJjMmJlNjMtYTM1Yi00N2UyLTg0NzEtZmYyMDkyZmE5Y2M5In0.a-9u3wedqO1hQpDnN7sJAwUkjmy6UK1Wz5z56lx4H8M","token_type":"Bearer","not-before-policy":0,"session_state":"abc2be63-a35b-47e2-8471-ff2092fa9cc9","scope":"AUDIENCE_PUBLIC 22 | openid email profile ondemand_processing user-context"}' 23 | headers: 24 | cache-control: 25 | - no-store 26 | content-length: 27 | - '3483' 28 | content-type: 29 | - application/json 30 | pragma: 31 | - no-cache 32 | referrer-policy: 33 | - no-referrer 34 | set-cookie: 35 | - KEYCLOAK_LOCALE=; Version=1; Comment=Expiring cookie; Expires=Thu, 01-Jan-1970 36 | 00:00:10 GMT; Max-Age=0; Path=/auth/realms/CDSE/; Secure; HttpOnly 37 | - KC_RESTART=; Version=1; Expires=Thu, 01-Jan-1970 00:00:10 GMT; Max-Age=0; 38 | Path=/auth/realms/CDSE/; Secure; HttpOnly 39 | - KC_AUTH_STATE=; Version=1; Expires=Thu, 01-Jan-1970 00:00:10 GMT; Max-Age=0; 40 | Path=/auth/realms/CDSE/; Secure 41 | - SERVERID=keycloak001.waw2; path=/ 42 | strict-transport-security: 43 | - max-age=31536000; includeSubDomains 44 | x-content-type-options: 45 | - nosniff 46 | x-frame-options: 47 | - ALLOW-FROM https://www.google.com 48 | x-xss-protection: 49 | - 1; mode=block 50 | status: 51 | code: 200 52 | message: OK 53 | - request: 54 | body: null 55 | headers: 56 | Accept: 57 | - '*/*' 58 | Accept-Encoding: 59 | - gzip, deflate, br, zstd 60 | Connection: 61 | - keep-alive 62 | User-Agent: 63 | - python-requests/2.32.3 64 | method: GET 65 | uri: https://catalogue.dataspace.copernicus.eu/odata/v1/Products?%24filter=startswith%28Name%2C%27S1A%27%29+and+contains%28Name%2C%27AUX_RESORB%27%29+and+ContentDate%2FStart+lt+%272025-03-10T20%3A41%3A28.000000Z%27+and+ContentDate%2FEnd+gt+%272025-03-10T20%3A43%3A28.000000Z%27&%24orderby=ContentDate%2FStart+asc&%24top=1 66 | response: 67 | body: 68 | string: '{"@odata.context":"$metadata#Products","value":[{"@odata.mediaContentType":"application/octet-stream","Id":"6f5734c8-fdfc-11ef-a200-0242ac120002","Name":"S1A_OPER_AUX_RESORB_OPOD_20250310T220905_V20250310T180852_20250310T212622.EOF","ContentType":"application/octet-stream","ContentLength":590749,"OriginDate":"2025-03-10T22:10:11.570000Z","PublicationDate":"2025-03-10T22:10:47.026472Z","ModificationDate":"2025-03-10T22:10:47.026472Z","Online":true,"EvictionDate":"9999-12-31T23:59:59.999999Z","S3Path":"/eodata/Sentinel-1/AUX/AUX_RESORB/2025/03/10/S1A_OPER_AUX_RESORB_OPOD_20250310T220905_V20250310T180852_20250310T212622.EOF","Checksum":[{"Value":"ead3d2760014f518fe0521c6930d5fc5","Algorithm":"MD5","ChecksumDate":"2025-03-10T22:10:45.527687Z"},{"Value":"485c5afb96399347b4e75dcdb0532d6b438a9cd2bb0ae0d9c0fbb29382777ad3","Algorithm":"BLAKE3","ChecksumDate":"2025-03-10T22:10:45.546864Z"}],"ContentDate":{"Start":"2025-03-10T18:08:52.000000Z","End":"2025-03-10T21:26:22.000000Z"},"Footprint":null,"GeoFootprint":null}],"@odata.nextLink":"https://catalogue.dataspace.copernicus.eu/odata/v1/Products?%24filter=startswith%28Name%2C%27S1A%27%29+and+contains%28Name%2C%27AUX_RESORB%27%29+and+ContentDate%2FStart+lt+%272025-03-10T20%3A41%3A28.000000Z%27+and+ContentDate%2FEnd+gt+%272025-03-10T20%3A43%3A28.000000Z%27&%24orderby=ContentDate%2FStart+asc&%24top=1&%24skip=1"}' 69 | headers: 70 | Access-Control-Allow-Credentials: 71 | - 'true' 72 | Access-Control-Allow-Headers: 73 | - DNT,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization 74 | Access-Control-Allow-Methods: 75 | - GET, PUT, POST, DELETE, PATCH, OPTIONS 76 | Access-Control-Allow-Origin: 77 | - '*' 78 | Access-Control-Max-Age: 79 | - '1728000' 80 | Connection: 81 | - keep-alive 82 | Content-Length: 83 | - '1370' 84 | Content-Type: 85 | - application/json 86 | Date: 87 | - Mon, 13 Oct 2025 15:35:57 GMT 88 | Strict-Transport-Security: 89 | - max-age=15724800; includeSubDomains 90 | request-id: 91 | - c9a6838b-16d5-4741-a4f9-378379a89e70 92 | x-envoy-upstream-service-time: 93 | - '394' 94 | status: 95 | code: 200 96 | message: OK 97 | version: 1 98 | -------------------------------------------------------------------------------- /eof/asf_client.py: -------------------------------------------------------------------------------- 1 | """Client to get orbit files from ASF via the public S3 bucket. 2 | 3 | Now uses public S3 endpoints and does not require authentication. 4 | """ 5 | 6 | from __future__ import annotations 7 | 8 | import os 9 | from datetime import datetime 10 | from pathlib import Path 11 | from typing import Literal 12 | 13 | import requests 14 | 15 | from ._asf_s3 import get_orbit_files, ASF_BUCKET_NAME 16 | from ._select_orbit import ValidityError, last_valid_orbit 17 | from ._types import Filename 18 | from .log import logger 19 | from .products import SentinelOrbit 20 | 21 | 22 | class ASFClient: 23 | eof_lists = {"precise": None, "restituted": None} 24 | 25 | def __init__( 26 | self, 27 | cache_dir: Filename | None = None, 28 | username: str = "", 29 | password: str = "", 30 | netrc_file: Filename | None = None, 31 | ): 32 | """Initialize the ASF client. 33 | 34 | The interface still accepts username, password, etc., 35 | these are now ignored since orbit files are publicly available via S3. 36 | """ 37 | self._cache_dir = cache_dir 38 | 39 | def get_full_eof_list( 40 | self, orbit_type="precise", max_dt=None 41 | ) -> list[SentinelOrbit]: 42 | """Get the list of orbit files from the public S3 bucket. 43 | 44 | If a cached file list exists and is current, that file is used. 45 | Otherwise the list is retrieved fresh from S3. 46 | 47 | Args: 48 | orbit_type (str): Either "precise" or "restituted" 49 | max_dt (datetime, optional): latest datetime requested; if the cached 50 | list is older than this, the cache is cleared. 51 | 52 | Returns: 53 | list[SentinelOrbit]: list of orbit file objects 54 | """ 55 | if orbit_type not in ("precise", "restituted"): 56 | raise ValueError(f"Unknown orbit type: {orbit_type}") 57 | 58 | if self.eof_lists.get(orbit_type) is not None: 59 | return self.eof_lists[orbit_type] 60 | # Try to see if we have the list of EOFs in the cache 61 | cache_path = self._get_filename_cache_path(orbit_type) 62 | if os.path.exists(cache_path): 63 | eof_list = self._get_cached_filenames(orbit_type) 64 | # Clear the cache if the newest saved file is older than requested 65 | max_saved = max(e.start_time for e in eof_list) 66 | if max_dt is not None and max_saved < max_dt: 67 | logger.warning("Clearing cached %s EOF list:", orbit_type) 68 | logger.warning("%s is older than requested %s", max_saved, max_dt) 69 | self._clear_cache(orbit_type) 70 | else: 71 | logger.info("Using cached EOF list") 72 | self.eof_lists[orbit_type] = eof_list 73 | return eof_list 74 | 75 | logger.info("Downloading orbit file list from public S3 bucket") 76 | keys = get_orbit_files(orbit_type) 77 | eof_list = [SentinelOrbit(f) for f in keys] 78 | self.eof_lists[orbit_type] = eof_list 79 | self._write_cached_filenames(orbit_type, eof_list) 80 | return eof_list 81 | 82 | def get_download_urls( 83 | self, 84 | orbit_dts: list[datetime], 85 | missions: list[str], 86 | orbit_type="precise", 87 | margin0=None, 88 | margin1=None, 89 | ) -> list[str]: 90 | """Find the download URL for an orbit file covering the specified datetime. 91 | 92 | Args: 93 | orbit_dts (list[datetime]): requested dates for orbit coverage. 94 | missions (list[str]): specify S1A or S1B (should be same length as orbit_dts). 95 | orbit_type (str): either "precise" or "restituted". 96 | margin0 (timedelta, optional): margin before the start time. 97 | margin1 (timedelta, optional): margin after the end time. 98 | 99 | Returns: 100 | list[str]: URLs for the orbit files. 101 | 102 | Raises: 103 | ValidityError if an orbit is not found. 104 | """ 105 | eof_list = self.get_full_eof_list(orbit_type=orbit_type, max_dt=max(orbit_dts)) 106 | # Split up for quicker parsing by mission 107 | mission_to_eof_list = { 108 | "S1A": [eof for eof in eof_list if eof.mission == "S1A"], 109 | "S1B": [eof for eof in eof_list if eof.mission == "S1B"], 110 | "S1C": [eof for eof in eof_list if eof.mission == "S1C"], 111 | } 112 | 113 | remaining_orbits = [] 114 | urls = [] 115 | for dt, mission in zip(orbit_dts, missions): 116 | try: 117 | filename = last_valid_orbit( 118 | dt, 119 | dt, 120 | mission_to_eof_list[mission], 121 | margin0=margin0, 122 | margin1=margin1, 123 | ) 124 | # Construct the full download URL using the bucket name from _asf_s3 125 | url = f"https://{ASF_BUCKET_NAME}.s3.amazonaws.com/{filename}" 126 | urls.append(url) 127 | except ValidityError: 128 | remaining_orbits.append((dt, mission)) 129 | 130 | if remaining_orbits: 131 | logger.warning("The following dates were not found: %s", remaining_orbits) 132 | if orbit_type == "precise": 133 | logger.warning( 134 | "Attempting to download the restituted orbits for these dates." 135 | ) 136 | remaining_dts, remaining_missions = zip(*remaining_orbits) 137 | urls.extend( 138 | self.get_download_urls( 139 | remaining_dts, 140 | remaining_missions, 141 | orbit_type="restituted", 142 | margin0=margin0, 143 | margin1=margin1, 144 | ) 145 | ) 146 | 147 | return urls 148 | 149 | def _get_cached_filenames( 150 | self, orbit_type: Literal["precise", "restituted"] 151 | ) -> list[SentinelOrbit] | None: 152 | """Read the cache file for the ASF orbit filenames.""" 153 | filepath = self._get_filename_cache_path(orbit_type) 154 | logger.debug(f"ASF file path cache: {filepath = }") 155 | if os.path.exists(filepath): 156 | with open(filepath, "r") as f: 157 | return [SentinelOrbit(f) for f in f.read().splitlines()] 158 | return None 159 | 160 | def _write_cached_filenames(self, orbit_type="precise", eof_list=[]) -> None: 161 | """Cache the ASF orbit filenames.""" 162 | filepath = self._get_filename_cache_path(orbit_type) 163 | with open(filepath, "w") as f: 164 | for e in eof_list: 165 | f.write(e.filename + "\n") 166 | 167 | def _clear_cache(self, orbit_type="precise") -> None: 168 | """Clear the cache for the ASF orbit filenames.""" 169 | filepath = self._get_filename_cache_path(orbit_type) 170 | os.remove(filepath) 171 | 172 | def _get_filename_cache_path(self, orbit_type="precise") -> str: 173 | fname = f"{orbit_type.lower()}_filenames.txt" 174 | return os.path.join(self.get_cache_dir(), fname) 175 | 176 | def get_cache_dir(self) -> Path | str: 177 | """Determine the directory to store orbit file caches. 178 | 179 | Returns: 180 | str: Directory path 181 | """ 182 | if self._cache_dir is not None: 183 | return self._cache_dir 184 | path = os.getenv("XDG_CACHE_HOME", os.path.expanduser("~/.cache")) 185 | path = os.path.join(path, "sentineleof") 186 | logger.debug("Cache path: %s", path) 187 | if not os.path.exists(path): 188 | os.makedirs(path) 189 | return path 190 | 191 | def _download_and_write(self, url: str, save_dir: str = ".") -> Path: 192 | """Download an orbit file from a URL and save it to save_dir. 193 | 194 | Args: 195 | url (str): URL of the orbit file to download. 196 | save_dir (str): Directory to save the orbit file. 197 | 198 | Returns: 199 | Path: Path to the saved orbit file. 200 | """ 201 | fname = Path(save_dir) / url.split("/")[-1] 202 | if os.path.isfile(fname): 203 | logger.info("%s already exists, skipping download.", url) 204 | return fname 205 | 206 | logger.info("Downloading %s", url) 207 | try: 208 | response = requests.get(url) 209 | response.raise_for_status() 210 | except requests.exceptions.HTTPError as e: 211 | logger.error("Failed to download %s: %s", url, e) 212 | raise 213 | 214 | logger.info("Saving to %s", fname) 215 | with open(fname, "wb") as f: 216 | f.write(response.content) 217 | return fname 218 | -------------------------------------------------------------------------------- /eof/download.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Utility for downloading Sentinel precise orbit ephemerides (EOF) files 4 | 5 | Example filtering URL: 6 | ?validity_start_time=2014-08&page=2 7 | 8 | Example EOF: 'S1A_OPER_AUX_POEORB_OPOD_20140828T122040_V20140806T225944_20140808T005944.EOF' 9 | 10 | 'S1A' : mission id (satellite it applies to) 11 | 'OPER' : OPER for "Routine Operations" file 12 | 'AUX_POEORB' : AUX_ for "auxiliary data file", POEORB=Precise Orbit Ephemerides (POE) Orbit File 13 | 'OPOD' Site Center of the file originator 14 | 15 | '20140828T122040' creation date of file 16 | 'V20140806T225944' Validity start time (when orbit is valid) 17 | '20140808T005944' Validity end time 18 | 19 | Full EOF sentinel doumentation: 20 | https://earth.esa.int/documents/247904/349490/GMES_Sentinels_POD_Service_File_Format_Specification_GMES-GSEG-EOPG-FS-10-0075_Issue1-3.pdf 21 | 22 | See parsers for Sentinel file naming description 23 | """ 24 | 25 | from __future__ import annotations 26 | 27 | import glob 28 | import itertools 29 | import os 30 | from collections.abc import Sequence 31 | from datetime import datetime 32 | from multiprocessing.pool import ThreadPool 33 | from pathlib import Path 34 | from typing import Literal, Optional 35 | 36 | from dateutil.parser import parse 37 | from requests.exceptions import HTTPError 38 | 39 | from ._types import Filename 40 | from .asf_client import ASFClient 41 | from .dataspace_client import DataspaceClient 42 | from .log import logger 43 | from .products import Sentinel, SentinelOrbit 44 | 45 | MAX_WORKERS = 6 # workers to download in parallel (for ASF backup) 46 | 47 | 48 | def download_eofs( 49 | orbit_dts: Sequence[datetime | str] | None = None, 50 | missions: Sequence[str] | None = None, 51 | sentinel_file: str | None = None, 52 | save_dir: str = ".", 53 | orbit_type: Literal["precise", "restituted"] = "precise", 54 | force_asf: bool = False, 55 | asf_user: str = "", 56 | asf_password: str = "", 57 | cdse_access_token: Optional[str] = None, 58 | cdse_user: str = "", 59 | cdse_password: str = "", 60 | cdse_2fa_token: str = "", 61 | netrc_file: Optional[Filename] = None, 62 | max_workers: int = MAX_WORKERS, 63 | s1reader_compat: bool = False, 64 | ) -> list[Path]: 65 | """Downloads and saves Sentinel precise or restituted orbit files (EOF). 66 | 67 | By default, it tries to download from Copernicus Data Space Ecosystem first, 68 | then falls back to ASF if the first source fails (unless `force_asf` is True). 69 | 70 | Parameters 71 | ---------- 72 | orbit_dts : list[datetime] or list[str] or None 73 | List of datetimes for orbit coverage. 74 | If strings are provided, they will be parsed into datetime objects. 75 | missions : list[str] or None 76 | List of mission identifiers ('S1A', 'S1B', or 'S1C'). If None, 77 | orbits for all missions will be attempted. Must be same length as orbit_dts. 78 | sentinel_file : str or None 79 | Path to a Sentinel-1 file to download a matching EOF for. 80 | If provided, orbit_dts and missions are ignored. 81 | save_dir : str, default="." 82 | Directory to save the downloaded EOF files into. 83 | orbit_type : {'precise', 'restituted'}, default='precise' 84 | Type of orbit file to download (precise=POEORB or restituted=RESORB). 85 | force_asf : bool, default=False 86 | If True, skip trying Copernicus Data Space and download directly from ASF. 87 | asf_user : str, default="" 88 | ASF username (deprecated, ASF orbits are now publicly available). 89 | asf_password : str, default="" 90 | ASF password (deprecated, ASF orbits are now publicly available). 91 | cdse_access_token : str or None, default=None 92 | Copernicus Data Space Ecosystem access token. 93 | cdse_user : str, default="" 94 | Copernicus Data Space Ecosystem username. 95 | cdse_password : str, default="" 96 | Copernicus Data Space Ecosystem password. 97 | cdse_2fa_token : str, default="" 98 | Copernicus Data Space Ecosystem two-factor authentication token. 99 | netrc_file : str or None, default=None 100 | Path to .netrc file for authentication credentials. 101 | max_workers : int, default=MAX_WORKERS 102 | Number of parallel downloads to run. 103 | s1reader_compat : bool, default=False 104 | Use strict margins (>1 orbit before start) for OPERA s1-reader compatibility. 105 | 106 | Returns 107 | ------- 108 | list[Path] 109 | Paths to all successfully downloaded orbit files. 110 | 111 | Raises 112 | ------ 113 | ValueError 114 | If missions argument contains values other than 'S1A', 'S1B', 'S1C', 115 | if missions and orbit_dts have different lengths, or if sentinel_file is invalid. 116 | HTTPError 117 | If there's an HTTP error during download that's not a rate limit error. 118 | """ 119 | # TODO: condense list of same dates, different hours? 120 | if missions and all(m not in ("S1A", "S1B", "S1C") for m in missions): 121 | raise ValueError('missions argument must be "S1A", "S1B" or "S1C"') 122 | if sentinel_file: 123 | sent = Sentinel(sentinel_file) 124 | orbit_dts, missions = [sent.start_time], [sent.mission] 125 | assert orbit_dts is not None and missions is not None 126 | if missions and len(missions) != len(orbit_dts): 127 | raise ValueError("missions arg must be same length as orbit_dts") 128 | if not missions: 129 | missions = itertools.repeat(None) 130 | 131 | # First make sure all are datetimes if given string 132 | orbit_dts = [parse(dt) if isinstance(dt, str) else dt for dt in orbit_dts] 133 | 134 | # Set margins based on s1reader compatibility mode 135 | from datetime import timedelta 136 | from ._select_orbit import T_ORBIT 137 | 138 | if s1reader_compat: 139 | t0_margin = timedelta(seconds=T_ORBIT + 60) 140 | t1_margin = timedelta(seconds=60) 141 | else: 142 | t0_margin = timedelta(seconds=60) 143 | t1_margin = timedelta(seconds=60) 144 | 145 | filenames = [] 146 | dataspace_successful = False 147 | 148 | # First, check that Scihub isn't having issues 149 | if not force_asf: 150 | client = DataspaceClient( 151 | access_token=cdse_access_token, 152 | username=cdse_user, 153 | password=cdse_password, 154 | token_2fa=cdse_2fa_token, 155 | netrc_file=netrc_file, 156 | ) 157 | if client: 158 | # try to search on scihub 159 | if sentinel_file: 160 | query = client.query_orbit_for_product( 161 | sentinel_file, 162 | orbit_type=orbit_type, 163 | t0_margin=t0_margin, 164 | t1_margin=t1_margin, 165 | ) 166 | else: 167 | query = client.query_orbit_by_dt( 168 | orbit_dts, 169 | missions, 170 | orbit_type=orbit_type, 171 | t0_margin=t0_margin, 172 | t1_margin=t1_margin, 173 | ) 174 | 175 | if query: 176 | logger.info("Attempting download from Copernicus Data Space Ecosystem") 177 | try: 178 | results = client.download_all( 179 | query, output_directory=save_dir, max_workers=max_workers 180 | ) 181 | filenames.extend(results) 182 | dataspace_successful = True 183 | except HTTPError as e: 184 | assert e.response is not None 185 | if e.response.status_code == 429: 186 | logger.warning(f"Failed due to too many requests: {e.args}") 187 | # Dataspace failed -> try asf 188 | else: 189 | raise 190 | 191 | # For failures from scihub, try ASF 192 | if not dataspace_successful: 193 | if not force_asf: 194 | logger.warning("Dataspace failed, trying ASF") 195 | 196 | asf_client = ASFClient() 197 | urls = asf_client.get_download_urls( 198 | orbit_dts, 199 | missions, 200 | orbit_type=orbit_type, 201 | margin0=t0_margin, 202 | margin1=t1_margin, 203 | ) 204 | # Download and save all links in parallel 205 | pool = ThreadPool(processes=max_workers) 206 | result_url_dict = { 207 | pool.apply_async( 208 | asf_client._download_and_write, 209 | args=[url, save_dir], 210 | ): url 211 | for url in urls 212 | } 213 | 214 | for result, url in result_url_dict.items(): 215 | cur_filename = result.get() 216 | if cur_filename is None: 217 | logger.error("Failed to download orbit for %s", url) 218 | else: 219 | logger.info("Finished %s, saved to %s", url, cur_filename) 220 | filenames.append(cur_filename) 221 | 222 | return filenames 223 | 224 | 225 | def find_current_eofs(cur_path): 226 | """Returns a list of SentinelOrbit objects located in `cur_path`""" 227 | return sorted( 228 | [ 229 | SentinelOrbit(filename) 230 | for filename in glob.glob(os.path.join(cur_path, "S1*OPER*.EOF")) 231 | ] 232 | ) 233 | 234 | 235 | def find_unique_safes(search_path): 236 | file_set = set() 237 | for filename in glob.glob(os.path.join(search_path, "S1*")): 238 | try: 239 | parsed_file = Sentinel(filename) 240 | except ValueError: # Doesn't match a sentinel file 241 | logger.debug("Skipping {}, not a Sentinel 1 file".format(filename)) 242 | continue 243 | file_set.add(parsed_file) 244 | return file_set 245 | 246 | 247 | def find_scenes_to_download(search_path="./", save_dir="./"): 248 | """Parse the search_path directory for any Sentinel 1 products' date and mission""" 249 | orbit_dts = [] 250 | missions = [] 251 | # Check for already-downloaded orbit files, skip ones we have 252 | current_eofs = find_current_eofs(save_dir) 253 | 254 | # Now loop through each Sentinel scene in search_path 255 | for parsed_file in find_unique_safes(search_path): 256 | if parsed_file.start_time in orbit_dts: 257 | # start_time is a datetime, already found 258 | continue 259 | if any(parsed_file.start_time in orbit for orbit in current_eofs): 260 | logger.info( 261 | "Skipping {}, already have EOF file".format( 262 | os.path.splitext(parsed_file.filename)[0] 263 | ) 264 | ) 265 | continue 266 | 267 | logger.info( 268 | "Downloading precise orbits for {} on {}".format( 269 | parsed_file.mission, parsed_file.start_time.strftime("%Y-%m-%d") 270 | ) 271 | ) 272 | orbit_dts.append(parsed_file.start_time) 273 | missions.append(parsed_file.mission) 274 | 275 | return orbit_dts, missions 276 | 277 | 278 | def main( 279 | search_path=".", 280 | save_dir=".", 281 | sentinel_file=None, 282 | mission=None, 283 | date=None, 284 | orbit_type="precise", 285 | force_asf: bool = False, 286 | asf_user: str = "", 287 | asf_password: str = "", 288 | cdse_access_token: Optional[str] = None, 289 | cdse_user: str = "", 290 | cdse_password: str = "", 291 | cdse_2fa_token: str = "", 292 | netrc_file: Optional[Filename] = None, 293 | max_workers: int = MAX_WORKERS, 294 | s1reader_compat: bool = False, 295 | ): 296 | """Function used for entry point to download eofs""" 297 | 298 | if not os.path.exists(save_dir): 299 | logger.info("Creating directory for output: %s", save_dir) 300 | os.mkdir(save_dir) 301 | 302 | if mission and not date: 303 | raise ValueError("Must specify date if providing mission.") 304 | 305 | if sentinel_file: 306 | # Handle parsing in download_eof 307 | orbit_dts, missions = None, None 308 | elif date: 309 | missions = [mission] if mission else ["S1A", "S1B", "S1C"] 310 | orbit_dts = [parse(date)] * len(missions) 311 | # Check they didn't pass a whole datetime 312 | if all((dt.hour == 0 and dt.minute == 0) for dt in orbit_dts): 313 | # If we only specify dates, make sure the whole thing is covered 314 | # This means we should set the `hour` to be late in the day 315 | orbit_dts = [dt.replace(hour=23) for dt in orbit_dts] 316 | else: 317 | # No command line args given: search current directory 318 | orbit_dts, missions = find_scenes_to_download( 319 | search_path=search_path, save_dir=save_dir 320 | ) 321 | if not orbit_dts: 322 | logger.info( 323 | "No Sentinel products found in directory %s, exiting", search_path 324 | ) 325 | return [] 326 | 327 | return download_eofs( 328 | orbit_dts=orbit_dts, 329 | missions=missions, 330 | sentinel_file=sentinel_file, 331 | save_dir=save_dir, 332 | orbit_type=orbit_type, 333 | force_asf=force_asf, 334 | asf_user=asf_user, 335 | asf_password=asf_password, 336 | cdse_access_token=cdse_access_token, 337 | cdse_user=cdse_user, 338 | cdse_password=cdse_password, 339 | cdse_2fa_token=cdse_2fa_token, 340 | netrc_file=netrc_file, 341 | max_workers=max_workers, 342 | s1reader_compat=s1reader_compat, 343 | ) 344 | -------------------------------------------------------------------------------- /eof/products.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Scott Staniewicz 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in all 11 | # copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE 20 | 21 | # Source code copied form: 22 | # https://github.com/scottstanie/apertools/blob/77e6330499adc01c3860f49ee6b3875c49532b76/apertools/parsers.py 23 | 24 | """Utilities for parsing file names of SAR products for relevant info.""" 25 | 26 | from __future__ import annotations 27 | 28 | import re 29 | from datetime import datetime 30 | 31 | __all__ = ["Sentinel", "SentinelOrbit"] 32 | 33 | 34 | class Base(object): 35 | """Base parser to illustrate expected interface/ minimum data available""" 36 | 37 | def __init__(self, filename, verbose=False): 38 | """ 39 | Extract data from filename 40 | filename (str): name of SAR/InSAR product 41 | verbose (bool): print extra logging into about file loading 42 | """ 43 | self.filename = filename 44 | self.full_parse() # Run a parse to check validity of filename 45 | self.verbose = verbose 46 | 47 | def __str__(self): 48 | return "{} product: {}".format(self.__class__.__name__, self.filename) 49 | 50 | def __repr__(self): 51 | return "{}({!r})".format(self.__class__.__name__, self.filename) 52 | 53 | def __lt__(self, other): 54 | return self.filename < other.filename 55 | 56 | def full_parse(self): 57 | """Returns all parts of the data contained in filename 58 | 59 | Returns: 60 | tuple: parsed file data. Entry order will match `field_meanings` 61 | 62 | Raises: 63 | ValueError: if filename string is invalid 64 | """ 65 | if not hasattr(self, "FILE_REGEX"): 66 | raise NotImplementedError("Must define class FILE_REGEX to parse") 67 | 68 | match = re.search(self.FILE_REGEX, str(self.filename)) 69 | if not match: 70 | raise ValueError( 71 | "Invalid {} filename: {}".format(self.__class__.__name__, self.filename) 72 | ) 73 | else: 74 | return match.groupdict() 75 | 76 | @property 77 | def field_meanings(self): 78 | """List the fields returned by full_parse()""" 79 | return self.full_parse().keys() 80 | 81 | def _get_field(self, fieldname): 82 | """Pick a specific field based on its name""" 83 | return self.full_parse()[fieldname] 84 | 85 | def __getitem__(self, item): 86 | """Access properties with uavsar[item] syntax""" 87 | return self._get_field(item) 88 | 89 | 90 | class Sentinel(Base): 91 | """ 92 | Sentinel 1 reference: 93 | https://sentinel.esa.int/web/sentinel/user-guides/sentinel-1-sar/naming-conventions 94 | or https://sentinel.esa.int/documents/247904/349449/Sentinel-1_Product_Specification 95 | 96 | Example: 97 | S1A_IW_SLC__1SDV_20180408T043025_20180408T043053_021371_024C9B_1B70.zip 98 | S1A_IW_RAW__0SSV_20151018T005110_20151018T005142_008200_00B886_61EC.zip 99 | 100 | File name format: 101 | MMM_BB_TTTR_LFPP_YYYYMMDDTHHMMSS_YYYYMMDDTHHMMSS_OOOOOO_DDDDDD_CCCC.EEEE 102 | 103 | MMM: mission/satellite S1A, S1B or S1C 104 | BB: Mode/beam identifier. The S1-S6 beams apply to SM products IW, 105 | EW and WV identifiers appply to products from the respective modes. 106 | TTT: Product Type: RAW, SLC, GRD, OCN 107 | R: Resolution class: F, H, M, or _ (N/A) 108 | L: Processing Level: 0, 1, 2 109 | F: Product class: S (standard), A (annotation, only used internally) 110 | - we only care about standard 111 | PP: Polarization: SH (single HH), SV (single VV), DH (dual HH+HV), DV (dual VV+VH) 112 | Start date + time (date/time separated by T) 113 | Stop date + time 114 | OOOOOO: absolute orbit number: 000001-999999 115 | DDDDDD: mission data-take identifier: 000001-FFFFFF. 116 | CCCC: product unique identifier: hexadecimal string from CRC-16 hashing 117 | the manifest file using CRC-CCITT. 118 | 119 | Once unzipped, the folder extension is always "SAFE" 120 | 121 | Attributes: 122 | filename (str) name of the sentinel data product 123 | """ 124 | 125 | FILE_REGEX = re.compile( 126 | r"(?PS1A|S1B|S1C|S1D|S1E)_" 127 | r"(?P[\w\d]{2})_" 128 | r"(?P[\w_]{3})" 129 | r"(?P[FHM_])_" 130 | r"(?P[012])[SA]" 131 | r"(?P[SDHV]{2})_" 132 | r"(?P[T\d]{15})_" 133 | r"(?P[T\d]{15})_" 134 | r"(?P\d{6})_" 135 | r"(?P[\d\w]{6})_" 136 | r"(?P[\d\w]{4})" 137 | ) 138 | TIME_FMT = "%Y%m%dT%H%M%S" 139 | 140 | def __init__(self, filename, **kwargs): 141 | super().__init__(filename, **kwargs) 142 | 143 | def __str__(self): 144 | return "{} {}, path {} from {}".format( 145 | self.__class__.__name__, self.mission, self.path, self.date 146 | ) 147 | 148 | def __lt__(self, other): 149 | return (self.start_time, self.filename) < (other.start_time, other.filename) 150 | 151 | def __eq__(self, other): 152 | # TODO: Do we just want to compare product_uids?? or filenames? 153 | return self.product_uid == other.product_uid 154 | # return self.filename == other.filename 155 | 156 | def __hash__(self): 157 | return hash(self.product_uid) 158 | 159 | @property 160 | def start_time(self): 161 | """Returns start datetime from a sentinel file name 162 | 163 | Example: 164 | >>> s = Sentinel('S1A_IW_SLC__1SDV_20180408T043025_20180408T043053_021371_024C9B_1B70') 165 | >>> print(s.start_time) 166 | 2018-04-08 04:30:25 167 | """ 168 | start_time_str = self._get_field("start_datetime") 169 | return datetime.strptime(start_time_str, self.TIME_FMT) 170 | 171 | @property 172 | def stop_time(self): 173 | """Returns stop datetime from a sentinel file name 174 | 175 | Example: 176 | >>> s = Sentinel('S1A_IW_SLC__1SDV_20180408T043025_20180408T043053_021371_024C9B_1B70') 177 | >>> print(s.stop_time) 178 | 2018-04-08 04:30:53 179 | """ 180 | stop_time_str = self._get_field("stop_datetime") 181 | return datetime.strptime(stop_time_str, self.TIME_FMT) 182 | 183 | @property 184 | def polarization(self): 185 | """Returns type of polarization of product 186 | 187 | Example: 188 | >>> s = Sentinel('S1A_IW_SLC__1SDV_20180408T043025_20180408T043053_021371_024C9B_1B70') 189 | >>> print(s.polarization) 190 | DV 191 | """ 192 | return self._get_field("polarization") 193 | 194 | @property 195 | def product_type(self): 196 | """Returns product type/level 197 | 198 | Example: 199 | >>> s = Sentinel('S1A_IW_SLC__1SDV_20180408T043025_20180408T043053_021371_024C9B_1B70') 200 | >>> print(s.product_type) 201 | SLC 202 | """ 203 | return self._get_field("product_type") 204 | 205 | @property 206 | def level(self): 207 | """Alias for product type/level""" 208 | return self.product_type 209 | 210 | @property 211 | def mission(self): 212 | """Returns satellite/mission of product (S1A/S1B/S1C) 213 | 214 | Example: 215 | >>> s = Sentinel('S1A_IW_SLC__1SDV_20180408T043025_20180408T043053_021371_024C9B_1B70') 216 | >>> print(s.mission) 217 | S1A 218 | """ 219 | return self._get_field("mission") 220 | 221 | @property 222 | def absolute_orbit(self): 223 | """Absolute orbit of data, included in file name 224 | 225 | Example: 226 | >>> s = Sentinel('S1A_IW_SLC__1SDV_20180408T043025_20180408T043053_021371_024C9B_1B70') 227 | >>> print(s.absolute_orbit) 228 | 21371 229 | """ 230 | return int(self._get_field("orbit_number")) 231 | 232 | @property 233 | def relative_orbit(self): 234 | """Relative orbit number/ path 235 | 236 | Formulas for relative orbit from absolute come from: 237 | https://forum.step.esa.int/t/sentinel-1-relative-orbit-from-filename/7042 238 | 239 | Example: 240 | >>> s = Sentinel('S1A_IW_SLC__1SDV_20180408T043025_20180408T043053_021371_024C9B_1B70') 241 | >>> print(s.relative_orbit) 242 | 124 243 | >>> s = Sentinel('S1B_WV_OCN__2SSV_20180522T161319_20180522T164846_011036_014389_67D8') 244 | >>> print(s.relative_orbit) 245 | 160 246 | """ 247 | if self.mission == "S1A": 248 | return ((self.absolute_orbit - 73) % 175) + 1 249 | elif self.mission == "S1B": 250 | return ((self.absolute_orbit - 27) % 175) + 1 251 | elif self.mission == "S1C": 252 | return ((self.absolute_orbit - 172) % 175) + 1 253 | else: 254 | raise ValueError(f"Unknown relative orbit for mission {self.mission}") 255 | 256 | @property 257 | def path(self): 258 | """Alias for relative orbit number""" 259 | return self.relative_orbit 260 | 261 | @property 262 | def product_uid(self): 263 | """Unique identifier of product (last 4 of filename)""" 264 | return self._get_field("unique_id") 265 | 266 | @property 267 | def date(self): 268 | """Date of acquisition: shortcut for start_time.date()""" 269 | return self.start_time.date() 270 | 271 | 272 | class SentinelOrbit(Base): 273 | """ 274 | Sentinel 1 orbit reference: 275 | https://sentinel.esa.int/documents/247904/351187/GMES_Sentinels_POD_Service_File_Format_Specification 276 | section 2 277 | https://qc.sentinel1.eo.esa.int/doc/api/ 278 | https://sentinels.copernicus.eu/documents/247904/3372484/Copernicus-POD-Regular-Service-Review-Jun-Sep-2018.pdf 279 | see here (section 3.6) for differences in orbit accuracy) 280 | 281 | Example: 282 | S1A_OPER_AUX_PREORB_OPOD_20200325T131800_V20200325T121452_20200325T184952.EOF 283 | 284 | The filename must comply with the following pattern: 285 | MMM_CCCC_TTTTTTTTTT_.EOF 286 | 287 | MMM = mission, S1A, S1B or S1C 288 | CCCC = File Class, we only want OPER = routine operational 289 | TTTTTTTTTT = File type 290 | = FFFF DDDDDD 291 | FFFF = file category, we want AUX_:auxiliary data files; 292 | DDDDDD = Semantic Descriptor 293 | most common = POEORB: Precise Orbit Ephemerides (POE) Orbit File 294 | (available after 1-2 weeks) 295 | also, RESORB: Restituted orbit file 296 | (covers 6 hour windows, less accurate, more immediate) 297 | TODO: do I ever want to deal with the AUX antenna files? 298 | 299 | has a couple: 300 | ssss_yyyymmddThhmmsswhere: 301 | ssss is the Site Centre of the file originator (OPOD for S-1 and S-2) 302 | and a validity start/stop, same date format 303 | 304 | Attributes: 305 | filename (str) name of the sentinel data product 306 | """ 307 | 308 | TIME_FMT = "%Y%m%dT%H%M%S" 309 | FILE_REGEX = ( 310 | r"(?PS1A|S1B|S1C|S1D|S1E)_OPER_AUX_" 311 | r"(?P[\w_]{6})_OPOD_" 312 | r"(?P[T\d]{15})_" 313 | r"V(?P[T\d]{15})_" 314 | r"(?P[T\d]{15})" 315 | ) 316 | 317 | def __init__(self, filename, **kwargs): 318 | super().__init__(filename, **kwargs) 319 | 320 | def __str__(self): 321 | return "{} {} from {} to {}".format( 322 | self.orbit_type, self.__class__.__name__, self.start_time, self.stop_time 323 | ) 324 | 325 | def __lt__(self, other): 326 | return (self.start_time, self.filename) < (other.start_time, other.filename) 327 | 328 | def __contains__(self, dt): 329 | """Checks if a datetime lies within the validity window""" 330 | return self.start_time < dt < self.stop_time 331 | 332 | def __eq__(self, other): 333 | return ( 334 | self.mission, 335 | self.start_time, 336 | self.stop_time, 337 | self.orbit_type, 338 | ) == ( 339 | other.mission, 340 | other.start_time, 341 | other.stop_time, 342 | other.orbit_type, 343 | ) 344 | 345 | @property 346 | def mission(self): 347 | """Returns satellite/mission of product (S1A/S1B/S1C) 348 | 349 | Example: 350 | >>> s = SentinelOrbit('S1A_OPER_AUX_POEORB_OPOD_20200121T120654_V20191231T225942_20200102T005942.EOF') 351 | >>> print(s.mission) 352 | S1A 353 | """ 354 | return self._get_field("mission") 355 | 356 | @property 357 | def start_time(self): 358 | """Returns start datetime of an orbit 359 | 360 | Example: 361 | >>> s = SentinelOrbit('S1A_OPER_AUX_POEORB_OPOD_20200121T120654_V20191231T225942_20200102T005942.EOF') 362 | >>> print(s.start_time) 363 | 2019-12-31 22:59:42 364 | """ 365 | start_time_str = self._get_field("start_datetime") 366 | return datetime.strptime(start_time_str, self.TIME_FMT) 367 | 368 | @property 369 | def stop_time(self): 370 | """Returns stop datetime from a sentinel file name 371 | 372 | Example: 373 | >>> s = SentinelOrbit('S1A_OPER_AUX_POEORB_OPOD_20200121T120654_V20191231T225942_20200102T005942.EOF') 374 | >>> print(s.stop_time) 375 | 2020-01-02 00:59:42 376 | """ 377 | stop_time_str = self._get_field("stop_datetime") 378 | return datetime.strptime(stop_time_str, self.TIME_FMT) 379 | 380 | @property 381 | def created_time(self): 382 | """Returns created datetime from a orbit file name 383 | 384 | Example: 385 | >>> s = SentinelOrbit('S1A_OPER_AUX_POEORB_OPOD_20200121T120654_V20191231T225942_20200102T005942.EOF') 386 | >>> print(s.created_time) 387 | 2020-01-21 12:06:54 388 | """ 389 | stop_time_str = self._get_field("created_datetime") 390 | return datetime.strptime(stop_time_str, self.TIME_FMT) 391 | 392 | @property 393 | def orbit_type(self): 394 | """Type of orbit file (e.g precise, restituted) 395 | 396 | Example: 397 | >>> s = SentinelOrbit('S1A_OPER_AUX_POEORB_OPOD_20200121T120654_V20191231T225942_20200102T005942.EOF') 398 | >>> print(s.orbit_type) 399 | precise 400 | >>> s = SentinelOrbit('S1B_OPER_AUX_RESORB_OPOD_20200325T151938_V20200325T112442_20200325T144212.EOF') 401 | >>> print(s.orbit_type) 402 | restituted 403 | """ 404 | o = self._get_field("orbit_type") 405 | if o == "POEORB": 406 | return "precise" 407 | elif o == "RESORB": 408 | return "restituted" 409 | elif o == "PREORB": 410 | return "predicted" 411 | else: 412 | raise ValueError("unknown orbit type: %s" % self.filename) 413 | 414 | @property 415 | def date(self): 416 | """Date of acquisition: shortcut for start_time.date()""" 417 | return self.start_time.date() 418 | -------------------------------------------------------------------------------- /eof/dataspace_client.py: -------------------------------------------------------------------------------- 1 | """Client to get orbit files from dataspace.copernicus.eu .""" 2 | 3 | from __future__ import annotations 4 | 5 | from concurrent.futures import ThreadPoolExecutor 6 | from datetime import datetime, timedelta 7 | from pathlib import Path 8 | from typing import Optional 9 | 10 | import requests 11 | 12 | from ._auth import DATASPACE_HOST, get_netrc_credentials 13 | from ._types import Filename 14 | from .log import logger 15 | from .products import Sentinel as S1Product 16 | 17 | QUERY_URL = "https://catalogue.dataspace.copernicus.eu/odata/v1/Products" 18 | """Default URL endpoint for the Copernicus Data Space Ecosystem (CDSE) query REST service""" 19 | 20 | AUTH_URL = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token" 21 | """Default URL endpoint for performing user authentication with CDSE""" 22 | 23 | DOWNLOAD_URL = "https://zipper.dataspace.copernicus.eu/odata/v1/Products" 24 | """Default URL endpoint for CDSE download REST service""" 25 | 26 | SIGNUP_URL = "https://dataspace.copernicus.eu/" 27 | """Url to prompt user to sign up for CDSE account.""" 28 | 29 | 30 | class DataspaceClient: 31 | T0 = timedelta(seconds=60) 32 | T1 = timedelta(seconds=60) 33 | 34 | def __init__( 35 | self, 36 | access_token: Optional[str] = None, 37 | username: str = "", 38 | password: str = "", 39 | token_2fa: str = "", 40 | netrc_file: Optional[Filename] = None, 41 | ): 42 | self._access_token = access_token 43 | if access_token: 44 | logger.debug("Using provided CDSE access token") 45 | else: 46 | try: 47 | if not (username and password): 48 | logger.debug(f"Get credentials form netrc ({netrc_file!r})") 49 | # Shall we keep username if explicitly set? 50 | username, password = get_netrc_credentials( 51 | DATASPACE_HOST, netrc_file 52 | ) 53 | else: 54 | logger.debug("Using provided username and password") 55 | self._access_token = get_access_token(username, password, token_2fa) 56 | except FileNotFoundError: 57 | logger.warning("No netrc file found.") 58 | except ValueError as e: 59 | if DATASPACE_HOST not in e.args[0]: 60 | raise e 61 | logger.warning( 62 | f"No CDSE credentials found in netrc file {netrc_file!r}. Please create one using {SIGNUP_URL}" 63 | ) 64 | except Exception as e: 65 | logger.warning(f"Error: {str(e)}") 66 | 67 | # Obtain an access token the download request from the provided credentials 68 | 69 | def __bool__(self): 70 | """Tells whether the object has been correctly initialized""" 71 | return bool(self._access_token) 72 | 73 | @staticmethod 74 | def query_orbit( 75 | t0: datetime, 76 | t1: datetime, 77 | satellite_id: str, 78 | product_type: str = "AUX_POEORB", 79 | ) -> list[dict]: 80 | assert satellite_id in {"S1A", "S1B", "S1C"} 81 | assert product_type in {"AUX_POEORB", "AUX_RESORB"} 82 | # return run_query(t0, t1, satellite_id, product_type) 83 | # Construct the query based on the time range parsed from the input file 84 | logger.info( 85 | f"Querying for {product_type} orbit files from endpoint {QUERY_URL}" 86 | ) 87 | query = _construct_orbit_file_query(satellite_id, product_type, t0, t1) 88 | # Make the query to determine what Orbit files are available for the time 89 | # range 90 | return query_orbit_file_service(query) 91 | 92 | @staticmethod 93 | def query_orbit_for_product( 94 | product: str | S1Product, 95 | orbit_type: str = "precise", 96 | t0_margin: timedelta = T0, 97 | t1_margin: timedelta = T1, 98 | ): 99 | if isinstance(product, str): 100 | product = S1Product(product) 101 | 102 | return DataspaceClient.query_orbit_by_dt( 103 | [product.start_time], 104 | [product.mission], 105 | orbit_type=orbit_type, 106 | t0_margin=t0_margin, 107 | t1_margin=t1_margin, 108 | ) 109 | 110 | @staticmethod 111 | def query_orbit_by_dt( 112 | orbit_dts, 113 | missions, 114 | orbit_type: str = "precise", 115 | t0_margin: timedelta = T0, 116 | t1_margin: timedelta = T1, 117 | ): 118 | """Query the Scihub api for product info for the specified missions/orbit_dts. 119 | 120 | Parameters 121 | ---------- 122 | orbit_dts : list[datetime.datetime] 123 | List of datetimes to query for 124 | missions : list[str], choices = {"S1A", "S1B", "S1C"} 125 | List of missions to query for. Must be same length as orbit_dts 126 | orbit_type : str, choices = {"precise", "restituted"} 127 | String identifying the type of orbit file to query for. 128 | t0_margin : timedelta 129 | Margin to add to the start time of the orbit file in the query 130 | t1_margin : timedelta 131 | Margin to add to the end time of the orbit file in the query 132 | 133 | Returns 134 | ------- 135 | list[dict] 136 | list of results from the query 137 | """ 138 | remaining_dates: list[tuple[str, datetime]] = [] 139 | all_results = [] 140 | for dt, mission in zip(orbit_dts, missions): 141 | # Only check for precise orbits if that is what we want 142 | if orbit_type == "precise": 143 | products = DataspaceClient.query_orbit( 144 | dt - t0_margin, 145 | dt + t1_margin, 146 | # dt - timedelta(seconds=T_ORBIT + 60), 147 | # dt + timedelta(seconds=60), 148 | mission, 149 | product_type="AUX_POEORB", 150 | ) 151 | if len(products) == 1: 152 | result = products[0] 153 | elif len(products) > 1: 154 | logger.warning(f"Found more than one result: {products}") 155 | result = products[0] 156 | else: 157 | result = None 158 | else: 159 | result = None 160 | 161 | if result is not None: 162 | all_results.append(result) 163 | else: 164 | # try with RESORB 165 | products = DataspaceClient.query_orbit( 166 | dt - t0_margin, 167 | dt + t1_margin, 168 | mission, 169 | product_type="AUX_RESORB", 170 | ) 171 | if len(products) == 1: 172 | result = products[0] 173 | elif len(products) > 1: 174 | logger.warning(f"Found more than one result: {products}") 175 | result = products[0] 176 | else: 177 | result = None 178 | logger.warning(f"Found no restituted results for {dt} {mission}") 179 | 180 | if result: 181 | all_results.append(result) 182 | 183 | if result is None: 184 | remaining_dates.append((mission, dt)) 185 | 186 | if remaining_dates: 187 | logger.warning("The following dates were not found: %s", remaining_dates) 188 | return all_results 189 | 190 | def download_all( 191 | self, 192 | query_results: list[dict], 193 | output_directory: Filename, 194 | max_workers: int = 3, 195 | ): 196 | """Download all the specified orbit products.""" 197 | return download_all( 198 | query_results, 199 | output_directory=output_directory, 200 | access_token=self._access_token, 201 | max_workers=max_workers, 202 | ) 203 | 204 | 205 | def _construct_orbit_file_query( 206 | mission_id: str, orbit_type: str, search_start: datetime, search_stop: datetime 207 | ): 208 | """Constructs the query used with the query URL to determine the 209 | available Orbit files for the given time range. 210 | 211 | Parameters 212 | ---------- 213 | mission_id : str 214 | The mission ID parsed from the SAFE file name, should always be one 215 | of S1A, S1B or S1C. 216 | orbit_type : str 217 | String identifying the type of orbit file to query for. Should be either 218 | POEORB for Precise Orbit files, or RESORB for Restituted. 219 | search_start : datetime 220 | The start time to use with the query in YYYYmmddTHHMMSS format. 221 | Any resulting orbit files will have a starting time before this value. 222 | search_stop : datetime 223 | The stop time to use with the query in YYYYmmddTHHMMSS format. 224 | Any resulting orbit files will have an ending time after this value. 225 | 226 | Returns 227 | ------- 228 | query : str 229 | The Orbit file query string formatted as the query service expects. 230 | 231 | """ 232 | # Set up templates that use the OData domain specific syntax expected by the 233 | # query service 234 | query_template = ( 235 | "startswith(Name,'{mission_id}') and contains(Name,'{orbit_type}') " 236 | "and ContentDate/Start lt '{start_time}' and ContentDate/End gt '{stop_time}'" 237 | ) 238 | 239 | # Format the query template using the values we were provided 240 | query_start_date_str = search_start.strftime("%Y-%m-%dT%H:%M:%S.%fZ") 241 | query_stop_date_str = search_stop.strftime("%Y-%m-%dT%H:%M:%S.%fZ") 242 | 243 | query = query_template.format( 244 | start_time=query_start_date_str, 245 | stop_time=query_stop_date_str, 246 | mission_id=mission_id, 247 | orbit_type=orbit_type, 248 | ) 249 | 250 | logger.debug(f"query: {query}") 251 | 252 | return query 253 | 254 | 255 | def query_orbit_file_service(query: str) -> list[dict]: 256 | """Submit a request to the Orbit file query REST service. 257 | 258 | Parameters 259 | ---------- 260 | query : str 261 | The query for the Orbit files to find, filtered by a time range and mission 262 | ID corresponding to the provided SAFE SLC archive file. 263 | 264 | Returns 265 | ------- 266 | query_results : list of dict 267 | The list of results from a successful query. Each result should 268 | be a Python dictionary containing the details of the orbit file which 269 | matched the query. 270 | 271 | Raises 272 | ------ 273 | requests.HTTPError 274 | If the request fails for any reason (HTTP return code other than 200). 275 | 276 | References 277 | ---------- 278 | .. [1] https://documentation.dataspace.copernicus.eu/APIs/OData.html#query-by-sensing-date 279 | """ 280 | # Set up parameters to be included with query request 281 | query_params = {"$filter": query, "$orderby": "ContentDate/Start asc", "$top": 1} 282 | 283 | # Make the HTTP GET request on the endpoint URL, no credentials are required 284 | response = requests.get(QUERY_URL, params=query_params) # type: ignore 285 | 286 | logger.debug(f"response.url: {response.url}") 287 | logger.debug(f"response.status_code: {response.status_code}") 288 | 289 | response.raise_for_status() 290 | 291 | # Response should be within the text body as JSON 292 | json_response = response.json() 293 | logger.debug(f"json_response: {json_response}") 294 | 295 | query_results = json_response["value"] 296 | 297 | return query_results 298 | 299 | 300 | def get_access_token( 301 | username: Optional[str], password: Optional[str], token_2fa: Optional[str] 302 | ) -> str: 303 | """Get an access token for the Copernicus Data Space Ecosystem (CDSE) API. 304 | 305 | Code from https://documentation.dataspace.copernicus.eu/APIs/Token.html 306 | 307 | :raises ValueError: if either username or password is empty 308 | :raises RuntimeError: if the access token cannot be created 309 | """ 310 | if not (username and password): 311 | raise ValueError("Username and password values are expected!") 312 | 313 | data = { 314 | "client_id": "cdse-public", 315 | "username": username, 316 | "password": password, 317 | "grant_type": "password", 318 | } 319 | if token_2fa: # Double authentication is used 320 | data["totp"] = token_2fa 321 | 322 | try: 323 | r = requests.post(AUTH_URL, data=data) 324 | r.raise_for_status() 325 | except Exception as err: 326 | raise RuntimeError(f"CDSE access token creation failed. Reason: {str(err)}") 327 | 328 | # Parse the access token from the response 329 | try: 330 | access_token = r.json()["access_token"] 331 | return access_token 332 | except KeyError: 333 | raise RuntimeError( 334 | 'Failed to parse expected field "access_token" from CDSE authentication response.' 335 | ) 336 | 337 | 338 | def download_orbit_file( 339 | request_url, output_directory, orbit_file_name, access_token 340 | ) -> Path: 341 | """Downloads an Orbit file using the provided request URL. 342 | 343 | Should contain product ID for the file to download, as obtained from a query result. 344 | 345 | The output file is named according to the orbit_file_name parameter, and 346 | should correspond to the file name parsed from the query result. The output 347 | file is written to the directory indicated by output_directory. 348 | 349 | Parameters 350 | ---------- 351 | request_url : str 352 | The full request URL, which includes the download endpoint, as well as 353 | a payload that contains the product ID for the Orbit file to be downloaded. 354 | output_directory : str 355 | The directory to store the downloaded Orbit file to. 356 | orbit_file_name : str 357 | The file name to assign to the Orbit file once downloaded to disk. This 358 | should correspond to the file name parsed from a query result. 359 | access_token : str 360 | Access token returned from an authentication request with the provided 361 | username and password. Must be provided with all download requests for 362 | the download service to respond. 363 | 364 | Returns 365 | ------- 366 | output_orbit_file_path : Path 367 | The full path to where the resulting Orbit file was downloaded to. 368 | 369 | Raises 370 | ------ 371 | requests.HTTPError 372 | If the request fails for any reason (HTTP return code other than 200). 373 | 374 | """ 375 | # Make the HTTP GET request to obtain the Orbit file contents 376 | headers = {"Authorization": f"Bearer {access_token}"} 377 | session = requests.Session() 378 | session.headers.update(headers) 379 | response = session.get(request_url, headers=headers, stream=True) 380 | 381 | logger.debug(f"r.url: {response.url}") 382 | logger.debug(f"r.status_code: {response.status_code}") 383 | 384 | response.raise_for_status() 385 | 386 | # Write the contents to disk 387 | output_orbit_file_path = Path(output_directory) / orbit_file_name 388 | 389 | with open(output_orbit_file_path, "wb") as outfile: 390 | for chunk in response.iter_content(chunk_size=8192): 391 | if chunk: 392 | outfile.write(chunk) 393 | 394 | logger.info(f"Orbit file downloaded to {output_orbit_file_path!r}") 395 | return output_orbit_file_path 396 | 397 | 398 | def download_all( 399 | query_results: list[dict], 400 | output_directory: Filename, 401 | access_token: Optional[str], 402 | max_workers: int = 3, 403 | ) -> list[Path]: 404 | """Download all the specified orbit products. 405 | 406 | Parameters 407 | ---------- 408 | query_results : list[dict] 409 | list of results from the query 410 | output_directory : str | Path 411 | Directory to save the orbit files to. 412 | username : str 413 | CDSE username 414 | password : str 415 | CDSE password 416 | token_2fa : str 417 | 2FA Token used in profiles with double authentication 418 | max_workers : int, default = 3 419 | Maximum parallel downloads from CDSE. 420 | Note that >4 connections will result in a HTTP 429 Error 421 | 422 | """ 423 | if not access_token: 424 | raise RuntimeError("Invalid CDSE access token. Aborting.") 425 | downloaded_paths: list[Path] = [] 426 | # Select an appropriate orbit file from the list returned from the query 427 | # orbit_file_name, orbit_file_request_id = select_orbit_file( 428 | # query_results, start_time, stop_time 429 | # ) 430 | 431 | output_names = [] 432 | download_urls = [] 433 | for query_result in query_results: 434 | orbit_file_request_id = query_result["Id"] 435 | 436 | # Construct the URL used to download the Orbit file 437 | download_url = f"{DOWNLOAD_URL}({orbit_file_request_id})/$value" 438 | download_urls.append(download_url) 439 | 440 | orbit_file_name = query_result["Name"] 441 | output_names.append(orbit_file_name) 442 | 443 | logger.debug( 444 | f"Downloading Orbit file {orbit_file_name} from service endpoint " 445 | f"{download_url}" 446 | ) 447 | 448 | downloaded_paths = [] 449 | with ThreadPoolExecutor(max_workers=max_workers) as exc: 450 | futures = [ 451 | exc.submit( 452 | download_orbit_file, 453 | request_url=u, 454 | output_directory=output_directory, 455 | orbit_file_name=n, 456 | access_token=access_token, 457 | ) 458 | for (u, n) in zip(download_urls, output_names) 459 | ] 460 | for f in futures: 461 | downloaded_paths.append(f.result()) 462 | 463 | return downloaded_paths 464 | --------------------------------------------------------------------------------