├── cogeo_mosaic ├── py.typed ├── scripts │ └── __init__.py ├── __init__.py ├── logger.py ├── models.py ├── backends │ ├── utils.py │ ├── memory.py │ ├── file.py │ ├── __init__.py │ ├── web.py │ ├── s3.py │ ├── gs.py │ ├── az.py │ ├── stac.py │ ├── base.py │ └── dynamodb.py ├── errors.py ├── cache.py ├── utils.py └── mosaic.py ├── docs ├── src │ ├── index.md │ ├── release-notes.md │ ├── contributing.md │ ├── v6_migration.md │ ├── advanced │ │ ├── readers.md │ │ ├── dynamic.md │ │ ├── custom.md │ │ └── backends.md │ ├── cli.md │ ├── v3_migration.md │ ├── intro.md │ └── examples │ │ ├── Create_a_Dynamic_StacBackend.ipynb │ │ └── Create_a_Dynamic_RtreeBackend.ipynb └── mkdocs.yml ├── tests ├── fixtures │ ├── cog1.tif │ ├── cog2.tif │ ├── mosaic.bin │ ├── mosaics.db │ ├── cog1_small.tif │ ├── mosaic.json.gz │ ├── cog1_uint32.tif │ ├── mosaic_0.0.1.json │ ├── mosaic.json │ └── mars_ctx_stac_asset.json ├── test_cache_config.py ├── test_backends_utils.py ├── test_utils.py ├── test_model.py ├── test_create.py └── test_cli.py ├── .github ├── codecov.yml └── workflows │ ├── deploy_mkdocs.yml │ └── ci.yml ├── .pre-commit-config.yaml ├── LICENSE ├── .gitignore ├── CONTRIBUTING.md ├── README.md ├── pyproject.toml └── CHANGES.md /cogeo_mosaic/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/src/index.md: -------------------------------------------------------------------------------- 1 | ../../README.md -------------------------------------------------------------------------------- /docs/src/release-notes.md: -------------------------------------------------------------------------------- 1 | ../../CHANGES.md -------------------------------------------------------------------------------- /docs/src/contributing.md: -------------------------------------------------------------------------------- 1 | ../../CONTRIBUTING.md -------------------------------------------------------------------------------- /cogeo_mosaic/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | """cogeo_mosaic: cli.""" 2 | -------------------------------------------------------------------------------- /cogeo_mosaic/__init__.py: -------------------------------------------------------------------------------- 1 | """Cogeo_mosaic.""" 2 | 3 | __version__ = "9.0.2" 4 | -------------------------------------------------------------------------------- /tests/fixtures/cog1.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/developmentseed/cogeo-mosaic/HEAD/tests/fixtures/cog1.tif -------------------------------------------------------------------------------- /tests/fixtures/cog2.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/developmentseed/cogeo-mosaic/HEAD/tests/fixtures/cog2.tif -------------------------------------------------------------------------------- /tests/fixtures/mosaic.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/developmentseed/cogeo-mosaic/HEAD/tests/fixtures/mosaic.bin -------------------------------------------------------------------------------- /tests/fixtures/mosaics.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/developmentseed/cogeo-mosaic/HEAD/tests/fixtures/mosaics.db -------------------------------------------------------------------------------- /tests/fixtures/cog1_small.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/developmentseed/cogeo-mosaic/HEAD/tests/fixtures/cog1_small.tif -------------------------------------------------------------------------------- /tests/fixtures/mosaic.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/developmentseed/cogeo-mosaic/HEAD/tests/fixtures/mosaic.json.gz -------------------------------------------------------------------------------- /tests/fixtures/cog1_uint32.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/developmentseed/cogeo-mosaic/HEAD/tests/fixtures/cog1_uint32.tif -------------------------------------------------------------------------------- /cogeo_mosaic/logger.py: -------------------------------------------------------------------------------- 1 | """cogeo-mosaic logger.""" 2 | 3 | import logging 4 | 5 | logger = logging.getLogger("cogeo-mosaic") 6 | -------------------------------------------------------------------------------- /.github/codecov.yml: -------------------------------------------------------------------------------- 1 | comment: off 2 | 3 | coverage: 4 | status: 5 | project: 6 | default: 7 | target: auto 8 | threshold: 5 9 | -------------------------------------------------------------------------------- /tests/test_cache_config.py: -------------------------------------------------------------------------------- 1 | from cogeo_mosaic.cache import CacheSettings 2 | 3 | 4 | def test_cache_config(monkeypatch): 5 | monkeypatch.setenv("COGEO_MOSAIC_CACHE_TTL", "500") 6 | 7 | conf = CacheSettings() 8 | assert conf.ttl == 500 9 | assert conf.maxsize == 512 10 | 11 | conf = CacheSettings(disable=True) 12 | assert conf.ttl == 0 13 | assert conf.maxsize == 0 14 | -------------------------------------------------------------------------------- /cogeo_mosaic/models.py: -------------------------------------------------------------------------------- 1 | """cogeo-mosaic models.""" 2 | 3 | from typing import List, Optional 4 | 5 | from pydantic import Field 6 | from rio_tiler.mosaic.backend import MosaicInfo 7 | 8 | 9 | class Info(MosaicInfo): 10 | """Mosaic info responses.""" 11 | 12 | name: Optional[str] = None 13 | quadkeys: List[str] = [] 14 | mosaic_tilematrixset: Optional[str] = None 15 | mosaic_minzoom: int = Field(0, ge=0, le=30) 16 | mosaic_maxzoom: int = Field(30, ge=0, le=30) 17 | -------------------------------------------------------------------------------- /tests/fixtures/mosaic_0.0.1.json: -------------------------------------------------------------------------------- 1 | {"mosaicjson": "0.0.1", "version": "1.0.0", "minzoom": 7, "maxzoom": 9, "bounds": [-75.98703377413767, 44.93504283293786, -71.337604723999, 47.09685599202324], "center": [-73.66231924906833, 46.01594941248055, 7], "tiles": {"0302300": ["cog1.tif"], "0302301": ["cog1.tif", "cog2.tif"], "0302310": ["cog1.tif", "cog2.tif"], "0302302": ["cog1.tif"], "0302303": ["cog1.tif", "cog2.tif"], "0302312": ["cog1.tif", "cog2.tif"], "0302320": ["cog1.tif"], "0302321": ["cog1.tif", "cog2.tif"], "0302330": ["cog1.tif", "cog2.tif"]}} 2 | -------------------------------------------------------------------------------- /tests/fixtures/mosaic.json: -------------------------------------------------------------------------------- 1 | {"mosaicjson": "0.0.2", "version": "1.0.0", "minzoom": 7, "maxzoom": 9, "quadkey_zoom": 7, "bounds": [-75.98703377413767, 44.93504283293786, -71.337604723999, 47.09685599202324], "center": [-73.66231924906833, 46.01594941248055, 7], "tiles": {"0302300": ["cog1.tif"], "0302301": ["cog1.tif", "cog2.tif"], "0302310": ["cog1.tif", "cog2.tif"], "0302302": ["cog1.tif"], "0302303": ["cog1.tif", "cog2.tif"], "0302312": ["cog1.tif", "cog2.tif"], "0302320": ["cog1.tif"], "0302321": ["cog1.tif", "cog2.tif"], "0302330": ["cog1.tif", "cog2.tif"]}} -------------------------------------------------------------------------------- /cogeo_mosaic/backends/utils.py: -------------------------------------------------------------------------------- 1 | """cogeo-mosaic.backends utility functions.""" 2 | 3 | import hashlib 4 | import json 5 | import zlib 6 | from typing import Any 7 | 8 | 9 | def _compress_gz_json(data: str) -> bytes: 10 | gzip_compress = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16) 11 | 12 | return gzip_compress.compress(data.encode("utf-8")) + gzip_compress.flush() 13 | 14 | 15 | def _decompress_gz(gzip_buffer: bytes): 16 | return zlib.decompress(gzip_buffer, zlib.MAX_WBITS | 16).decode() 17 | 18 | 19 | def get_hash(**kwargs: Any) -> str: 20 | """Create hash from a dict.""" 21 | return hashlib.sha224( 22 | json.dumps(kwargs, sort_keys=True, default=str).encode() 23 | ).hexdigest() 24 | -------------------------------------------------------------------------------- /cogeo_mosaic/backends/memory.py: -------------------------------------------------------------------------------- 1 | """cogeo-mosaic In-Memory backend.""" 2 | 3 | import attr 4 | 5 | from cogeo_mosaic.backends.base import MosaicJSONBackend 6 | from cogeo_mosaic.mosaic import MosaicJSON 7 | 8 | 9 | @attr.s 10 | class MemoryBackend(MosaicJSONBackend): 11 | """InMemory Backend Adapter 12 | 13 | Examples: 14 | >>> with MemoryBackend(mosaicJSON) as mosaic: 15 | mosaic.tile(0, 0, 0) 16 | """ 17 | 18 | # We put `input` outside the init method 19 | input: str = attr.ib(init=False, default=":memory:") 20 | 21 | _backend_name = "MEM" 22 | 23 | def write(self, overwrite: bool = True): 24 | """Write mosaicjson document.""" 25 | pass 26 | 27 | def _read(self) -> MosaicJSON: 28 | pass 29 | -------------------------------------------------------------------------------- /.github/workflows/deploy_mkdocs.yml: -------------------------------------------------------------------------------- 1 | name: Publish docs via GitHub Pages 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | # Only rebuild website when docs have changed 9 | - 'README.md' 10 | - 'CHANGES.md' 11 | - 'CONTRIBUTING.md' 12 | - 'docs/**' 13 | - '.github/workflows/deploy_mkdocs.yml' 14 | 15 | jobs: 16 | build: 17 | name: Deploy docs 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Checkout main 21 | uses: actions/checkout@v5 22 | 23 | - name: Install uv 24 | uses: astral-sh/setup-uv@v7 25 | with: 26 | version: "0.9.*" 27 | enable-cache: true 28 | 29 | - name: Deploy docs 30 | run: uv run --group docs mkdocs gh-deploy --force -f docs/mkdocs.yml 31 | 32 | -------------------------------------------------------------------------------- /cogeo_mosaic/errors.py: -------------------------------------------------------------------------------- 1 | """Custom exceptions""" 2 | 3 | from rio_tiler.errors import NoAssetFoundError # noqa 4 | 5 | 6 | class MosaicError(Exception): 7 | """Base exception""" 8 | 9 | 10 | class MosaicAuthError(MosaicError): 11 | """Authentication error""" 12 | 13 | 14 | class MosaicNotFoundError(MosaicError): 15 | """Mosaic not found error""" 16 | 17 | 18 | class MosaicExists(MosaicError): 19 | """MosaicJSON already exists.""" 20 | 21 | 22 | class MosaicExistsError(MosaicError): 23 | """MosaicJSON already exists.""" 24 | 25 | 26 | class MultipleDataTypeError(MosaicError): 27 | """Can create mosaic will dataset of multiple datatype.""" 28 | 29 | 30 | _HTTP_EXCEPTIONS = { 31 | 401: MosaicAuthError, 32 | 403: MosaicAuthError, 33 | 404: MosaicNotFoundError, 34 | } 35 | 36 | _FILE_EXCEPTIONS = {FileNotFoundError: MosaicNotFoundError} 37 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/abravalheri/validate-pyproject 3 | rev: v0.24 4 | hooks: 5 | - id: validate-pyproject 6 | 7 | - repo: https://github.com/PyCQA/isort 8 | rev: 5.12.0 9 | hooks: 10 | - id: isort 11 | language_version: python 12 | 13 | - repo: https://github.com/astral-sh/ruff-pre-commit 14 | rev: v0.3.5 15 | hooks: 16 | - id: ruff 17 | args: ["--fix"] 18 | - id: ruff-format 19 | 20 | - repo: https://github.com/pre-commit/mirrors-mypy 21 | rev: v1.11.2 22 | hooks: 23 | - id: mypy 24 | language_version: python 25 | # No reason to run if only tests have changed. They intentionally break typing. 26 | exclude: tests/.* 27 | additional_dependencies: 28 | - types-attrs 29 | - types-cachetools 30 | - types-setuptools 31 | -------------------------------------------------------------------------------- /cogeo_mosaic/cache.py: -------------------------------------------------------------------------------- 1 | """cogeo-mosaic cache configuration""" 2 | 3 | from pydantic import model_validator 4 | from pydantic_settings import BaseSettings, SettingsConfigDict 5 | 6 | 7 | class CacheSettings(BaseSettings): 8 | """Application settings""" 9 | 10 | # TTL of the cache in seconds 11 | ttl: int = 300 12 | 13 | # Maximum size of the LRU cache in Number of Items 14 | maxsize: int = 512 15 | 16 | # Whether or not caching is enabled 17 | disable: bool = False 18 | 19 | model_config = SettingsConfigDict(env_prefix="COGEO_MOSAIC_CACHE_") 20 | 21 | @model_validator(mode="before") 22 | def check_enable(cls, values): 23 | """Check if cache is disabled.""" 24 | if values.get("disable"): 25 | values["ttl"] = 0 26 | values["maxsize"] = 0 27 | 28 | return values 29 | 30 | 31 | cache_config = CacheSettings() 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Development Seed 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/test_backends_utils.py: -------------------------------------------------------------------------------- 1 | """Test backends utils.""" 2 | 3 | import json 4 | import os 5 | import re 6 | 7 | from cogeo_mosaic.backends import utils 8 | 9 | mosaic_gz = os.path.join(os.path.dirname(__file__), "fixtures", "mosaic.json.gz") 10 | mosaic_json = os.path.join(os.path.dirname(__file__), "fixtures", "mosaic.json") 11 | 12 | with open(mosaic_json, "r") as f: 13 | mosaic_content = json.loads(f.read()) 14 | 15 | 16 | def test_decompress(): 17 | """Test valid gz decompression.""" 18 | with open(mosaic_gz, "rb") as f: 19 | body = f.read() 20 | res = json.loads(utils._decompress_gz(body)) 21 | assert sorted(res.keys()) == sorted( 22 | [ 23 | "mosaicjson", 24 | "quadkey_zoom", 25 | "minzoom", 26 | "maxzoom", 27 | "bounds", 28 | "center", 29 | "tiles", 30 | "version", 31 | ] 32 | ) 33 | 34 | 35 | def test_compress(): 36 | """Test valid gz compression.""" 37 | with open(mosaic_json, "r") as f: 38 | mosaic = json.loads(f.read()) 39 | 40 | body = utils._compress_gz_json(json.dumps(mosaic)) 41 | assert isinstance(body, bytes) 42 | res = json.loads(utils._decompress_gz(body)) 43 | assert res == mosaic 44 | 45 | 46 | def test_hash(): 47 | """Should return a 56 characters long string.""" 48 | hash = utils.get_hash(a=1) 49 | assert re.match(r"[0-9A-Fa-f]{56}", hash) 50 | -------------------------------------------------------------------------------- /docs/src/v6_migration.md: -------------------------------------------------------------------------------- 1 | # cogeo-mosaic 5.0 to 6.0 migration guide 2 | 3 | ### MosaicJSON specification `0.0.3` 4 | 5 | Starting with `6.0`, cogeo-mosaic will follow the MosaicJSON [`0.0.3`](https://github.com/developmentseed/mosaicjson-spec/tree/main/0.0.3) specification ([changes](https://github.com/developmentseed/mosaicjson-spec/blob/main/CHANGES.md#003-2023-05-31)). Old mosaic files should still be usable. 6 | 7 | If updating a previous mosaic file, a warning should be printed. 8 | 9 | ### Multiple TileMatrixSets support (create) 10 | 11 | Following specification `0.0.3`, we can now create Mosaics using other TileMatrixSet than the default `WebMercatorQuad`. 12 | 13 | ```python 14 | 15 | import morecantile 16 | from cogeo_mosaic.mosaic import MosaicJSON 17 | 18 | tms_5041 = morecantile.tms.get("UPSArcticWGS84Quad") 19 | mosaic = MosaicJSON.from_urls([...], tilematrixset=tms_5041) 20 | assert mosaic.tilematrixset.id == "UPSArcticWGS84Quad" 21 | ``` 22 | 23 | ### Multiple TileMatrixSets support in Backend (read) 24 | 25 | You can now pass `TileMatrixSet` as input parameters to MosaicBackend to read tiles in other TileMatrixSet than the default `WebMercatorQuad`. 26 | 27 | ```python 28 | import morecantile 29 | from cogeo_mosaic.backends import MosaicBackend 30 | 31 | tms = morecantile.tms.get("WGS1984Quad") 32 | with MosaicBackend(mosaic_path, tms=tms) as mosaic: 33 | img, assets = mosaic.tile(1, 2, 3) 34 | assert img.crs == "epsg:4326" 35 | ``` 36 | 37 | Note: When passing a different TileMatrixSet than the mosaic's TileMatrixSet, the `minzoom/maxzoom` will default to the TileMatrixSet levels. 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | *.c 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | 58 | # Flask stuff: 59 | instance/ 60 | .webassets-cache 61 | 62 | # Scrapy stuff: 63 | .scrapy 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # Jupyter Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # celery beat schedule file 78 | celerybeat-schedule 79 | 80 | # SageMath parsed files 81 | *.sage.py 82 | 83 | # dotenv 84 | .env 85 | 86 | # virtualenv 87 | .venv 88 | venv/ 89 | ENV/ 90 | 91 | # Spyder project settings 92 | .spyderproject 93 | .spyproject 94 | 95 | # Rope project settings 96 | .ropeproject 97 | 98 | # mkdocs documentation 99 | /site 100 | 101 | # mypy 102 | .mypy_cache/ 103 | 104 | 105 | .pytest_cache 106 | 107 | package.zip 108 | 109 | .serverless -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Development - Contributing 2 | 3 | Issues and pull requests are more than welcome. 4 | 5 | We recommand using [`uv`](https://docs.astral.sh/uv) as project manager for development. 6 | 7 | See https://docs.astral.sh/uv/getting-started/installation/ for installation 8 | 9 | ### dev install 10 | 11 | ``` 12 | git clone http://github.com/developmentseed/cogeo-mosaic.git 13 | cd cogeo-mosaic 14 | 15 | uv sync --all-extras 16 | ``` 17 | 18 | You can then run the tests with the following command: 19 | 20 | ```sh 21 | uv run pytest --cov cogeo_mosaic --cov-report term-missing 22 | ``` 23 | 24 | ### pre-commit 25 | 26 | This repo is set to use `pre-commit` to run *isort*, *flake8*, *pydocstring*, *black* ("uncompromising Python code formatter") and mypy when committing new code. 27 | 28 | ```bash 29 | uv run pre-commit install 30 | 31 | git add . 32 | 33 | git commit -m'my change' 34 | isort....................................................................Passed 35 | black....................................................................Passed 36 | Flake8...................................................................Passed 37 | Verifying PEP257 Compliance..............................................Passed 38 | mypy.....................................................................Passed 39 | 40 | git push origin 41 | ``` 42 | 43 | ### Docs 44 | 45 | ```bash 46 | git clone https://github.com/developmentseed/cogeo-mosaic.git 47 | cd cogeo-mosaic 48 | ``` 49 | 50 | Hot-reloading docs: 51 | 52 | ```bash 53 | uv run --group docs mkdocs serve -f docs/mkdocs.yml 54 | ``` 55 | 56 | To manually deploy docs (note you should never need to do this because Github 57 | Actions deploys automatically for new commits.): 58 | 59 | ```bash 60 | uv run --group docs mkdocs gh-deploy -f docs/mkdocs.yml 61 | ``` 62 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | """tests cogeo_mosaic.utils.""" 2 | 3 | import os 4 | from concurrent import futures 5 | 6 | import morecantile 7 | import pytest 8 | 9 | from cogeo_mosaic import utils 10 | 11 | asset1 = os.path.join(os.path.dirname(__file__), "fixtures", "cog1.tif") 12 | asset2 = os.path.join(os.path.dirname(__file__), "fixtures", "cog2.tif") 13 | 14 | 15 | def test_filtering_futurestask(): 16 | """Should filter failed task.""" 17 | 18 | def _is_odd(val: int) -> int: 19 | if not val % 2: 20 | raise ValueError(f"{val} is Even.") 21 | return val 22 | 23 | with futures.ThreadPoolExecutor() as executor: 24 | future_work = [executor.submit(_is_odd, item) for item in range(0, 8)] 25 | assert list(utils._filter_futures(future_work)) == [1, 3, 5, 7] 26 | 27 | with pytest.raises(ValueError): 28 | with futures.ThreadPoolExecutor() as executor: 29 | future_work = [executor.submit(_is_odd, item) for item in range(0, 8)] 30 | [f.result() for f in future_work] 31 | 32 | 33 | def test_dataset_info(): 34 | """Read raster metadata and return spatial info.""" 35 | info = utils.get_dataset_info(asset1) 36 | assert info["geometry"] 37 | assert info["properties"]["path"] 38 | assert info["properties"]["bounds"] 39 | assert info["properties"]["datatype"] 40 | assert info["properties"]["minzoom"] == 7 41 | assert info["properties"]["maxzoom"] == 9 42 | 43 | 44 | def test_footprint(): 45 | """Fetch footprints from asset list.""" 46 | assets = [asset1, asset2] 47 | foot = utils.get_footprints(assets) 48 | assert len(foot) == 2 49 | 50 | 51 | def test_tiles_to_bounds(): 52 | """Get tiles bounds for zoom level.""" 53 | tiles = [morecantile.Tile(x=150, y=182, z=9), morecantile.Tile(x=151, y=182, z=9)] 54 | assert len(utils.tiles_to_bounds(tiles)) == 4 55 | -------------------------------------------------------------------------------- /docs/src/advanced/readers.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## COGReader / STACReader 4 | 5 | The MosaicJSON backend classes have `.tile` and `.point` methods to access the data for a specific mercator tile or point. 6 | 7 | Because a MosaicJSON can host different assets type, a `reader` option is available. 8 | Set by default to `rio_tiler.io.COGReader`, or to `rio_tiler.io.STACReader` for the STACBackend, the reader should know how to read the assets to either create mosaic tile or read points value. 9 | 10 | ```python 11 | from cogeo_mosaic.mosaic import MosaicJSON 12 | from cogeo_mosaic.backends import MosaicBackend 13 | from rio_tiler.models import ImageData 14 | 15 | dataset = ["1.tif", "2.tif"] 16 | mosaic_definition = MosaicJSON.from_urls(dataset) 17 | 18 | # Create a mosaic object in memory 19 | with MosaicBackend(None, mosaid_def=mosaic_definition, reader=COGReader) as mosaic: 20 | img, assets_used = mosaic.tile(1, 1, 1) 21 | assert isinstance(img, ImageData) 22 | 23 | # By default the STACbackend will store the Item url as assets, but STACReader (default reader) will know how to read them. 24 | with MosaicBackend( 25 | "stac+https://my-stac.api/search", 26 | query={"collections": ["satellite"]}, # required 27 | minzoom=7, # required 28 | maxzoom=12, # required 29 | ) as mosaic: 30 | img, assets_used = mosaic.tile(1, 1, 1, assets="red") 31 | ``` 32 | 33 | Let's use a custom accessor to save some specific assets url in the mosaic 34 | 35 | ```python 36 | # accessor to return the url for the `visual` asset (COG) 37 | def accessor(item): 38 | return feature["assets"]["visual"]["href"] 39 | 40 | # The accessor will set the mosaic assets as a list of COG url so we can use the COGReader instead of the STACReader 41 | with MosaicBackend( 42 | "stac+https://my-stac.api/search", 43 | query={"collections": ["satellite"]}, # required 44 | minzoom=7, # required 45 | maxzoom=12, # required 46 | reader=COGReader, 47 | mosaic_options={"accessor": accessor}, 48 | ) as mosaic: 49 | img, assets_used = mosaic.tile(1, 1, 1) 50 | ``` 51 | -------------------------------------------------------------------------------- /cogeo_mosaic/backends/file.py: -------------------------------------------------------------------------------- 1 | """cogeo-mosaic File backend.""" 2 | 3 | import json 4 | import pathlib 5 | from threading import Lock 6 | 7 | import attr 8 | from cachetools import TTLCache, cached 9 | from cachetools.keys import hashkey 10 | 11 | from cogeo_mosaic.backends.base import MosaicJSONBackend 12 | from cogeo_mosaic.backends.utils import _compress_gz_json, _decompress_gz 13 | from cogeo_mosaic.cache import cache_config 14 | from cogeo_mosaic.errors import _FILE_EXCEPTIONS, MosaicError, MosaicExistsError 15 | from cogeo_mosaic.mosaic import MosaicJSON 16 | 17 | 18 | @attr.s 19 | class FileBackend(MosaicJSONBackend): 20 | """Local File Backend Adapter""" 21 | 22 | _backend_name = "File" 23 | 24 | def write(self, overwrite: bool = False): 25 | """Write mosaicjson document to a file.""" 26 | if not overwrite and pathlib.Path(self.input).exists(): 27 | raise MosaicExistsError("Mosaic file already exist, use `overwrite=True`.") 28 | 29 | body = self.mosaic_def.model_dump_json(exclude_none=True) 30 | with open(self.input, "wb") as f: 31 | try: 32 | if self.input.endswith(".gz"): 33 | f.write(_compress_gz_json(body)) 34 | else: 35 | f.write(body.encode("utf-8")) 36 | except Exception as e: 37 | exc = _FILE_EXCEPTIONS.get(e, MosaicError) # type: ignore 38 | raise exc(str(e)) from e 39 | 40 | @cached( # type: ignore 41 | TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl), 42 | key=lambda self: hashkey(self.input), 43 | lock=Lock(), 44 | ) 45 | def _read(self) -> MosaicJSON: # type: ignore 46 | """Get mosaicjson document.""" 47 | try: 48 | with open(self.input, "rb") as f: 49 | body = f.read() 50 | except Exception as e: 51 | exc = _FILE_EXCEPTIONS.get(e, MosaicError) # type: ignore 52 | raise exc(str(e)) from e 53 | 54 | self._file_byte_size = len(body) 55 | 56 | if self.input.endswith(".gz"): 57 | body = _decompress_gz(body) 58 | 59 | return MosaicJSON(**json.loads(body)) 60 | -------------------------------------------------------------------------------- /docs/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: cogeo-mosaic 2 | site_description: Create and use mosaics of Cloud Optimized GeoTIFF based on mosaicJSON specification. 3 | 4 | docs_dir: 'src' 5 | site_dir: 'build' 6 | 7 | repo_name: "developmentseed/cogeo-mosaic" 8 | repo_url: "https://github.com/developmentseed/cogeo-mosaic" 9 | edit_uri: 'blob/main/docs/src/' 10 | 11 | extra: 12 | social: 13 | - icon: "fontawesome/brands/github" 14 | link: "https://github.com/developmentseed" 15 | - icon: "fontawesome/brands/twitter" 16 | link: "https://twitter.com/developmentseed" 17 | - icon: "fontawesome/brands/medium" 18 | link: "https://medium.com/devseed" 19 | 20 | nav: 21 | - Home: "index.md" 22 | - User Guide: 23 | - Intro: "intro.md" 24 | - CLI: "cli.md" 25 | 26 | - Advanced User Guide: 27 | - Backends: "advanced/backends.md" 28 | - Reader: "advanced/readers.md" 29 | - Customization: "advanced/custom.md" 30 | - Dynamic Backends: "advanced/dynamic.md" 31 | 32 | - Examples: 33 | - Dynamic RTree backend: "examples/Create_a_Dynamic_RtreeBackend.ipynb" 34 | - Dynamic STAC backend: "examples/Create_a_Dynamic_StacBackend.ipynb" 35 | 36 | - Migration Guides: 37 | - Migration to v3.0: "v3_migration.md" 38 | - Migration to v6.0: "v6_migration.md" 39 | 40 | - Development - Contributing: "contributing.md" 41 | - Release Notes: "release-notes.md" 42 | 43 | plugins: 44 | - search 45 | - mkdocs-jupyter: 46 | include_source: True 47 | 48 | theme: 49 | name: material 50 | palette: 51 | primary: indigo 52 | scheme: default 53 | 54 | markdown_extensions: 55 | - admonition 56 | - attr_list 57 | - codehilite: 58 | guess_lang: false 59 | - def_list 60 | - footnotes 61 | - pymdownx.arithmatex 62 | - pymdownx.betterem 63 | - pymdownx.caret: 64 | insert: false 65 | - pymdownx.details 66 | - pymdownx.emoji 67 | - pymdownx.escapeall: 68 | hardbreak: true 69 | nbsp: true 70 | - pymdownx.magiclink: 71 | hide_protocol: true 72 | repo_url_shortener: true 73 | - pymdownx.smartsymbols 74 | - pymdownx.superfences 75 | - pymdownx.tasklist: 76 | custom_checkbox: true 77 | - pymdownx.tilde 78 | - toc: 79 | permalink: true 80 | -------------------------------------------------------------------------------- /cogeo_mosaic/backends/__init__.py: -------------------------------------------------------------------------------- 1 | """cogeo_mosaic.backends.""" 2 | 3 | from typing import Any 4 | from urllib.parse import urlparse 5 | 6 | from cogeo_mosaic.backends.az import ABSBackend 7 | from cogeo_mosaic.backends.base import MosaicJSONBackend 8 | from cogeo_mosaic.backends.dynamodb import DynamoDBBackend 9 | from cogeo_mosaic.backends.file import FileBackend 10 | from cogeo_mosaic.backends.gs import GCSBackend 11 | from cogeo_mosaic.backends.memory import MemoryBackend 12 | from cogeo_mosaic.backends.s3 import S3Backend 13 | from cogeo_mosaic.backends.sqlite import SQLiteBackend 14 | from cogeo_mosaic.backends.stac import STACBackend 15 | from cogeo_mosaic.backends.web import HttpBackend 16 | 17 | 18 | def MosaicBackend(input: str, *args: Any, **kwargs: Any) -> MosaicJSONBackend: # noqa: C901 19 | """Select mosaic backend for input.""" 20 | parsed = urlparse(input) 21 | 22 | if not input or input == ":memory:": 23 | return MemoryBackend(*args, **kwargs) 24 | 25 | # `stac+https//{hostname}/{path}` 26 | elif parsed.scheme and parsed.scheme.startswith("stac+"): 27 | input = input.replace("stac+", "") 28 | return STACBackend(input, *args, **kwargs) 29 | 30 | # `s3:///{bucket}{key}` 31 | elif parsed.scheme == "s3": 32 | return S3Backend(input, *args, **kwargs) 33 | 34 | # `gs://{bucket}/{key}` 35 | elif parsed.scheme == "gs": 36 | return GCSBackend(input, *args, **kwargs) 37 | 38 | # `az://{storageaccount}.blob.core.windows.net/{container}/{key}` 39 | elif parsed.scheme == "az": 40 | return ABSBackend(input, *args, **kwargs) 41 | 42 | # `dynamodb://{region}/{table}:{mosaic}` 43 | elif parsed.scheme == "dynamodb": 44 | return DynamoDBBackend(input, *args, **kwargs) 45 | 46 | # `sqlite:///{path.db}:{mosaic}` 47 | elif parsed.scheme == "sqlite": 48 | return SQLiteBackend(input, *args, **kwargs) 49 | 50 | # https://{hostname}/{path} 51 | elif parsed.scheme in ["https", "http"]: 52 | return HttpBackend(input, *args, **kwargs) 53 | 54 | # file:///{path} 55 | elif parsed.scheme == "file": 56 | return FileBackend(parsed.path, *args, **kwargs) 57 | 58 | # Invalid Scheme 59 | elif parsed.scheme: 60 | raise ValueError(f"'{parsed.scheme}' is not supported") 61 | 62 | # fallback to FileBackend 63 | else: 64 | return FileBackend(input, *args, **kwargs) 65 | -------------------------------------------------------------------------------- /cogeo_mosaic/backends/web.py: -------------------------------------------------------------------------------- 1 | """cogeo-mosaic HTTP backend. 2 | 3 | This file is named web.py instead of http.py because http is a Python standard 4 | lib module 5 | """ 6 | 7 | import json 8 | from threading import Lock 9 | from typing import Dict, Sequence 10 | 11 | import attr 12 | import httpx 13 | from cachetools import TTLCache, cached 14 | from cachetools.keys import hashkey 15 | 16 | from cogeo_mosaic.backends.base import MosaicJSONBackend 17 | from cogeo_mosaic.backends.utils import _decompress_gz 18 | from cogeo_mosaic.cache import cache_config 19 | from cogeo_mosaic.errors import _HTTP_EXCEPTIONS, MosaicError 20 | from cogeo_mosaic.mosaic import MosaicJSON 21 | 22 | 23 | @attr.s 24 | class HttpBackend(MosaicJSONBackend): 25 | """Http/Https Backend Adapter""" 26 | 27 | # Because the HttpBackend is a Read-Only backend, there is no need for 28 | # mosaic_def to be in the init method. 29 | mosaic_def: MosaicJSON = attr.ib(init=False, default=None) 30 | 31 | _backend_name = "HTTP" 32 | 33 | @cached( # type: ignore 34 | TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl), 35 | key=lambda self: hashkey(self.input), 36 | lock=Lock(), 37 | ) 38 | def _read(self) -> MosaicJSON: # type: ignore 39 | """Get mosaicjson document.""" 40 | try: 41 | r = httpx.get(self.input) 42 | r.raise_for_status() 43 | except httpx.HTTPStatusError as e: 44 | # post-flight errors 45 | status_code = e.response.status_code 46 | exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError) 47 | raise exc(e.response.content) from e 48 | except httpx.RequestError as e: 49 | # pre-flight errors 50 | raise MosaicError(e.args[0].reason) from e 51 | 52 | body = r.content 53 | 54 | self._file_byte_size = len(body) 55 | 56 | if self.input.endswith(".gz"): 57 | body = _decompress_gz(body) 58 | 59 | return MosaicJSON(**json.loads(body)) 60 | 61 | def write(self, overwrite: bool = True): 62 | """Write mosaicjson document.""" 63 | raise NotImplementedError 64 | 65 | def update( 66 | self, 67 | features: Sequence[Dict], 68 | add_first: bool = True, 69 | quiet: bool = False, 70 | **kwargs, 71 | ): 72 | """Update the mosaicjson document.""" 73 | raise NotImplementedError 74 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | # On every pull request, but only on push to main 4 | on: 5 | push: 6 | branches: 7 | - main 8 | tags: 9 | - '*' 10 | pull_request: 11 | env: 12 | LATEST_PY_VERSION: '3.14' 13 | 14 | jobs: 15 | tests: 16 | runs-on: ubuntu-latest 17 | strategy: 18 | matrix: 19 | python-version: ['3.11', '3.12', '3.13', '3.14'] 20 | 21 | steps: 22 | - uses: actions/checkout@v5 23 | 24 | - name: Install uv 25 | uses: astral-sh/setup-uv@v7 26 | with: 27 | version: "0.9.*" 28 | enable-cache: true 29 | python-version: ${{ matrix.python-version }} 30 | 31 | - name: Install dependencies 32 | run: | 33 | uv sync --all-extras 34 | 35 | - name: Run pre-commit 36 | if: ${{ matrix.python-version == env.LATEST_PY_VERSION }} 37 | run: | 38 | uv run pre-commit run --all-files 39 | 40 | 41 | - name: Run tests 42 | run: uv run pytest --cov cogeo_mosaic --cov-report xml --cov-report term-missing 43 | 44 | - name: Upload Results 45 | if: ${{ matrix.python-version == env.LATEST_PY_VERSION }} 46 | uses: codecov/codecov-action@v1 47 | with: 48 | file: ./coverage.xml 49 | flags: unittests 50 | name: ${{ matrix.python-version }} 51 | fail_ci_if_error: false 52 | token: ${{ secrets.CODECOV_TOKEN }} 53 | 54 | publish: 55 | needs: [tests] 56 | runs-on: ubuntu-latest 57 | if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' 58 | steps: 59 | - uses: actions/checkout@v5 60 | 61 | - name: Install uv 62 | uses: astral-sh/setup-uv@v7 63 | with: 64 | version: "0.9.*" 65 | enable-cache: true 66 | python-version: ${{ env.LATEST_PY_VERSION }} 67 | 68 | - name: Install dependencies 69 | run: | 70 | uv sync --group deploy 71 | 72 | - name: Set tag version 73 | id: tag 74 | run: | 75 | echo "version=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT 76 | 77 | - name: Set module version 78 | id: module 79 | run: | 80 | echo "version=$(uv run hatch --quiet version | tr '\n' ' ')" >> $GITHUB_OUTPUT 81 | 82 | - name: Show version 83 | run: | 84 | echo "${{ steps.tag.outputs.version }}" 85 | echo "${{ steps.module.outputs.version }}" 86 | 87 | - name: Build and publish 88 | if: ${{ steps.tag.outputs.version }} == ${{ steps.module.outputs.version }} 89 | env: 90 | HATCH_INDEX_USER: ${{ secrets.PYPI_USERNAME }} 91 | HATCH_INDEX_AUTH: ${{ secrets.PYPI_PASSWORD }} 92 | run: | 93 | uv run hatch build 94 | uv run hatch publish 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cogeo-mosaic 2 | 3 |

4 | rio-tiler 5 |

6 |

7 | Create mosaics of Cloud Optimized GeoTIFF based on the mosaicJSON specification. 8 |

9 |

10 | 11 | Test 12 | 13 | 14 | Coverage 15 | 16 | 17 | Package version 18 | 19 | 20 | 21 | Downloads 22 | 23 | 24 | Downloads 25 | 26 |

27 | 28 | --- 29 | 30 | **Documentation**: https://developmentseed.org/cogeo-mosaic/ 31 | 32 | **Source Code**: https://github.com/developmentseed/cogeo-mosaic 33 | 34 | --- 35 | 36 | **Read the official announcement https://medium.com/devseed/cog-talk-part-2-mosaics-bbbf474e66df** 37 | 38 | ## Install 39 | ```bash 40 | python -m pip install pip -U 41 | python -m pip install cogeo-mosaic --pre 42 | 43 | # Or from source 44 | 45 | python -m pip install git+http://github.com/developmentseed/cogeo-mosaic 46 | ``` 47 | 48 | **Notes**: 49 | 50 | - Starting with version 5.0, pygeos has been replaced by shapely and thus makes `libgeos` a requirement. 51 | Shapely wheels should be available for most environment, if not, you'll need to have libgeos installed. 52 | 53 | ## See it in action 54 | 55 | - [**TiTiler**](http://github.com/developmentseed/titiler): A lightweight Cloud Optimized GeoTIFF dynamic tile server (COG, STAC and MosaicJSON). 56 | 57 | ## Contribution & Development 58 | 59 | See [CONTRIBUTING.md](https://github.com/developmentseed/cogeo-mosaic/blob/master/CONTRIBUTING.md) 60 | 61 | ## License 62 | 63 | See [LICENSE](https://github.com/developmentseed/cogeo-mosaic/blob/master/LICENSE) 64 | 65 | ## Authors 66 | 67 | Created by [Development Seed]() 68 | 69 | See [contributors](https://github.com/developmentseed/cogeo-mosaic/graphs/contributors) for a listing of individual contributors. 70 | -------------------------------------------------------------------------------- /tests/test_model.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | import morecantile 5 | import pytest 6 | from pydantic import BaseModel, ValidationError 7 | 8 | from cogeo_mosaic.mosaic import MosaicJSON 9 | 10 | basepath = os.path.join(os.path.dirname(__file__), "fixtures") 11 | mosaic_json = os.path.join(basepath, "mosaic.json") 12 | asset1 = os.path.join(basepath, "cog1.tif") 13 | asset2 = os.path.join(basepath, "cog2.tif") 14 | 15 | 16 | def test_model(): 17 | with open(mosaic_json, "r") as f: 18 | mosaic = MosaicJSON.model_validate_json(f.read()) 19 | assert isinstance(mosaic.bounds, tuple) 20 | assert isinstance(mosaic.center, tuple) 21 | assert isinstance(mosaic, BaseModel) 22 | 23 | 24 | def test_validation_error(): 25 | with open(mosaic_json) as f: 26 | data = json.load(f) 27 | data["minzoom"] = -1 28 | with pytest.raises(ValidationError): 29 | MosaicJSON.model_validate(data) 30 | 31 | 32 | def test_compute_center(): 33 | with open(mosaic_json, "r") as f: 34 | data = json.load(f) 35 | del data["center"] 36 | 37 | mosaic = MosaicJSON.model_validate(data) 38 | assert mosaic.center 39 | 40 | 41 | def test_validate_assignment(): 42 | with open(mosaic_json, "r") as f: 43 | mosaic = MosaicJSON.model_validate_json(f.read()) 44 | with pytest.raises(ValidationError): 45 | mosaic.minzoom = -1 46 | 47 | 48 | def test_mosaic_reverse(): 49 | """MosaicJSON dict can be reversed""" 50 | assets = [asset1, asset2] 51 | tms = morecantile.tms.get("WebMercatorQuad") 52 | mosaic = MosaicJSON.from_urls(assets, quiet=True, tilematrixset=tms) 53 | 54 | mosaic_dict = mosaic.model_dump(exclude_none=True) 55 | assert MosaicJSON.model_validate(mosaic_dict).tilematrixset.id == "WebMercatorQuad" 56 | 57 | mosaic_json = mosaic.model_dump_json(exclude_none=True) 58 | assert ( 59 | MosaicJSON.model_validate(json.loads(mosaic_json)).tilematrixset.id 60 | == "WebMercatorQuad" 61 | ) 62 | 63 | 64 | def test_mosaic_model(): 65 | assert MosaicJSON.model_validate( 66 | { 67 | "mosaicjson": "0.0.3", 68 | "minzoom": 0, 69 | "maxzoom": 2, 70 | "tiles": {}, 71 | } 72 | ) 73 | 74 | assert MosaicJSON.model_validate( 75 | { 76 | "mosaicjson": "0.0.3", 77 | "minzoom": 0, 78 | "maxzoom": 2, 79 | "tiles": {}, 80 | "tilematrixset": morecantile.tms.get("WebMercatorQuad"), 81 | } 82 | ) 83 | 84 | assert MosaicJSON.model_validate( 85 | { 86 | "mosaicjson": "0.0.3", 87 | "minzoom": 0, 88 | "maxzoom": 2, 89 | "tiles": {}, 90 | "tilematrixset": morecantile.tms.get("WebMercatorQuad").model_dump( 91 | exclude_none=True 92 | ), 93 | } 94 | ) 95 | 96 | with pytest.raises(ValidationError): 97 | MosaicJSON.model_validate( 98 | { 99 | "mosaicjson": "0.0.3", 100 | "minzoom": 0, 101 | "maxzoom": 2, 102 | "tiles": {}, 103 | "tilematrixset": morecantile.tms.get("WorldCRS84Quad"), 104 | } 105 | ) 106 | -------------------------------------------------------------------------------- /cogeo_mosaic/backends/s3.py: -------------------------------------------------------------------------------- 1 | """cogeo-mosaic AWS S3 backend.""" 2 | 3 | import json 4 | from threading import Lock 5 | from typing import Any 6 | from urllib.parse import urlparse 7 | 8 | import attr 9 | from cachetools import TTLCache, cached 10 | from cachetools.keys import hashkey 11 | 12 | from cogeo_mosaic.backends.base import MosaicJSONBackend 13 | from cogeo_mosaic.backends.utils import _compress_gz_json, _decompress_gz 14 | from cogeo_mosaic.cache import cache_config 15 | from cogeo_mosaic.errors import _HTTP_EXCEPTIONS, MosaicError, MosaicExistsError 16 | from cogeo_mosaic.mosaic import MosaicJSON 17 | 18 | try: 19 | from boto3.session import Session as boto3_session 20 | from botocore.exceptions import ClientError 21 | except ImportError: # pragma: nocover 22 | boto3_session = None # type: ignore 23 | ClientError = None # type: ignore 24 | 25 | 26 | @attr.s 27 | class S3Backend(MosaicJSONBackend): 28 | """S3 Backend Adapter""" 29 | 30 | client: Any = attr.ib(default=None) 31 | bucket: str = attr.ib(init=False) 32 | key: str = attr.ib(init=False) 33 | 34 | _backend_name = "AWS S3" 35 | 36 | def __attrs_post_init__(self): 37 | """Post Init: parse path and create client.""" 38 | assert boto3_session is not None, "'boto3' must be installed to use S3Backend" 39 | 40 | parsed = urlparse(self.input) 41 | self.bucket = parsed.netloc 42 | self.key = parsed.path.strip("/") 43 | self.client = self.client or boto3_session().client("s3") 44 | super().__attrs_post_init__() 45 | 46 | def write(self, overwrite: bool = False, **kwargs: Any): 47 | """Write mosaicjson document to AWS S3.""" 48 | if not overwrite and self._head_object(self.key, self.bucket): 49 | raise MosaicExistsError("Mosaic file already exist, use `overwrite=True`.") 50 | 51 | mosaic_doc = self.mosaic_def.model_dump_json(exclude_none=True) 52 | if self.key.endswith(".gz"): 53 | body = _compress_gz_json(mosaic_doc) 54 | else: 55 | body = mosaic_doc.encode("utf-8") 56 | 57 | self._put_object(self.key, self.bucket, body, **kwargs) 58 | 59 | @cached( # type: ignore 60 | TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl), 61 | key=lambda self: hashkey(self.input), 62 | lock=Lock(), 63 | ) 64 | def _read(self) -> MosaicJSON: # type: ignore 65 | """Get mosaicjson document.""" 66 | body = self._get_object(self.key, self.bucket) 67 | 68 | self._file_byte_size = len(body) 69 | 70 | if self.key.endswith(".gz"): 71 | body = _decompress_gz(body) 72 | 73 | return MosaicJSON(**json.loads(body)) 74 | 75 | def _get_object(self, key: str, bucket: str) -> bytes: 76 | try: 77 | response = self.client.get_object(Bucket=bucket, Key=key) 78 | except ClientError as e: 79 | status_code = e.response["ResponseMetadata"]["HTTPStatusCode"] 80 | exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError) 81 | raise exc(e.response["Error"]["Message"]) from e 82 | 83 | return response["Body"].read() 84 | 85 | def _put_object(self, key: str, bucket: str, body: bytes, **kwargs) -> str: 86 | try: 87 | self.client.put_object(Bucket=bucket, Key=key, Body=body, **kwargs) 88 | except ClientError as e: 89 | status_code = e.response["ResponseMetadata"]["HTTPStatusCode"] 90 | exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError) 91 | raise exc(e.response["Error"]["Message"]) from e 92 | 93 | return key 94 | 95 | def _head_object(self, key: str, bucket: str) -> bool: 96 | try: 97 | return self.client.head_object(Bucket=bucket, Key=key) 98 | except ClientError: 99 | return False 100 | -------------------------------------------------------------------------------- /cogeo_mosaic/backends/gs.py: -------------------------------------------------------------------------------- 1 | """cogeo-mosaic Google Cloud Storage backend.""" 2 | 3 | import json 4 | from threading import Lock 5 | from typing import Any 6 | from urllib.parse import urlparse 7 | 8 | import attr 9 | from cachetools import TTLCache, cached 10 | from cachetools.keys import hashkey 11 | 12 | from cogeo_mosaic.backends.base import MosaicJSONBackend 13 | from cogeo_mosaic.backends.utils import _compress_gz_json, _decompress_gz 14 | from cogeo_mosaic.cache import cache_config 15 | from cogeo_mosaic.errors import _HTTP_EXCEPTIONS, MosaicError, MosaicExistsError 16 | from cogeo_mosaic.mosaic import MosaicJSON 17 | 18 | try: 19 | from google.auth.exceptions import GoogleAuthError 20 | from google.cloud.storage import Client as gcp_session 21 | except ImportError: # pragma: nocover 22 | gcp_session = None # type: ignore 23 | GoogleAuthError = None # type: ignore 24 | 25 | 26 | @attr.s 27 | class GCSBackend(MosaicJSONBackend): 28 | """GCS Backend Adapter""" 29 | 30 | client: Any = attr.ib(default=None) 31 | bucket: str = attr.ib(init=False) 32 | key: str = attr.ib(init=False) 33 | 34 | _backend_name = "Google Cloud Storage" 35 | 36 | def __attrs_post_init__(self): 37 | """Post Init: parse path and create client.""" 38 | assert ( 39 | gcp_session is not None 40 | ), "'google-cloud-storage' must be installed to use GCSBackend" 41 | 42 | parsed = urlparse(self.input) 43 | self.bucket = parsed.netloc 44 | self.key = parsed.path.strip("/") 45 | self.client = self.client or gcp_session() 46 | super().__attrs_post_init__() 47 | 48 | def write(self, overwrite: bool = False, **kwargs: Any): 49 | """Write mosaicjson document to Google Cloud Storage.""" 50 | if not overwrite and self._head_object(self.key, self.bucket): 51 | raise MosaicExistsError("Mosaic file already exist, use `overwrite=True`.") 52 | 53 | mosaic_doc = self.mosaic_def.model_dump_json(exclude_none=True) 54 | if self.key.endswith(".gz"): 55 | body = _compress_gz_json(mosaic_doc) 56 | else: 57 | body = mosaic_doc.encode("utf-8") 58 | 59 | self._put_object(self.key, self.bucket, body, **kwargs) 60 | 61 | @cached( # type: ignore 62 | TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl), 63 | key=lambda self: hashkey(self.input), 64 | lock=Lock(), 65 | ) 66 | def _read(self) -> MosaicJSON: # type: ignore 67 | """Get mosaicjson document.""" 68 | body = self._get_object(self.key, self.bucket) 69 | self._file_byte_size = len(body) 70 | 71 | if self.key.endswith(".gz"): 72 | body = _decompress_gz(body) 73 | 74 | return MosaicJSON(**json.loads(body)) 75 | 76 | def _get_object(self, key: str, bucket: str) -> bytes: 77 | try: 78 | gcs_bucket = self.client.bucket(bucket) 79 | response = gcs_bucket.blob(key).download_as_bytes() 80 | except GoogleAuthError as e: 81 | status_code = e.response["ResponseMetadata"]["HTTPStatusCode"] 82 | exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError) 83 | raise exc(e.response["Error"]["Message"]) from e 84 | 85 | return response 86 | 87 | def _put_object(self, key: str, bucket: str, body: bytes, **kwargs) -> str: 88 | try: 89 | gcs_bucket = self.client.bucket(bucket) 90 | blob = gcs_bucket.blob(key) 91 | blob.upload_from_string(body) 92 | except GoogleAuthError as e: 93 | status_code = e.response["ResponseMetadata"]["HTTPStatusCode"] 94 | exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError) 95 | raise exc(e.response["Error"]["Message"]) from e 96 | 97 | return key 98 | 99 | def _head_object(self, key: str, bucket: str) -> bool: 100 | try: 101 | gcs_bucket = self.client.bucket(bucket) 102 | blob = gcs_bucket.blob(key) 103 | return blob.exists() 104 | except GoogleAuthError: 105 | return False 106 | -------------------------------------------------------------------------------- /docs/src/cli.md: -------------------------------------------------------------------------------- 1 | 2 | # CLI 3 | ``` 4 | $ cogeo-mosaic --help 5 | Usage: cogeo-mosaic [OPTIONS] COMMAND [ARGS]... 6 | 7 | cogeo_mosaic cli. 8 | 9 | Options: 10 | --version Show the version and exit. 11 | --help Show this message and exit. 12 | 13 | Commands: 14 | create Create mosaic definition from list of files 15 | create-from-features Create mosaic definition from GeoJSON features or features collection 16 | footprint Create geojson from list of files 17 | info Return info about the mosaic 18 | to-geojson Create GeoJSON from a MosaicJSON document 19 | update Update a mosaic definition from list of files 20 | upload Upload mosaic definition to backend 21 | ``` 22 | 23 | ## Create 24 | 25 | ```bash 26 | $ cogeo-mosaic create --help 27 | Usage: cogeo-mosaic create [OPTIONS] [INPUT_FILES] 28 | 29 | Create mosaic definition file. 30 | 31 | Options: 32 | -o, --output PATH Output file name 33 | --minzoom INTEGER An integer to overwrite the minimum zoom level derived from the COGs. 34 | --maxzoom INTEGER An integer to overwrite the maximum zoom level derived from the COGs. 35 | --quadkey-zoom INTEGER An integer to overwrite the quadkey zoom level used for keys in the MosaicJSON. 36 | --min-tile-cover FLOAT Minimum % overlap 37 | --tile-cover-sort Sort files by covering % 38 | --threads INTEGER threads 39 | -q, --quiet Remove progressbar and other non-error output. 40 | --help Show this message and exit. 41 | ``` 42 | 43 | **[INPUT_FILES]** must be a list of valid Cloud Optimized GeoTIFF. 44 | 45 | ```bash 46 | $ cogeo-mosaic create list.txt -o mosaic.json 47 | 48 | # or 49 | 50 | $ cat list.txt | cogeo-mosaic create - | gzip > mosaic.json.gz 51 | 52 | # or use backends like AWS S3 or DynamoDB 53 | 54 | $ cogeo-mosaic create list.txt -o s3://my-bucket/my-key.json.gz 55 | ``` 56 | 57 | #### Example: create a mosaic from OAM 58 | 59 | ```bash 60 | # Create Mosaic 61 | $ curl https://api.openaerialmap.org/user/5d6a0d1a2103c90007707fa0 | jq -r '.results.images[] | .uuid' | cogeo-mosaic create - | gzip > 5d6a0d1a2103c90007707fa0.json.gz 62 | 63 | # Create Footprint (optional) 64 | $ curl https://api.openaerialmap.org/user/5d6a0d1a2103c90007707fa0 | jq -r '.results.images[] | .uuid' | cogeo-mosaic footprint | gist -p -f test.geojson 65 | ``` 66 | 67 | ### Create Mosaic definition from a GeoJSON features collection (e.g STAC) 68 | 69 | This module is first design to create mosaicJSON from a set of COG urls but starting in version `3.0.0` we have added a CLI to be able to create mosaicJSON from GeoJSON features. 70 | ``` 71 | $ cogeo-mosaic create-from-features --help 72 | Usage: cogeo-mosaic create-from-features [OPTIONS] FEATURES... 73 | 74 | Create mosaic definition file. 75 | 76 | Options: 77 | -o, --output PATH Output file name 78 | --minzoom INTEGER Mosaic minimum zoom level. [required] 79 | --maxzoom INTEGER Mosaic maximum zoom level. [required] 80 | --property TEXT Define accessor property [required] 81 | --quadkey-zoom INTEGER An integer to overwrite the quadkey zoom level used for keys in the MosaicJSON. 82 | --min-tile-cover FLOAT Minimum % overlap 83 | --tile-cover-sort Sort files by covering % 84 | -q, --quiet Remove progressbar and other non-error output. 85 | --help Show this message and exit. 86 | ``` 87 | 88 | #### Use it with STAC 89 | 90 | ```bash 91 | $ curl https://earth-search.aws.element84.com/collections/landsat-8-l1/items | \ 92 | cogeo-mosaic create-from-features --minzoom 7 --maxzoom 12 --property "landsat:scene_id" --quiet | \ 93 | jq 94 | 95 | { 96 | "mosaicjson": "0.0.2", 97 | "version": "1.0.0", 98 | "minzoom": 7, 99 | "maxzoom": 12, 100 | "quadkey_zoom": 7, 101 | "bounds": [16.142300225571994, -28.513088675819393, 67.21380296165974, 81.2067478836583], 102 | "center": [41.67805159361586, 26.346829603919453, 7], 103 | "tiles": { 104 | "1012123": [ 105 | "LC81930022020114LGN00" 106 | ], 107 | ... 108 | } 109 | } 110 | ``` 111 | -------------------------------------------------------------------------------- /cogeo_mosaic/backends/az.py: -------------------------------------------------------------------------------- 1 | """cogeo-mosaic Azure Blob Storage backend.""" 2 | 3 | import json 4 | from threading import Lock 5 | from typing import Any 6 | from urllib.parse import urlparse 7 | 8 | import attr 9 | from cachetools import TTLCache, cached 10 | from cachetools.keys import hashkey 11 | 12 | from cogeo_mosaic.backends.base import MosaicJSONBackend 13 | from cogeo_mosaic.backends.utils import _compress_gz_json, _decompress_gz 14 | from cogeo_mosaic.cache import cache_config 15 | from cogeo_mosaic.errors import _HTTP_EXCEPTIONS, MosaicError, MosaicExistsError 16 | from cogeo_mosaic.mosaic import MosaicJSON 17 | 18 | try: 19 | from azure.core.exceptions import HttpResponseError 20 | from azure.identity import DefaultAzureCredential 21 | from azure.storage.blob import BlobServiceClient 22 | except ImportError: 23 | HttpResponseError = None 24 | DefaultAzureCredential = None 25 | BlobServiceClient = None 26 | 27 | 28 | @attr.s 29 | class ABSBackend(MosaicJSONBackend): 30 | """Azure Blob Storage Backend Adapter""" 31 | 32 | client: Any = attr.ib(default=None) 33 | account_url: str = attr.ib(init=False) 34 | container: str = attr.ib(init=False) 35 | key: str = attr.ib(init=False) 36 | 37 | _backend_name = "Azure Blob Storage" 38 | 39 | def __attrs_post_init__(self): 40 | """Post Init: parse path and create client.""" 41 | assert ( 42 | HttpResponseError is not None 43 | ), "'azure-identity' and 'azure-storage-blob' must be installed to use ABSBackend" 44 | 45 | az_credentials = DefaultAzureCredential() 46 | 47 | parsed = urlparse(self.input) 48 | self.account_url = "https://%s" % parsed.netloc 49 | self.container = parsed.path.split("/")[1] 50 | self.key = parsed.path.strip("/%s" % self.container) 51 | self.client = self.client or BlobServiceClient( 52 | account_url=self.account_url, credential=az_credentials 53 | ) 54 | super().__attrs_post_init__() 55 | 56 | def write(self, overwrite: bool = False, **kwargs: Any): 57 | """Write mosaicjson document to Azure Blob Storage.""" 58 | if not overwrite and self._head_object(self.key, self.container): 59 | raise MosaicExistsError("Mosaic file already exist, use `overwrite=True`.") 60 | 61 | mosaic_doc = self.mosaic_def.model_dump_json(exclude_none=True) 62 | if self.key.endswith(".gz"): 63 | body = _compress_gz_json(mosaic_doc) 64 | else: 65 | body = mosaic_doc.encode("utf-8") 66 | 67 | self._put_object(self.key, self.container, body, **kwargs) 68 | 69 | @cached( # type: ignore 70 | TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl), 71 | key=lambda self: hashkey(self.input), 72 | lock=Lock(), 73 | ) 74 | def _read(self) -> MosaicJSON: # type: ignore 75 | """Get mosaicjson document.""" 76 | body = self._get_object(self.key, self.container) 77 | self._file_byte_size = len(body) 78 | 79 | if self.key.endswith(".gz"): 80 | body = _decompress_gz(body) 81 | 82 | return MosaicJSON(**json.loads(body)) 83 | 84 | def _get_object(self, key: str, container: str) -> bytes: 85 | try: 86 | container_client = self.client.get_container_client(container) 87 | blob_client = container_client.get_blob_client(key) 88 | response = blob_client.download_blob().readall() 89 | except HttpResponseError as e: 90 | exc = _HTTP_EXCEPTIONS.get(e.status_code, MosaicError) 91 | raise exc(e.reason) from e 92 | 93 | return response 94 | 95 | def _put_object(self, key: str, container: str, body: bytes, **kwargs) -> str: 96 | try: 97 | container_client = self.client.get_container_client(container) 98 | blob_client = container_client.get_blob_client(key) 99 | blob_client.upload_blob(body) 100 | except HttpResponseError as e: 101 | exc = _HTTP_EXCEPTIONS.get(e.status_code, MosaicError) 102 | raise exc(e.reason) from e 103 | 104 | return key 105 | 106 | def _head_object(self, key: str, container: str) -> bool: 107 | try: 108 | container_client = self.client.get_container_client(container) 109 | blob_client = container_client.get_blob_client(key) 110 | return blob_client.exists() 111 | except HttpResponseError: 112 | return False 113 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "cogeo-mosaic" 3 | description = 'CLI and Backends to work with MosaicJSON.' 4 | requires-python = ">=3.11" 5 | license = {file = "LICENSE"} 6 | authors = [ 7 | { name = "Vincent Sarago", email = "vincent@developmentseed.com" }, 8 | ] 9 | keywords = ["COG", "MosaicJSON"] 10 | classifiers = [ 11 | "Intended Audience :: Information Technology", 12 | "Intended Audience :: Science/Research", 13 | "License :: OSI Approved :: MIT License", 14 | "Programming Language :: Python :: 3", 15 | "Programming Language :: Python :: 3 :: Only", 16 | "Programming Language :: Python :: 3.11", 17 | "Programming Language :: Python :: 3.12", 18 | "Programming Language :: Python :: 3.13", 19 | "Programming Language :: Python :: 3.14", 20 | "Topic :: Scientific/Engineering :: GIS", 21 | ] 22 | dynamic = ["version", "readme"] 23 | dependencies = [ 24 | "attrs", 25 | "morecantile>=7.0", 26 | "rio-tiler>=8.0,<9.0", 27 | "shapely>=2.0,<3.0", 28 | "pydantic~=2.0", 29 | "pydantic-settings~=2.0", 30 | "httpx", 31 | "rasterio", 32 | "supermorecado", 33 | "cachetools", 34 | "numpy", 35 | "click", 36 | "cligj", 37 | "click_plugins", 38 | ] 39 | 40 | [project.optional-dependencies] 41 | aws = [ 42 | "boto3", 43 | ] 44 | az = [ 45 | "azure-identity", 46 | "azure-storage-blob", 47 | ] 48 | gcp = [ 49 | "google-cloud-storage" 50 | ] 51 | 52 | [dependency-groups] 53 | dev = [ 54 | "pytest", 55 | "pytest-cov", 56 | "boto3", 57 | "pre-commit", 58 | "bump-my-version>=1.2.4", 59 | ] 60 | docs = [ 61 | "mkdocs", 62 | "mkdocs-material", 63 | "pygments", 64 | "mkdocs-jupyter", 65 | ] 66 | deploy = [ 67 | "hatch", 68 | ] 69 | 70 | [project.urls] 71 | Homepage = 'https://github.com/developmentseed/cogeo-mosaic' 72 | Documentation = "https://developmentseed.org/cogeo-mosaic/" 73 | Issues = "https://github.com/developmentseed/cogeo-mosaic/issues" 74 | Source = "https://github.com/developmentseed/cogeo-mosaic" 75 | Changelog = "https://developmentseed.org/cogeo-mosaic/release-notes/" 76 | 77 | [project.scripts] 78 | cogeo-mosaic = "cogeo_mosaic.scripts.cli:cogeo_cli" 79 | 80 | [tool.hatch.metadata.hooks.fancy-pypi-readme] 81 | content-type = 'text/markdown' 82 | # construct the PyPI readme from README.md and HISTORY.md 83 | fragments = [ 84 | {path = "README.md"}, 85 | {text = "\n## Changelog\n\n"}, 86 | {path = "CHANGES.md"}, 87 | ] 88 | # convert GitHUB issue/PR numbers and handles to links 89 | substitutions = [ 90 | {pattern = '(\s+)#(\d+)', replacement = '\1[#\2](https://github.com/developmentseed/cogeo-mosaic/issues/\2)'}, 91 | {pattern = '(\s+)@([\w\-]+)', replacement = '\1[@\2](https://github.com/\2)'}, 92 | {pattern = '@@', replacement = '@'}, 93 | ] 94 | 95 | [build-system] 96 | requires = ["hatchling", "hatch-fancy-pypi-readme>=22.5.0"] 97 | build-backend = "hatchling.build" 98 | 99 | [tool.hatch.version] 100 | path = "cogeo_mosaic/__init__.py" 101 | 102 | [tool.hatch.build.targets.sdist] 103 | only-include = ["cogeo_mosaic/"] 104 | 105 | [tool.hatch.build.targets.wheel] 106 | only-include = ["cogeo_mosaic/"] 107 | 108 | [tool.coverage.run] 109 | branch = true 110 | parallel = true 111 | 112 | [tool.coverage.report] 113 | exclude_lines = [ 114 | "no cov", 115 | "if __name__ == .__main__.:", 116 | "if TYPE_CHECKING:", 117 | ] 118 | 119 | [tool.isort] 120 | profile = "black" 121 | known_first_party = ["cogeo_mosaic"] 122 | known_third_party = [ 123 | "rasterio", 124 | "rio_tiler", 125 | "morecantile", 126 | "supermorecado", 127 | "shapely", 128 | ] 129 | default_section = "THIRDPARTY" 130 | 131 | [tool.mypy] 132 | no_strict_optional = "True" 133 | 134 | [tool.ruff] 135 | line-length = 90 136 | 137 | [tool.ruff.lint] 138 | select = [ 139 | "D1", # pydocstyle errors 140 | "E", # pycodestyle errors 141 | "W", # pycodestyle warnings 142 | "F", # flake8 143 | "C", # flake8-comprehensions 144 | "B", # flake8-bugbear 145 | ] 146 | ignore = [ 147 | "E501", # line too long, handled by black 148 | "B008", # do not perform function calls in argument defaults 149 | "B905", # ignore zip() without an explicit strict= parameter, only support with python >3.10 150 | "B028", # ignore No explicit stacklevel keyword argument found 151 | ] 152 | 153 | [tool.ruff.lint.mccabe] 154 | max-complexity = 14 155 | 156 | [tool.ruff.lint.extend-per-file-ignores] 157 | "tests/*.py" = ["D1"] 158 | 159 | [tool.bumpversion] 160 | current_version = "9.0.2" 161 | search = "{current_version}" 162 | replace = "{new_version}" 163 | regex = false 164 | tag = true 165 | commit = true 166 | tag_name = "{new_version}" 167 | 168 | [[tool.bumpversion.files]] 169 | filename = "cogeo_mosaic/__init__.py" 170 | search = '__version__ = "{current_version}"' 171 | replace = '__version__ = "{new_version}"' 172 | -------------------------------------------------------------------------------- /docs/src/v3_migration.md: -------------------------------------------------------------------------------- 1 | # cogeo-mosaic 2.0 to 3.0 migration guide 2 | 3 | ### MosaicJSON pydantic model 4 | 5 | We now use [pydantic](https://pydantic-docs.helpmanual.io) to define the MosaicJSON document. Pydantic 6 | 7 | From Pydantic docs: 8 | > Define how data should be in pure, canonical python; validate it with pydantic. 9 | 10 | Pydantic model enforce the mosaicjson specification for the whole project by validating each items. 11 | 12 | ```python 13 | from pydantic import BaseModel 14 | 15 | class MosaicJSON(BaseModel): 16 | """ 17 | MosaicJSON model. 18 | 19 | Based on https://github.com/developmentseed/mosaicjson-spec 20 | 21 | """ 22 | 23 | mosaicjson: str 24 | name: Optional[str] 25 | description: Optional[str] 26 | version: str = "1.0.0" 27 | attribution: Optional[str] 28 | minzoom: int = Field(0, ge=0, le=30) 29 | maxzoom: int = Field(30, ge=0, le=30) 30 | quadkey_zoom: Optional[int] 31 | bounds: List[float] = Field([-180, -90, 180, 90]) 32 | center: Optional[Tuple[float, float, int]] 33 | tiles: Dict[str, List[str]] 34 | ``` 35 | 36 | ##### Validation 37 | 38 | ```python 39 | mosaic_definition = dict( 40 | mosaicjson="0.0.2", 41 | minzoom=1, 42 | maxzoom=2, 43 | quadkey_zoom=1, 44 | bounds=[-180, -90, 180, 90], 45 | center=(0, 0, 1), 46 | tiles={}, 47 | ) 48 | 49 | m = MosaicJSON(**mosaic_definition) 50 | > MosaicJSON(mosaicjson='0.0.2', name=None, description=None, version='1.0.0', attribution=None, minzoom=1, maxzoom=2, quadkey_zoom=1, bounds=[-180.0, -90.0, 180.0, 90.0], center=(0.0, 0.0, 1), tiles={}) 51 | ``` 52 | 53 | ```python 54 | # convert the mode to a dict 55 | m.dict(exclude_none=True) 56 | > {'mosaicjson': '0.0.2', 57 | 'version': '1.0.0', 58 | 'minzoom': 1, 59 | 'maxzoom': 2, 60 | 'quadkey_zoom': 1, 61 | 'bounds': [-180.0, -90.0, 180.0, 90.0], 62 | 'center': (0.0, 0.0, 1), 63 | 'tiles': {}} 64 | ``` 65 | 66 | ```python 67 | mosaic_definition = dict( 68 | mosaicjson="0.0.2", 69 | minzoom=1, 70 | maxzoom=100, 71 | quadkey_zoom=1, 72 | bounds=[-180, -90, 180, 90], 73 | center=(0, 0, 1), 74 | tiles={}, 75 | ) 76 | 77 | m = MosaicJSON(**mosaic_definition) 78 | ... 79 | ValidationError: 1 validation error for MosaicJSON 80 | maxzoom 81 | ensure this value is less than or equal to 30 (type=value_error.number.not_le; limit_value=30) 82 | ``` 83 | 84 | ### Creation 85 | 86 | The `MosaicJSON` class comes also with helper functions: 87 | - **MosaicJSON.from_urls**: Create a mosaicjson from a set of COG urls 88 | - **MosaicJSON.from_features**: Create a mosaicjson from a set of GeoJSON features 89 | - **MosaicJSON._create_mosaic** (semi-private): Low level mosaic creation methods used by public methods (`from_urls` and `from_features`). 90 | 91 | ```python 92 | #V2 93 | from cogeo_mosaic.utils import create_mosaic 94 | 95 | mosaic_definition: Dict = create_mosaic(dataset) 96 | 97 | 98 | #V3 99 | from cogeo_mosaic.mosaic import MosaicJSON 100 | 101 | mosaic_definition: MosaicJSON = MosaicJSON.from_urls(dataset) 102 | 103 | # or from a list of GeoJSON Features 104 | mosaic_definition: MosaicJSON = MosaicJSON.from_features(dataset, minzoom=1, maxzoom=3) 105 | ``` 106 | 107 | 108 | To learn more about the low-level api checkout [/docs/AdvancedTopics.md](/docs/AdvancedTopics.md) 109 | 110 | ### Backend Storage 111 | 112 | #### Read 113 | ```python 114 | # V2 115 | from cogeo_mosaic.utils import ( 116 | fetch_mosaic_definition, 117 | fetch_and_find_assets, 118 | fetch_and_find_assets_point, 119 | ) 120 | mosaic_definition = fetch_mosaic_definition(url) 121 | assets = fetch_and_find_assets(url, x, y, z) 122 | assets = fetch_and_find_assets_point(url, lng, lat) 123 | 124 | 125 | # V3 126 | from cogeo_mosaic.backends import MosaicBackend 127 | 128 | with MosaicBackend(url) as mosaic: 129 | mosaic_definition = mosaic.mosaic_def 130 | assets = mosaic.tile(x, y, z) # LRU cache 131 | assets = mosaic.point(lng, lat) # LRU cache 132 | ``` 133 | 134 | #### Write 135 | ```python 136 | 137 | #V2 138 | from cogeo_mosaic.utils import create_mosaic 139 | from boto3.session import Session as boto3_session 140 | 141 | mosaic_definition = create_mosaic(dataset) 142 | 143 | def _compress_gz_json(data): 144 | gzip_compress = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16) 145 | 146 | return ( 147 | gzip_compress.compress(json.dumps(data).encode("utf-8")) + gzip_compress.flush() 148 | ) 149 | 150 | session = boto3_session() 151 | client = session.client("s3") 152 | client.put_object( 153 | Bucket=bucket, 154 | Key=key, 155 | Body=_compress_gz_json(mosaic_definition), 156 | ) 157 | 158 | #V3 159 | from cogeo_mosaic.mosaic import MosaicJSON 160 | 161 | mosaic_definition = MosaicJSON.from_urls(dataset) 162 | 163 | with MosaicBackend("s3://{bucket}/{key}", mosaic_def=mosaic_definition) as mosaic: 164 | mosaic.write() 165 | ``` 166 | -------------------------------------------------------------------------------- /cogeo_mosaic/utils.py: -------------------------------------------------------------------------------- 1 | """cogeo_mosaic.utils: utility functions.""" 2 | 3 | import logging 4 | import os 5 | import sys 6 | from concurrent import futures 7 | from contextlib import ExitStack 8 | from typing import Dict, List, Optional, Sequence, Tuple 9 | 10 | import click 11 | import morecantile 12 | import numpy 13 | from rio_tiler.io import Reader 14 | from shapely import area, intersection 15 | 16 | logger = logging.getLogger() 17 | logger.setLevel(logging.INFO) 18 | 19 | WEB_MERCATOR_TMS = morecantile.tms.get("WebMercatorQuad") 20 | 21 | 22 | def _filter_futures(tasks): 23 | """ 24 | Filter future task to remove Exceptions. 25 | 26 | Attributes 27 | ---------- 28 | tasks : list 29 | List of 'concurrent.futures._base.Future' 30 | 31 | Yields 32 | ------ 33 | Successful task's result 34 | 35 | """ 36 | for future in tasks: 37 | try: 38 | yield future.result() 39 | except Exception as err: 40 | logger.warning(str(err)) 41 | pass 42 | 43 | 44 | def get_dataset_info( 45 | src_path: str, 46 | tms: morecantile.TileMatrixSet = WEB_MERCATOR_TMS, 47 | ) -> Dict: 48 | """Get rasterio dataset meta.""" 49 | with Reader(src_path, tms=tms) as src: 50 | bounds = src.get_geographic_bounds(tms.rasterio_geographic_crs) 51 | return { 52 | "geometry": { 53 | "type": "Polygon", 54 | "coordinates": [ 55 | [ 56 | (bounds[0], bounds[3]), 57 | (bounds[0], bounds[1]), 58 | (bounds[2], bounds[1]), 59 | (bounds[2], bounds[3]), 60 | (bounds[0], bounds[3]), 61 | ] 62 | ], 63 | }, 64 | "properties": { 65 | "path": src_path, 66 | "bounds": bounds, 67 | "minzoom": src.minzoom, 68 | "maxzoom": src.maxzoom, 69 | "datatype": src.dataset.meta["dtype"], 70 | }, 71 | "type": "Feature", 72 | } 73 | 74 | 75 | def get_footprints( 76 | dataset_list: Sequence[str], 77 | tms: Optional[morecantile.TileMatrixSet] = None, 78 | max_threads: int = 20, 79 | quiet: bool = True, 80 | ) -> List: 81 | """ 82 | Create footprint GeoJSON. 83 | 84 | Attributes 85 | ---------- 86 | dataset_listurl : tuple or list, required 87 | Dataset urls. 88 | tms : TileMatrixSet 89 | TileMartixSet to use (default WebMercatorQaud 90 | max_threads : int 91 | Max threads to use (default: 20). 92 | 93 | Returns 94 | ------- 95 | out : tuple 96 | tuple of footprint feature. 97 | 98 | """ 99 | tms = tms or WEB_MERCATOR_TMS 100 | 101 | with ExitStack() as ctx: 102 | fout = ctx.enter_context(open(os.devnull, "w")) if quiet else sys.stderr 103 | with futures.ThreadPoolExecutor(max_workers=max_threads) as executor: 104 | future_work = [ 105 | executor.submit(get_dataset_info, item, tms) for item in dataset_list 106 | ] 107 | with click.progressbar( # type: ignore 108 | futures.as_completed(future_work), 109 | file=fout, 110 | length=len(future_work), 111 | label="Get footprints", 112 | show_percent=True, 113 | ) as future: 114 | for _ in future: 115 | pass 116 | 117 | return list(_filter_futures(future_work)) 118 | 119 | 120 | def tiles_to_bounds( 121 | tiles: List[morecantile.Tile], 122 | tms: morecantile.TileMatrixSet = WEB_MERCATOR_TMS, 123 | ) -> Tuple[float, float, float, float]: 124 | """Get bounds from a set of mercator tiles.""" 125 | zoom = tiles[0].z 126 | xyz = numpy.array([[t.x, t.y, t.z] for t in tiles]) 127 | extrema = { 128 | "x": {"min": xyz[:, 0].min(), "max": xyz[:, 0].max() + 1}, 129 | "y": {"min": xyz[:, 1].min(), "max": xyz[:, 1].max() + 1}, 130 | } 131 | 132 | ulx, uly = tms.ul(extrema["x"]["min"], extrema["y"]["min"], zoom) 133 | lrx, lry = tms.ul(extrema["x"]["max"], extrema["y"]["max"], zoom) 134 | 135 | return (ulx, lry, lrx, uly) 136 | 137 | 138 | def _intersect_percent(tile, dataset_geoms): 139 | """Return the overlap percent.""" 140 | inter_areas = area(intersection(tile, dataset_geoms)) 141 | return [inter_area / area(tile) for inter_area in inter_areas] 142 | 143 | 144 | def bbox_union( 145 | bbox_1: Tuple[float, float, float, float], 146 | bbox_2: Tuple[float, float, float, float], 147 | ) -> Tuple[float, float, float, float]: 148 | """Return the union of two bounding boxes.""" 149 | return ( 150 | min(bbox_1[0], bbox_2[0]), 151 | min(bbox_1[1], bbox_2[1]), 152 | max(bbox_1[2], bbox_2[2]), 153 | max(bbox_1[3], bbox_2[3]), 154 | ) 155 | -------------------------------------------------------------------------------- /docs/src/advanced/dynamic.md: -------------------------------------------------------------------------------- 1 | 2 | **Deprecated**: With rio-tiler 8.0, creating dynamtic mosaic backend doesn't need to be done with the `MosaicJSONBaseBackend`. See https://cogeotiff.github.io/rio-tiler/advanced/mosaic_backend/. 3 | 4 | The mosaic backend abstract `MosaicJSONBaseBackend` has been designed to be really flexible and compatible with dynamic tiler built for rio-tiler `BaseReader`. It also enables the creation of `dynamic` mosaic where NO mosaicJSON document really exists. 5 | 6 | The `MosaicJSONBaseBackend` ABC class defines that the sub class should: 7 | 8 | - have a `mosaic_def` object (MosaicJSON) or a path as input 9 | - have `_read`, `write` and `update` methods defined 10 | 11 | Other attributes can default to the BaseBackend defaults: 12 | 13 | - `tms` is set to WebMercator 14 | - `minzoom` is set to `0` (mosaicJSON default) 15 | - `maxzoom` is set to `30` (mosaicJSON default) 16 | - `bounds` is set to `(-180, -90, 180, 90)` (mosaicJSON default) 17 | 18 | all other methods are built on top of the MosaicJSON definition. 19 | 20 | For a `dynamic` backend we do not want to construct nor store a mosaicJSON object but fetch the `assets` needed 21 | on each `tile()` or `point()` request. 22 | 23 | For this to be possible we need to : 24 | 25 | - create a `fake` empty mosaicJSON 26 | - create passthrough `_read`, `write` and `update` methods 27 | - create custom `get_assets()`, `assets_for_tile()` and `assets_for_point()` methods. 28 | 29 | Here is an example of a `Dynamic` STAC backend where on each `tile()` or `point()` call, the backend will send a request to the STAC api endpoint to find the assets interesecting with the request. 30 | 31 | ```python 32 | from typing import Dict, Tuple, Type, Optional, List 33 | 34 | import attr 35 | import morecantile 36 | from rasterio.crs import CRS 37 | from rio_tiler.constants import WEB_MERCATOR_TMS, WGS84_CRS 38 | from rio_tiler.io import BaseReader 39 | from rio_tiler.io import STACReader 40 | 41 | from cogeo_mosaic.backends.base import MosaicJSONBaseBackend 42 | from cogeo_mosaic.backends.stac import _fetch, default_stac_accessor 43 | from cogeo_mosaic.mosaic import MosaicJSON 44 | 45 | 46 | @attr.s 47 | class DynamicStacBackend(MosaicJSONBaseBackend): 48 | """Like a STAC backend but dynamic""" 49 | 50 | # input should be the STAC-API url 51 | input: str = attr.ib() 52 | 53 | # Addition required attribute (STAC Query) 54 | query: Dict = attr.ib(factory=dict) 55 | 56 | tms: morecantile.TileMatrixSet = attr.ib(default=WEB_MERCATOR_TMS) 57 | 58 | minzoom: int = attr.ib(default=None) 59 | maxzoom: int = attr.ib(default=None) 60 | 61 | reader: Type[BaseReader] = attr.ib(default=STACReader) 62 | reader_options: Dict = attr.ib(factory=dict) 63 | 64 | bounds: Tuple[float, float, float, float] = attr.ib( 65 | default=(-180, -90, 180, 90) 66 | ) 67 | crs: CRS = attr.ib(default=WGS84_CRS) 68 | 69 | # STAC API related options 70 | # max_items | next_link_key | limit 71 | stac_api_options: Dict = attr.ib(factory=dict) 72 | 73 | # The reader is read-only, we can't pass mosaic_def to the init method 74 | mosaic_def: MosaicJSON = attr.ib(init=False) 75 | 76 | _backend_name = "DynamicSTAC" 77 | 78 | def __attrs_post_init__(self): 79 | """Post Init.""" 80 | self.minzoom = self.minzoom if self.minzoom is not None else self.tms.minzoom 81 | self.maxzoom = self.maxzoom if self.maxzoom is not None else self.tms.maxzoom 82 | 83 | # Construct a FAKE/Empty mosaicJSON 84 | # mosaic_def has to be defined. As we do for the DynamoDB and SQLite backend 85 | self.mosaic_def = MosaicJSON( 86 | mosaicjson="0.0.3", 87 | name="it's fake but it's ok", 88 | bounds=self.bounds, 89 | minzoom=self.minzoom, 90 | maxzoom=self.maxzoom, 91 | tiles={} # we set `tiles` to an empty list. 92 | ) 93 | 94 | def write(self, overwrite: bool = True): 95 | """This method is not used but is required by the abstract class.""" 96 | raise NotImplementedError 97 | 98 | def update(self): 99 | """We overwrite the default method.""" 100 | raise NotImplementedError 101 | 102 | def _read(self) -> MosaicJSON: 103 | """This method is not used but is required by the abstract class.""" 104 | pass 105 | 106 | def assets_for_tile(self, x: int, y: int, z: int) -> List[str]: 107 | """Retrieve assets for tile.""" 108 | bounds = self.tms.bounds(x, y, z) 109 | geom = { 110 | "type": "Polygon", 111 | "coordinates": [ 112 | [ 113 | [bounds[0], bounds[3]], 114 | [bounds[0], bounds[1]], 115 | [bounds[2], bounds[1]], 116 | [bounds[2], bounds[3]], 117 | [bounds[0], bounds[3]], 118 | ] 119 | ], 120 | } 121 | return self.get_assets(geom) 122 | 123 | def assets_for_point(self, lng: float, lat: float) -> List[str]: 124 | """Retrieve assets for point. 125 | 126 | Note: some API only accept Polygon. 127 | """ 128 | EPSILON = 1e-14 129 | geom = { 130 | "type": "Polygon", 131 | "coordinates": [ 132 | [ 133 | [lng - EPSILON, lat + EPSILON], 134 | [lng - EPSILON, lat - EPSILON], 135 | [lng + EPSILON, lat - EPSILON], 136 | [lng + EPSILON, lat + EPSILON], 137 | [lng - EPSILON, lat + EPSILON], 138 | ] 139 | ], 140 | } 141 | return self.get_assets(geom) 142 | 143 | def get_assets(self, geom) -> List[str]: 144 | """Send query to the STAC-API and retrieve assets.""" 145 | query = self.query.copy() 146 | query["intersects"] = geom 147 | 148 | features = _fetch( 149 | self.input, 150 | query, 151 | **self.stac_api_options, 152 | ) 153 | return [default_stac_accessor(f) for f in features] 154 | 155 | @property 156 | def _quadkeys(self) -> List[str]: 157 | return [] 158 | ``` 159 | 160 | Full examples can be found at [examples/Create_a_Dynamic_StacBackend/](https://developmentseed.org/cogeo-mosaic/examples/Create_a_Dynamic_StacBackend/) and [examples/Create_a_Dynamic_RtreeBackend/](https://developmentseed.org/cogeo-mosaic/examples/Create_a_Dynamic_RtreeBackend/). 161 | -------------------------------------------------------------------------------- /docs/src/advanced/custom.md: -------------------------------------------------------------------------------- 1 | ## Custom mosaic creation 2 | 3 | `MosaicJSON._create_mosaic()` method is the low level method that creates mosaicjson document. It has multiple required arguments and options with default values which more advanced users would change. 4 | 5 | 6 | ```python 7 | # cogeo_mosaic.mosaic.MosaicJSON._create_mosaic 8 | def _create_mosaic( 9 | cls, 10 | features: Sequence[Dict], 11 | minzoom: int, 12 | maxzoom: int, 13 | quadkey_zoom: Optional[int] = None, 14 | accessor: Callable[[Dict], str] = default_accessor, 15 | asset_filter: Callable = default_filter, 16 | version: str = "0.0.3", 17 | tilematrixset: Optional[morecantile.TileMatrixSet] = None, 18 | asset_type: Optional[str] = None, 19 | asset_prefix: Optional[str] = None, 20 | data_type: Optional[str] = None, 21 | colormap: Optional[Dict[int, Tuple[int, int, int, int]]] = None, 22 | layers: Optional[Dict] = None, 23 | quiet: bool = True, 24 | **kwargs, 25 | ): 26 | ``` 27 | 28 | #### Custom Accessor 29 | 30 | MosaicJSON `create` method takes a list of GeoJSON features has input, those can be the output of [cogeo_mosaic.utils.get_footprints](https://github.com/developmentseed/cogeo-mosaic/blob/9e8cfd0d65706faaac3e3d785974f890f3b6b180/cogeo_mosaic/utils.py#L80-L111) or can be provided by the user (e.g STAC items). MosaicJSON defines it's tile assets as a `MUST be arrays of strings (url or sceneid) pointing to a COG`. To access those values, `_create_mosaic` needs to know which **property** to read from the GeoJSON feature. 31 | 32 | The **accessor** option is here to enable user to pass their own accessor model. By default, `_create_mosaic` expect features from `get_footprints` and thus COG path stored in `feature["properties"]["path"]`. 33 | 34 | Example: 35 | 36 | ```python 37 | from cogeo_mosaic.mosaic import MosaicJSON 38 | 39 | features = [{"url": "1.tif", "geometry": {...}}, {"url": "2.tif", "geometry": {...}}] 40 | minzoom = 1 41 | maxzoom = 6 42 | 43 | custom_id = lambda feature: feature["url"] 44 | 45 | # 'from_features' will pass all args and kwargs to '_create_mosaic' 46 | mosaicjson = MosaicJSON.from_features( 47 | features, 48 | minzoom, 49 | maxzoom, 50 | accessor=custom_id, 51 | ) 52 | ``` 53 | 54 | #### Custom asset filtering 55 | 56 | On **mosaicjson** creation ones would want to perform more advanced assets filtering or sorting. To enable this, users can define their own `filter` method and pass it using the `asset_filter` options. 57 | 58 | **!!!** In the current implementation, `asset_filter` method **have to** allow at least 3 arguments: 59 | - **tile** - morecantile.Tile: Morecantile tile 60 | - **dataset** - Sequence[Dict]: GeoJSON Feature list intersecting with the `tile` 61 | - **geoms** - Sequence[polygons]: Geos Polygon list for the features 62 | 63 | Example: 64 | 65 | ```python 66 | import datetime 67 | from cogeo_mosaic.mosaic import MosaicJSON, default_filter 68 | 69 | features = [{"url": "20190101.tif", "geometry": {...}}, {"url": "20190102.tif", "geometry": {...}}] 70 | minzoom = 1 71 | maxzoom = 6 72 | 73 | def custom_filter(**args, **kwargs): 74 | """Default filter + sort.""" 75 | dataset = default_filter(**args, **kwargs) 76 | return sorted( 77 | dataset, 78 | key=lambda x: datetime.datetime.strptime(x["url"].split(".")[0], "%Y%m%d") 79 | ) 80 | 81 | mosaicjson = MosaicJSON.from_features( 82 | features, 83 | minzoom, 84 | maxzoom, 85 | asset_filter=custom_filter, 86 | ) 87 | ``` 88 | 89 | ## Custom mosaic update 90 | 91 | Update method is **backend specific** because you don't write a mosaicjson document in the same way in AWS S3 and in AWS DynamoDB. 92 | 93 | The **main** method is defined in [cogeo_mosaic.backends.base.BaseBackend](https://github.com/developmentseed/cogeo-mosaic/blob/main/cogeo_mosaic/backends/base.py). 94 | 95 | On update, here is what is happening: 96 | 1. create mosaic with the new dataset 97 | 2. loop through the new `quadkeys` and edit `old` mosaic assets 98 | 3. update bounds, center and version of the updated mosaic 99 | 4. write the mosaic 100 | 101 | ```python 102 | # cogeo_mosaic.backends.base.BaseBackend 103 | def update( 104 | self, 105 | features: Sequence[Dict], 106 | add_first: bool = True, 107 | quiet: bool = False, 108 | **kwargs, 109 | ): 110 | """Update existing MosaicJSON on backend.""" 111 | # Create mosaic with the new features 112 | new_mosaic = self.mosaic_def.from_features( 113 | features, 114 | self.mosaic_def.minzoom, 115 | self.mosaic_def.maxzoom, 116 | tilematrixset=self.mosaic_def.tilematrixset, 117 | quadkey_zoom=self.quadkey_zoom, 118 | quiet=quiet, 119 | **kwargs, 120 | ) 121 | 122 | # Loop through the new `quadkeys` and edit `old` mosaic assets 123 | for quadkey, new_assets in new_mosaic.tiles.items(): 124 | tile = self.tms.quadkey_to_tile(quadkey) 125 | assets = self.tile(*tile) 126 | assets = [*new_assets, *assets] if add_first else [*assets, *new_assets] 127 | 128 | # [PLACEHOLDER] add custom sorting algorithm (e.g based on path name) 129 | self.mosaic_def.tiles[quadkey] = assets 130 | 131 | # Update bounds, center and version of the updated mosaic 132 | bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds) 133 | self.mosaic_def._increase_version() # Increate mosaicjson document version 134 | self.mosaic_def.bounds = bounds 135 | self.mosaic_def.center = ( 136 | (bounds[0] + bounds[2]) / 2, 137 | (bounds[1] + bounds[3]) / 2, 138 | self.mosaic_def.minzoom, 139 | ) 140 | 141 | # Write the mosaic 142 | if self.input: 143 | self.write() 144 | 145 | return 146 | ``` 147 | 148 | Sometime you'll will want to do more advanced filtering/sorting with the newly dataset stack (e.g keep a max number of COG). For this you'll need to create custom backend: 149 | 150 | ```python 151 | from cogeo_mosaic.backends.s3 import S3Backend 152 | import morecantile 153 | 154 | class CustomS3Backend(S3Backend): 155 | 156 | _backend_name = "Custom AWS S3" 157 | 158 | def update( 159 | self, 160 | features: Sequence[Dict], 161 | quiet: bool = False, 162 | max_image: int = 5, 163 | **kwargs, 164 | ): 165 | """Update existing MosaicJSON on backend.""" 166 | new_mosaic = self.mosaic_def.from_features( 167 | features, 168 | self.mosaic_def.minzoom, 169 | self.mosaic_def.maxzoom, 170 | quadkey_zoom=self.quadkey_zoom, 171 | quiet=quiet, 172 | **kwargs, 173 | ) 174 | 175 | mosaic_tms = self.mosaic_def.tilematrixset or morecantile.tms.get("WebMercatorQuad") 176 | for quadkey, new_assets in new_mosaic.tiles.items(): 177 | tile = mosaic_tms.quadkey_to_tile(quadkey) 178 | assets = self.tile(*tile) 179 | assets = [*new_assets, *assets] 180 | 181 | self.mosaic_def.tiles[quadkey] = assets[:maximum_items_per_tile] 182 | 183 | bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds) 184 | self.mosaic_def._increase_version() # Increate mosaicjson document version 185 | self.mosaic_def.bounds = bounds 186 | self.mosaic_def.center = ( 187 | (bounds[0] + bounds[2]) / 2, 188 | (bounds[1] + bounds[3]) / 2, 189 | self.mosaic_def.minzoom, 190 | ) 191 | 192 | self.write() 193 | 194 | return 195 | ``` 196 | -------------------------------------------------------------------------------- /docs/src/intro.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | **cogeo-mosaic** is set of [CLI](/CLI) and API to create, store and read [MosaicJSON](https://github.com/developmentseed/mosaicjson-spec) documents. 4 | 5 | 6 | ### MosaicJSON Model 7 | 8 | cogeo-mosaic uses [Pydantic](https://pydantic-docs.helpmanual.io) model to store and validate mosaicJSON documents. 9 | ```python 10 | class MosaicJSON(BaseModel): 11 | mosaicjson: str 12 | name: Optional[str] 13 | description: Optional[str] 14 | version: str = "1.0.0" 15 | attribution: Optional[str] 16 | minzoom: int = Field(0, ge=0, le=30) 17 | maxzoom: int = Field(30, ge=0, le=30) 18 | quadkey_zoom: Optional[int] 19 | bounds: List[float] = Field([-180, -90, 180, 90]) 20 | center: Optional[Tuple[float, float, int]] 21 | tiles: Dict[str, List[str]] 22 | tilematrixset: Optional[morecantile.TileMatrixSet] 23 | asset_type: Optional[str] 24 | asset_prefix: Optional[str] 25 | data_type: Optional[str] 26 | colormap: Optional[Dict[int, Tuple[int, int, int, int]]] 27 | layers: Optional[Dict] 28 | ``` 29 | 30 | The model is based on the mosaicjson specification: https://github.com/developmentseed/mosaicjson-spec 31 | 32 | Pydantic models are python classes which are extansible. Here is an example of how we can use the MosaicJSON model to create a mosaic from a list of COG urls: 33 | 34 | ```python 35 | from cogeo_mosaic.mosaic import MosaicJSON 36 | 37 | # list of COG 38 | dataset = ["1.tif", "2.tif"] 39 | mosaic_definition = MosaicJSON.from_urls(dataset) 40 | 41 | print(mosaic_definition.tiles) 42 | > {"tile": {"00001": ["cog1.tif", "2.tif"]}} 43 | ``` 44 | 45 | Lear more on MosaicJSON class [API/mosaic](../API/mosaic). 46 | 47 | 48 | ### Backends 49 | 50 | MosaicJSON `backends` are python classes, based on rio-tiler [BaseBackend](https://github.com/cogeotiff/rio-tiler/blob/main/rio_tiler/mosaic/backend.py), which are used to interact with MosaicJSON documents, stored on AWS DynamoDB, AWS S3, locally, or on the web (http://). 51 | 52 | Because each Backends extend rio-tiler [BaseBackend](https://github.com/cogeotiff/rio-tiler/blob/main/rio_tiler/mosaic/backend.py) they share the same minimal methods/properties 53 | 54 | ```python 55 | from cogeo_mosaic.backends import MosaicJSONBackend 56 | print(MosaicJSONBackend.__bases__) 57 | >> (,) 58 | ``` 59 | 60 | ```python 61 | from cogeo_mosaic.backends.s3 import S3Backend 62 | 63 | # Read 64 | with S3Backend("s3://mybucket/amosaic.json") as mosaic: 65 | mosaic.input # attribute - MosaicJSON path 66 | mosaic.mosaic_def # attribute - MosaicJSON document, wrapped in a Pydantic Model 67 | mosaic.reader # attribute - BaseReader, MultiBaseReader, MultiBandReader to use to fetch tile data 68 | mosaic.reader_options # attribute - Options for forward to `reader` 69 | mosaic.tms # attribute - TileMatrixSet (default to WebMercatorQuad) 70 | mosaic.minzoom # attribute - Mosaic (default to tms or mosaic minzoom) 71 | mosaic.maxzoom # attribute - Mosaic (default to tms or mosaic maxzoom) 72 | 73 | mosaic.crs # property - CRS (from mosaic's TMS geographic CRS) 74 | mosaic.bounds # property - Mosaic bounds in `mosaic.crs` 75 | 76 | mosaic.mosaicid # property - Return sha224 id from the mosaicjson doc 77 | mosaic.quadkey_zoom # property - Return Quadkey zoom of the mosaic 78 | 79 | mosaic.write() # method - Write the mosaicjson to the given location 80 | mosaic.update([features]) # method - Update the mosaicjson data with a list of features 81 | 82 | mosaic.info(quadkeys=True/False) # method - spatial_info, list of quadkeys and mosaic name 83 | 84 | mosaic.get_geographic_bounds(crs: CRS) # method - Return mosaic bounds in a geographic CRS 85 | 86 | mosaic.assets_for_tile(x, y, z) # method - Find assets for a specific mercator tile 87 | mosaic.assets_for_point(lng, lat) # method - Find assets for a specific point 88 | mosaic.assets_for_bbox(xmin, ymin, xmax, ymax) # method - Find assets for a specific bbox 89 | 90 | mosaic.tile(1,2,3) # method - Create mosaic tile 91 | mosaic.point(lng, lat) # method - Read point value from multiple assets 92 | mosaic.part(bbox) # method - Create image from part of multiple assets 93 | mosaic.feature(feature) # method - Create image from GeoJSON feature of multiple assets 94 | ``` 95 | 96 | !!! Important 97 | 98 | `statistics()`, `preview()` methods are not implemented in BaseBackend 99 | 100 | #### Open Mosaic and Get assets list for a tile 101 | 102 | ```python 103 | from cogeo_mosaic.backends import MosaicBackend 104 | 105 | with MosaicBackend("s3://mybucket/amosaic.json") as mosaic: 106 | assets: List[str] = mosaic.assets_for_tile(1, 2, 3) # get assets for morecantile.Tile(1, 2, 3) 107 | ``` 108 | 109 | !!! Important 110 | 111 | `MosaicBackend` is a function which returns the correct `backend` by checking the path/url schema. 112 | 113 | see [MosaicBackend](https://developmentseed.org/cogeo-mosaic/advanced/backends/#mosaicbackend) 114 | 115 | #### Open Mosaic and get Tile Data (mosaic tile) 116 | 117 | ```python 118 | from cogeo_mosaic.backends import MosaicBackend 119 | 120 | with MosaicBackend("s3://mybucket/amosaic.json") as mosaic: 121 | img: ImageData, assets_used: List[str] = mosaic.tile(1, 2, 3) 122 | ``` 123 | 124 | #### Write Mosaic 125 | 126 | ```python 127 | from cogeo_mosaic.mosaic import MosaicJSON 128 | from cogeo_mosaic.backends import MosaicBackend 129 | 130 | # Create a MosaicJSON 131 | mosaicdata = MosaicJSON.from_urls(["1.tif", "2.tif"]) 132 | 133 | with MosaicBackend("s3://mybucket/amosaic.json", mosaic_def=mosaicdata) as mosaic: 134 | mosaic.write() # trigger upload to S3 135 | ``` 136 | 137 | #### Update a Mosaic 138 | 139 | ```python 140 | from cogeo_mosaic.utils import get_footprints 141 | from cogeo_mosaic.backends import MosaicBackend 142 | 143 | with MosaicBackend("s3://mybucket/amosaic.json") as mosaic: 144 | features = get_footprints(["3.tif"]) # Get footprint 145 | mosaic.update(features) # Update mosaicJSON and upload to S3 146 | ``` 147 | 148 | ### In Memory Mosaic 149 | 150 | ```python 151 | from cogeo_mosaic.mosaic import MosaicJSON 152 | from cogeo_mosaic.backends import MosaicBackend 153 | 154 | from cogeo_mosaic.backends.memory import MemoryBackend 155 | 156 | mosaic_definition = MosaicJSON.from_urls(["1.tif", "2.tif"]) 157 | 158 | # If set to None or :memory:, MosaicBackend will use the MemoryBackend 159 | with MosaicBackend(":memory:", mosaic_def=mosaicdata) as mosaic: 160 | assert isinstance(mosaic, MemoryBackend) 161 | img, assets_used = mosaic.tile(1, 2, 3) 162 | 163 | with MosaicBackend(None, mosaic_def=mosaicdata) as mosaic: 164 | assert isinstance(mosaic, MemoryBackend) 165 | img, assets_used = mosaic.tile(1, 2, 3) 166 | 167 | with MemoryBackend(mosaic_def=mosaicdata) as mosaic: 168 | img, assets_used = mosaic.tile(1, 2, 3) 169 | ``` 170 | 171 | ### TileMatrixSet attribute 172 | 173 | ```python 174 | from cogeo_mosaic.backends import MosaicBackend 175 | import morecantile 176 | 177 | # Mosaic in WebMercatorQuad (default), output tile in WGS84 178 | WGS1984Quad = morecantile.tms.get("WGS1984Quad") 179 | 180 | with MosaicBackend("s3://mybucket/amosaic.json", tms=WGS1984Quad) as mosaic: 181 | img: ImageData, assets_used: List[str] = mosaic.tile(1, 2, 3) 182 | 183 | # The mosaic might use a specific TMS (WebMercatorQuad by default) 184 | assert mosaic.mosaic_def.tilematrixset.rasterio_crs == "epsg:3857" 185 | 186 | # When passing `tms=`, the output tile image will have the CRS from the input TMS 187 | assert img.crs == "epsg:4326" 188 | ``` 189 | 190 | # Image Order 191 | 192 | **By default the order of the dataset, either passed via the CLI or in the API, defines the order of the quadkey's assets.** 193 | 194 | ```python 195 | from cogeo_mosaic.mosaic import MosaicJSON 196 | 197 | # list of COG 198 | dataset = ["1.tif", "2.tif"] 199 | mosaic_definition = MosaicJSON.from_urls(dataset) 200 | 201 | print(mosaic_definition.tiles) 202 | > {"tile": {"0": ["1.tif", "2.tif"]}} 203 | ``` 204 | -------------------------------------------------------------------------------- /cogeo_mosaic/backends/stac.py: -------------------------------------------------------------------------------- 1 | """cogeo-mosaic STAC backend.""" 2 | 3 | import json 4 | import os 5 | from threading import Lock 6 | from typing import Dict, List, Optional, Sequence, Type 7 | 8 | import attr 9 | import httpx 10 | from cachetools import TTLCache, cached 11 | from cachetools.keys import hashkey 12 | from morecantile import TileMatrixSet 13 | from rasterio.crs import CRS 14 | from rio_tiler.constants import WEB_MERCATOR_TMS, WGS84_CRS 15 | from rio_tiler.io import STACReader 16 | from rio_tiler.types import BBox 17 | 18 | from cogeo_mosaic.backends.base import MosaicJSONBackend 19 | from cogeo_mosaic.cache import cache_config 20 | from cogeo_mosaic.errors import _HTTP_EXCEPTIONS, MosaicError 21 | from cogeo_mosaic.logger import logger 22 | from cogeo_mosaic.mosaic import MosaicJSON 23 | 24 | 25 | def default_stac_accessor(feature: Dict): 26 | """Return feature identifier.""" 27 | link = list(filter(lambda link: link["rel"] == "self", feature["links"])) 28 | if link: 29 | return link[0]["href"] 30 | 31 | link = list(filter(lambda link: link["rel"] == "root", feature["links"])) 32 | if link: 33 | return os.path.join( 34 | link[0]["href"], 35 | "collections", 36 | feature["collection"], 37 | "items", 38 | feature["id"], 39 | ) 40 | 41 | # Fall back to the item ID 42 | return feature["id"] 43 | 44 | 45 | @attr.s 46 | class STACBackend(MosaicJSONBackend): 47 | """STAC Backend Adapter 48 | 49 | Examples: 50 | >>> with STACBackend( 51 | "https://earth-search.aws.element84.com/v0/search", 52 | query, 53 | 8, 54 | 15, 55 | ) as mosaic: 56 | mosaic.tile(0, 0, 0) 57 | 58 | """ 59 | 60 | input: str = attr.ib() 61 | query: Dict = attr.ib() 62 | 63 | minzoom: int = attr.ib() 64 | maxzoom: int = attr.ib() 65 | 66 | tms: TileMatrixSet = attr.ib(default=WEB_MERCATOR_TMS) 67 | 68 | reader: Type[STACReader] = attr.ib(default=STACReader) 69 | reader_options: Dict = attr.ib(factory=dict) 70 | 71 | bounds: BBox = attr.ib(init=False, default=(-180, -90, 180, 90)) 72 | crs: CRS = attr.ib(init=False, default=WGS84_CRS) 73 | 74 | # STAC API related options 75 | # max_items | next_link_key | limit 76 | stac_api_options: Dict = attr.ib(factory=dict) 77 | 78 | # Mosaic Creation options 79 | # e.g `accessor` 80 | mosaic_options: Dict = attr.ib(factory=dict) 81 | 82 | # Because the STACBackend is a Read-Only backend, there is no need for 83 | # mosaic_def to be in the init method. 84 | mosaic_def: MosaicJSON = attr.ib(init=False, default=None) 85 | 86 | _backend_name = "STAC" 87 | 88 | def _read(self) -> MosaicJSON: 89 | """ 90 | Fetch STAC API and construct the mosaicjson. 91 | 92 | Returns: 93 | MosaicJSON: Mosaic definition. 94 | 95 | """ 96 | logger.debug(f"Using STAC backend: {self.input}") 97 | 98 | features = _fetch( 99 | self.input, 100 | self.query, 101 | **self.stac_api_options, 102 | ) 103 | logger.debug(f"Creating mosaic from {len(features)} features") 104 | 105 | # We need a specific accessor for STAC 106 | options = self.mosaic_options.copy() 107 | if "accessor" not in options: 108 | options["accessor"] = default_stac_accessor 109 | 110 | minzoom = options.pop("minzoom", None) 111 | maxzoom = options.pop("maxzoom", None) 112 | mosaic_tms = options.get("tilematrixset", WEB_MERCATOR_TMS) 113 | if mosaic_tms == self.tms: 114 | minzoom, maxzoom = self.minzoom, self.maxzoom 115 | else: 116 | if minzoom is None or maxzoom is None: 117 | raise MosaicError( 118 | "Min/Max zoom HAVE TO be provided through `mosaic_options` when using different TMS for Read and Mosaic creation" 119 | ) 120 | 121 | return MosaicJSON.from_features(features, minzoom, maxzoom, **options) 122 | 123 | def write(self, overwrite: bool = True): 124 | """Write mosaicjson document.""" 125 | raise NotImplementedError 126 | 127 | def update( 128 | self, 129 | features: Sequence[Dict], 130 | add_first: bool = True, 131 | quiet: bool = False, 132 | **kwargs, 133 | ): 134 | """Update the mosaicjson document.""" 135 | raise NotImplementedError 136 | 137 | 138 | def query_from_link(link: Dict, query: Dict): 139 | """Handle Next Link.""" 140 | q = query.copy() 141 | if link["method"] != "POST": 142 | raise MosaicError("Fetch doesn't support GET for next request.") 143 | 144 | if link.get("merge", False): 145 | q.update(link.get("body", {})) 146 | else: 147 | q = link.get("body", {}) 148 | 149 | return q 150 | 151 | 152 | @cached( # type: ignore 153 | TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl), 154 | key=lambda url, query, **kwargs: hashkey(url, json.dumps(query), **kwargs), 155 | lock=Lock(), 156 | ) 157 | def _fetch( # noqa: C901 158 | stac_url: str, 159 | query: Dict, 160 | max_items: Optional[int] = None, 161 | next_link_key: Optional[str] = None, 162 | limit: int = 500, 163 | ) -> List[Dict]: 164 | """Call STAC API.""" 165 | features: List[Dict] = [] 166 | stac_query = query.copy() 167 | 168 | headers = { 169 | "Content-Type": "application/json", 170 | "Accept-Encoding": "gzip", 171 | "Accept": "application/geo+json", 172 | } 173 | 174 | if "limit" not in stac_query: 175 | stac_query.update({"limit": limit}) 176 | 177 | def _stac_search(url: str, q: Dict): 178 | try: 179 | r = httpx.post(url, headers=headers, json=q) 180 | r.raise_for_status() 181 | except httpx.HTTPStatusError as e: 182 | # post-flight errors 183 | status_code = e.response.status_code 184 | exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError) 185 | raise exc(e.response.content) from e 186 | except httpx.RequestError as e: 187 | # pre-flight errors 188 | raise MosaicError(e.args[0].reason) from e 189 | return r.json() 190 | 191 | page = 1 192 | while True: 193 | logger.debug(f"Fetching page {page}") 194 | logger.debug("query: " + json.dumps(stac_query)) 195 | 196 | results = _stac_search(stac_url, stac_query) 197 | if not results.get("features"): 198 | break 199 | 200 | features.extend(results["features"]) 201 | if max_items and len(features) >= max_items: 202 | features = features[:max_items] 203 | break 204 | 205 | # new STAC context spec 206 | # {"page": 1, "limit": 1000, "matched": 5671, "returned": 1000} 207 | # SAT-API META 208 | # {"page": 4, "limit": 100, "found": 350, "returned": 50} 209 | # STAC api `numberMatched` and `numberReturned` fields in ItemCollection 210 | # {"numberMatched": 10, "numberReturned": 5, "features": [...]} 211 | # otherwise we don't break early 212 | ctx = results.get("context", results.get("meta")) 213 | returned = ctx.get("returned", results.get("numberReturned")) 214 | matched = ctx.get("matched", ctx.get("found") or results.get("numberMatched")) 215 | 216 | logger.debug(json.dumps(ctx)) 217 | 218 | # Check if there is more data to fetch 219 | if matched is not None and returned is not None: 220 | if matched <= returned: 221 | break 222 | 223 | # We shouldn't fetch more item than matched 224 | if len(features) == matched: 225 | break 226 | 227 | if len(features) > matched: 228 | raise MosaicError( 229 | "Something weird is going on, please open an issue in https://github.com/developmentseed/cogeo-mosaic" 230 | ) 231 | page += 1 232 | 233 | # https://github.com/radiantearth/stac-api-spec/blob/master/api-spec.md#paging-extension 234 | if next_link_key: 235 | links = list( 236 | filter(lambda link: link["rel"] == next_link_key, results["links"]) 237 | ) 238 | if not links: 239 | break 240 | stac_query = query_from_link(links[0], stac_query) 241 | else: 242 | stac_query.update({"page": page}) 243 | 244 | return features 245 | -------------------------------------------------------------------------------- /tests/test_create.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import warnings 4 | 5 | import morecantile 6 | import pyproj 7 | import pytest 8 | 9 | from cogeo_mosaic.backends import MosaicBackend 10 | from cogeo_mosaic.backends.file import FileBackend 11 | from cogeo_mosaic.errors import MultipleDataTypeError 12 | from cogeo_mosaic.mosaic import MosaicJSON, default_filter 13 | 14 | tms_3857 = morecantile.tms.get("WebMercatorQuad") 15 | tms_4326 = morecantile.tms.get("WorldCRS84Quad") 16 | tms_5041 = morecantile.tms.get("UPSArcticWGS84Quad") 17 | tms_4087 = morecantile.TileMatrixSet.custom( 18 | [-20037508.34, -10018754.17, 20037508.34, 10018754.17], 19 | pyproj.CRS("EPSG:4087"), 20 | id="WGS84WorldEquidistantCylindrical", 21 | ) 22 | 23 | basepath = os.path.join(os.path.dirname(__file__), "fixtures") 24 | mosaic_gz = os.path.join(basepath, "mosaic.json.gz") 25 | mosaic_json = os.path.join(basepath, "mosaic.json") 26 | asset1 = os.path.join(basepath, "cog1.tif") 27 | asset2 = os.path.join(basepath, "cog2.tif") 28 | 29 | asset1_uint32 = os.path.join(basepath, "cog1_uint32.tif") 30 | asset1_small = os.path.join(basepath, "cog1_small.tif") 31 | 32 | mars_ctx_asset = os.path.join(basepath, "mars_ctx_stac_asset.json") 33 | 34 | with open(mosaic_json, "r") as f: 35 | mosaic_content = json.loads(f.read()) 36 | for qk, asset in mosaic_content["tiles"].items(): 37 | mosaic_content["tiles"][qk] = [os.path.join(basepath, item) for item in asset] 38 | 39 | 40 | def _filter_and_sort(*args, **kwargs): 41 | dataset = default_filter(*args, **kwargs) 42 | return sorted(dataset, key=lambda x: x["properties"]["path"], reverse=True) 43 | 44 | 45 | def test_mosaic_create(): 46 | """Fetch info from dataset and create the mosaicJSON definition.""" 47 | assets = [asset1, asset2] 48 | mosaic = MosaicJSON.from_urls(assets, quiet=True) 49 | assert [round(b, 3) for b in list(mosaic.bounds)] == [ 50 | round(b, 3) for b in mosaic_content["bounds"] 51 | ] 52 | assert mosaic.maxzoom == mosaic_content["maxzoom"] 53 | assert mosaic.minzoom == mosaic_content["minzoom"] 54 | assert list(mosaic.tiles.keys()) == list(mosaic_content["tiles"].keys()) 55 | assert mosaic.tiles == mosaic_content["tiles"] 56 | assert not mosaic.tilematrixset 57 | 58 | mosaic = MosaicJSON.from_urls( 59 | assets, tilematrixset=morecantile.tms.get("WebMercatorQuad"), quiet=True 60 | ) 61 | assert [round(b, 3) for b in list(mosaic.bounds)] == [ 62 | round(b, 3) for b in mosaic_content["bounds"] 63 | ] 64 | assert mosaic.maxzoom == mosaic_content["maxzoom"] 65 | assert mosaic.minzoom == mosaic_content["minzoom"] 66 | assert list(mosaic.tiles.keys()) == list(mosaic_content["tiles"].keys()) 67 | assert mosaic.tiles == mosaic_content["tiles"] 68 | assert mosaic.tilematrixset.id == "WebMercatorQuad" 69 | 70 | mosaic = MosaicJSON.from_urls(assets, minzoom=7, maxzoom=9) 71 | assert [round(b, 3) for b in list(mosaic.bounds)] == [ 72 | round(b, 3) for b in mosaic_content["bounds"] 73 | ] 74 | assert mosaic.maxzoom == mosaic_content["maxzoom"] 75 | assert mosaic.minzoom == mosaic_content["minzoom"] 76 | assert list(mosaic.tiles.keys()) == list(mosaic_content["tiles"].keys()) 77 | 78 | mosaic = MosaicJSON.from_urls(assets, minzoom=7, maxzoom=9, tilematrixset=tms_3857) 79 | assert mosaic.tilematrixset.id == "WebMercatorQuad" 80 | 81 | # 5% tile cover filter 82 | mosaic = MosaicJSON.from_urls(assets, minimum_tile_cover=0.059) 83 | assert not list(mosaic.tiles.keys()) == list(mosaic_content["tiles"].keys()) 84 | 85 | # sort by tile cover 86 | mosaic = MosaicJSON.from_urls(assets, tile_cover_sort=True) 87 | assert list(mosaic.tiles.keys()) == list(mosaic_content["tiles"].keys()) 88 | assert not mosaic.tiles == mosaic_content["tiles"] 89 | 90 | assets = [asset1, asset2] 91 | mosaic = MosaicJSON.from_urls(assets) 92 | assert [round(b, 3) for b in list(mosaic.bounds)] == [ 93 | round(b, 3) for b in mosaic_content["bounds"] 94 | ] 95 | assert mosaic.maxzoom == mosaic_content["maxzoom"] 96 | 97 | # Make sure no warning is emmited 98 | with warnings.catch_warnings(): 99 | warnings.simplefilter("error") 100 | MosaicJSON.from_urls([asset1_small, asset2], minzoom=7, maxzoom=9) 101 | 102 | # Multiple MaxZoom 103 | with pytest.warns(UserWarning): 104 | assets = [asset1_small, asset2] 105 | MosaicJSON.from_urls(assets) 106 | 107 | # Mixed datatype 108 | with pytest.raises(MultipleDataTypeError): 109 | assets = [asset1_uint32, asset2] 110 | MosaicJSON.from_urls(assets) 111 | 112 | assets = [asset1, asset2] 113 | mosaic = MosaicJSON.from_urls(assets, asset_filter=_filter_and_sort, quiet=True) 114 | assert not mosaic.tiles == mosaic_content["tiles"] 115 | 116 | 117 | def test_mosaic_create_tms(): 118 | """Create mosaic with TMS""" 119 | assets = [asset1, asset2] 120 | mosaic = MosaicJSON.from_urls(assets, quiet=True, tilematrixset=tms_3857) 121 | assert mosaic.tilematrixset.id == "WebMercatorQuad" 122 | 123 | # using non Quadkey TMS 124 | with pytest.raises(AssertionError): 125 | mosaic = MosaicJSON.from_urls(assets, tilematrixset=tms_4326, quiet=True) 126 | 127 | # Test a Quad (1:1) polar projection 128 | mosaic = MosaicJSON.from_urls(assets, tilematrixset=tms_5041, quiet=True) 129 | assert len(mosaic.tiles) == 6 130 | assert mosaic.tilematrixset.id == "UPSArcticWGS84Quad" 131 | 132 | # Test a Earth Equirectangular projection, currently improperly using quadtree indexing 133 | mosaic = MosaicJSON.from_urls(assets, tilematrixset=tms_4087, quiet=True) 134 | assert len(mosaic.tiles) == 6 135 | assert mosaic.tilematrixset.id == "WGS84WorldEquidistantCylindrical" 136 | 137 | 138 | def test_mosaic_create_additional_metadata(): 139 | """add metadata info to""" 140 | assets = [asset1, asset2] 141 | mosaic = MosaicJSON.from_urls( 142 | assets, 143 | quiet=True, 144 | tilematrixset=tms_3857, 145 | asset_type="COG", 146 | asset_prefix=basepath, 147 | data_type="uint16", 148 | layers={ 149 | "true-color": { 150 | "bidx": [1, 2, 3], 151 | "rescale": [(0, 1000), (0, 1100), (0, 3000)], 152 | } 153 | }, 154 | ) 155 | assert mosaic.asset_type == "COG" 156 | assert mosaic.asset_prefix == basepath 157 | assert mosaic.data_type == "uint16" 158 | assert mosaic.layers["true-color"] 159 | assert mosaic.tiles["0302301"] == ["/cog1.tif", "/cog2.tif"] 160 | 161 | 162 | @pytest.mark.skipif( 163 | tuple(map(int, pyproj.__version__.split("."))) < (3, 6, 0), 164 | reason="requires proj >= 9.2.1", 165 | ) 166 | def test_mars_mosaic_create(): 167 | # first define the tilematrixset manually 168 | MARS2000_SPHERE = pyproj.CRS.from_user_input("IAU_2015:49900") 169 | MARS_MERCATOR = pyproj.CRS.from_user_input("IAU_2015:49990") 170 | # same boundaries as Earth mercator 171 | mars_tms = morecantile.TileMatrixSet.custom( 172 | [ 173 | -179.9999999999996, 174 | -85.05112877980656, 175 | 179.9999999999996, 176 | 85.05112877980656, 177 | ], 178 | MARS_MERCATOR, 179 | extent_crs=MARS2000_SPHERE, 180 | title="Web Mercator Mars", 181 | ) 182 | # load the mars_ctx_stac_assset.json file 183 | with open(mars_ctx_asset, "r") as f: 184 | stac_asset = json.load(f) 185 | # these stac item has multiple assets so grab the DTM 186 | stac_asset["properties"]["path"] = stac_asset["assets"]["dtm"]["href"] 187 | # now construct the mosaicjson object 188 | mosaic_json = MosaicJSON.from_features( 189 | [ 190 | stac_asset, 191 | ], 192 | minzoom=7, 193 | maxzoom=30, 194 | tilematrixset=mars_tms, 195 | ) 196 | assert mosaic_json is not None 197 | assert mosaic_json.tilematrixset == mars_tms 198 | assert len(mosaic_json.tiles) > 0 199 | # now test the mosaic backend 200 | with MosaicBackend("mosaic.json", mosaic_def=mosaic_json, tms=mars_tms) as mosaic: 201 | assert isinstance(mosaic, FileBackend) 202 | assert mosaic.tms == mars_tms 203 | assert mosaic.crs == mars_tms.rasterio_geographic_crs 204 | assert len(mosaic.assets_for_point(77.28, 18, mars_tms.geographic_crs)) > 0 205 | assert ( 206 | len(mosaic.assets_for_bbox(77.2, 17.5, 77.4, 18.5, mars_tms.geographic_crs)) 207 | > 0 208 | ) 209 | assert ( 210 | len(mosaic.assets_for_point(77.28, 18, mars_tms.rasterio_geographic_crs)) > 0 211 | ) 212 | assert ( 213 | len( 214 | mosaic.assets_for_bbox( 215 | 77.2, 17.5, 77.4, 18.5, mars_tms.rasterio_geographic_crs 216 | ) 217 | ) 218 | > 0 219 | ) 220 | assert len(mosaic.assets_for_point(77.25, 18.0)) > 0 221 | assert len(mosaic.assets_for_bbox(77.2, 17.5, 77.4, 18.5)) > 0 222 | -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | """tests cogeo_mosaic.scripts.cli.""" 2 | 3 | import json 4 | import os 5 | 6 | from click.testing import CliRunner 7 | 8 | from cogeo_mosaic.mosaic import MosaicJSON 9 | from cogeo_mosaic.scripts.cli import cogeo_cli 10 | 11 | asset1 = os.path.join(os.path.dirname(__file__), "fixtures", "cog1.tif") 12 | asset2 = os.path.join(os.path.dirname(__file__), "fixtures", "cog2.tif") 13 | assets = [asset1, asset2] 14 | mosaic_content = MosaicJSON.from_urls(assets) 15 | 16 | 17 | def test_create_valid(): 18 | """Should work as expected.""" 19 | runner = CliRunner() 20 | with runner.isolated_filesystem(): 21 | with open("./list.txt", "w") as f: 22 | f.write("\n") 23 | f.write("\n".join(assets)) 24 | f.write("\n") 25 | 26 | result = runner.invoke(cogeo_cli, ["create", "list.txt", "--quiet"]) 27 | assert not result.exception 28 | assert result.exit_code == 0 29 | assert mosaic_content == MosaicJSON(**json.loads(result.output)) 30 | 31 | result = runner.invoke(cogeo_cli, ["create", "list.txt", "-o", "mosaic.json"]) 32 | assert not result.exception 33 | assert result.exit_code == 0 34 | with open("mosaic.json", "r") as f: 35 | assert mosaic_content == MosaicJSON.model_validate_json(f.read()) 36 | 37 | result = runner.invoke( 38 | cogeo_cli, 39 | [ 40 | "create", 41 | "list.txt", 42 | "-o", 43 | "mosaic.json", 44 | "--tms", 45 | "WorldMercatorWGS84Quad", 46 | "--name", 47 | "my_mosaic", 48 | "--description", 49 | "A mosaic", 50 | "--attribution", 51 | "someone", 52 | ], 53 | ) 54 | assert not result.exception 55 | assert result.exit_code == 0 56 | with open("mosaic.json", "r") as f: 57 | mosaic = MosaicJSON.model_validate_json(f.read()) 58 | assert mosaic.name == "my_mosaic" 59 | assert mosaic.description == "A mosaic" 60 | assert mosaic.attribution == "someone" 61 | 62 | 63 | def test_update_valid(): 64 | """Should work as expected.""" 65 | runner = CliRunner() 66 | with runner.isolated_filesystem(): 67 | with open("mosaic_1.json", "w") as f: 68 | f.write(MosaicJSON.from_urls([asset1]).model_dump_json(exclude_none=True)) 69 | 70 | with open("./list.txt", "w") as f: 71 | f.write("\n".join([asset2])) 72 | 73 | result = runner.invoke( 74 | cogeo_cli, ["update", "list.txt", "mosaic_1.json", "--quiet"] 75 | ) 76 | assert not result.exception 77 | assert result.exit_code == 0 78 | with open("mosaic_1.json", "r") as f: 79 | updated_mosaic = json.load(f) 80 | assert updated_mosaic["version"] == "1.0.1" 81 | assert not mosaic_content.tiles == updated_mosaic["tiles"] 82 | 83 | with open("mosaic_2.json", "w") as f: 84 | f.write(MosaicJSON.from_urls([asset1]).model_dump_json(exclude_none=True)) 85 | 86 | result = runner.invoke( 87 | cogeo_cli, ["update", "list.txt", "mosaic_2.json", "--add-last", "--quiet"] 88 | ) 89 | assert not result.exception 90 | assert result.exit_code == 0 91 | with open("mosaic_2.json", "r") as f: 92 | updated_mosaic = json.load(f) 93 | assert updated_mosaic["version"] == "1.0.1" 94 | assert mosaic_content.tiles == updated_mosaic["tiles"] 95 | 96 | 97 | def test_footprint_valid(): 98 | """Should work as expected.""" 99 | runner = CliRunner() 100 | with runner.isolated_filesystem(): 101 | with open("./list.txt", "w") as f: 102 | f.write("\n".join([asset1, asset2])) 103 | 104 | result = runner.invoke(cogeo_cli, ["footprint", "list.txt", "--quiet"]) 105 | assert not result.exception 106 | assert result.exit_code == 0 107 | footprint = json.loads(result.output) 108 | assert len(footprint["features"]) == 2 109 | 110 | result = runner.invoke( 111 | cogeo_cli, ["footprint", "list.txt", "-o", "mosaic.geojson"] 112 | ) 113 | assert not result.exception 114 | assert result.exit_code == 0 115 | with open("mosaic.geojson", "r") as f: 116 | footprint = json.load(f) 117 | assert len(footprint["features"]) == 2 118 | 119 | 120 | def test_from_features(): 121 | """Should work as expected.""" 122 | runner = CliRunner() 123 | with runner.isolated_filesystem(): 124 | with open("./list.txt", "w") as f: 125 | f.write("\n".join([asset1, asset2])) 126 | 127 | result = runner.invoke( 128 | cogeo_cli, ["footprint", "list.txt", "-o", "mosaic.geojson"] 129 | ) 130 | with open("mosaic.geojson", "r") as f: 131 | features = f.read() 132 | 133 | result = runner.invoke( 134 | cogeo_cli, 135 | [ 136 | "create-from-features", 137 | "--minzoom", 138 | "7", 139 | "--maxzoom", 140 | "9", 141 | "--property", 142 | "path", 143 | "--quiet", 144 | ], 145 | input=features, 146 | ) 147 | assert not result.exception 148 | assert result.exit_code == 0 149 | assert mosaic_content == MosaicJSON(**json.loads(result.output)) 150 | 151 | result = runner.invoke( 152 | cogeo_cli, 153 | [ 154 | "create-from-features", 155 | "--minzoom", 156 | "7", 157 | "--maxzoom", 158 | "9", 159 | "--property", 160 | "path", 161 | "-o", 162 | "mosaic.json", 163 | "--quiet", 164 | ], 165 | input=features, 166 | ) 167 | assert not result.exception 168 | assert result.exit_code == 0 169 | with open("mosaic.json", "r") as f: 170 | assert mosaic_content == MosaicJSON.model_validate_json(f.read()) 171 | 172 | result = runner.invoke( 173 | cogeo_cli, 174 | [ 175 | "create-from-features", 176 | "--minzoom", 177 | "7", 178 | "--maxzoom", 179 | "9", 180 | "--property", 181 | "path", 182 | "--name", 183 | "my_mosaic", 184 | "--description", 185 | "A mosaic", 186 | "--attribution", 187 | "someone", 188 | "--quiet", 189 | ], 190 | input=features, 191 | ) 192 | assert not result.exception 193 | assert result.exit_code == 0 194 | mosaic = MosaicJSON(**json.loads(result.output)) 195 | assert mosaic.name == "my_mosaic" 196 | assert mosaic.description == "A mosaic" 197 | assert mosaic.attribution == "someone" 198 | 199 | 200 | def test_info_valid(): 201 | """Should work as expected.""" 202 | runner = CliRunner() 203 | with runner.isolated_filesystem(): 204 | mosaic = os.path.join(os.path.dirname(__file__), "fixtures", "mosaic.json") 205 | result = runner.invoke(cogeo_cli, ["info", mosaic, "--json"]) 206 | assert not result.exception 207 | assert result.exit_code == 0 208 | info = json.loads(result.output) 209 | assert info["Backend"] == "File" 210 | assert not info["Compressed"] 211 | 212 | mosaic = os.path.join(os.path.dirname(__file__), "fixtures", "mosaic.json.gz") 213 | result = runner.invoke(cogeo_cli, ["info", mosaic, "--json"]) 214 | assert not result.exception 215 | assert result.exit_code == 0 216 | info = json.loads(result.output) 217 | assert info["Backend"] == "File" 218 | assert info["Compressed"] 219 | 220 | mosaic = os.path.join(os.path.dirname(__file__), "fixtures", "mosaic.json.gz") 221 | result = runner.invoke(cogeo_cli, ["info", mosaic]) 222 | assert not result.exception 223 | assert result.exit_code == 0 224 | assert "Compressed: True" in result.output 225 | 226 | 227 | def test_to_geojson(): 228 | """Should work as expected.""" 229 | runner = CliRunner() 230 | with runner.isolated_filesystem(): 231 | mosaic = os.path.join(os.path.dirname(__file__), "fixtures", "mosaic.json") 232 | result = runner.invoke(cogeo_cli, ["to-geojson", mosaic]) 233 | assert not result.exception 234 | assert result.exit_code == 0 235 | info = result.output.split("\n") 236 | assert len(info) == 10 237 | assert json.loads(info[0])["properties"]["nb_assets"] == 1 238 | 239 | mosaic = os.path.join(os.path.dirname(__file__), "fixtures", "mosaic.json") 240 | result = runner.invoke(cogeo_cli, ["to-geojson", mosaic, "--features"]) 241 | assert not result.exception 242 | assert result.exit_code == 0 243 | info = result.output.split("\n") 244 | assert len(info) == 10 245 | assert json.loads(info[0])["properties"]["nb_assets"] == 1 246 | 247 | mosaic = os.path.join(os.path.dirname(__file__), "fixtures", "mosaic.json") 248 | result = runner.invoke(cogeo_cli, ["to-geojson", mosaic, "--collect"]) 249 | assert not result.exception 250 | assert result.exit_code == 0 251 | info = json.loads(result.output) 252 | assert len(info["features"]) == 9 253 | -------------------------------------------------------------------------------- /tests/fixtures/mars_ctx_stac_asset.json: -------------------------------------------------------------------------------- 1 | {"type":"Feature","stac_version":"1.0.0","id":"P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W","properties":{"start_datetime":"2023-04-18T12:00:00.000000Z","end_datetime":"2023-04-18T12:00:00.000000Z","title":"DTM: P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W","description":"This is a digital terrain model (DTM) extracted from Context Camera (CTX) stereo images from the Mars Reconnaissance Orbiter mission. This data product is a DTM from stereo images acquired at approximately 6 meters/pixel resolution, which allows an output DTM resolution of 20 meters/pixel. The DTM was generated using the Ames Stereo Pipeline software (https://github.com/NeoGeographyToolkit/StereoPipeline)and using automated methods. All MRO CTX images were relatively controlled to one another before DTM generation. The final DTM is absolutely aligned to MOLA. Therefore,these are absolutely controlled DTMs and orthoimages. The isis_provenance and asp_provenance metadata files provide reproducible workflows for the generation of these DTM products. Input data were first relatively aligned and photogrammetrically controlled at the Mars quad level. Next, community sensor model (CSM) Image Support Data (ISDs) were created to support processing using the CSM in ASP. Low resolution, 100mpp DTMs were then generated for each stereopair and used to seed the generation of a high resolution DTM. This high resolution DTM was aligned to MOLA using a two step pc_align process. Fro the aligned point cloud the final DEM was extracted at the described post spacing and a final adjustment made to align with the Mars aeroid (geoid). All products for release were then converted to cloud optimized geotiffs (COGs) and appropriate metadata generated.","missions":["Mars Reconnaissance Orbiter (MRO)"],"instruments":["Context Camera (CTX)"],"gsd":20.162955124498982,"license":"CC0-1.0","proj:epsg":null,"proj:wkt2":"PROJCS[\"Mars (2015) - Sphere / Ocentric / Equirectangular, clon = 0\",GEOGCS[\"Mars (2015) - Sphere / Ocentric\",DATUM[\"Mars (2015) - Sphere\",SPHEROID[\"Mars (2015) - Sphere\",3396190,0]],PRIMEM[\"Reference Meridian\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]]],PROJECTION[\"Equirectangular\"],PARAMETER[\"standard_parallel_1\",0],PARAMETER[\"central_meridian\",0],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AXIS[\"Easting\",EAST],AXIS[\"Northing\",NORTH]]","proj:projjson":{"$schema":"https://proj.org/schemas/v0.5/projjson.schema.json","type":"ProjectedCRS","name":"Mars (2015) - Sphere / Ocentric / Equirectangular, clon = 0","base_crs":{"name":"Mars (2015) - Sphere / Ocentric","datum":{"type":"GeodeticReferenceFrame","name":"Mars (2015) - Sphere","ellipsoid":{"name":"Mars (2015) - Sphere","radius":3396190},"prime_meridian":{"name":"Reference Meridian","longitude":0}},"coordinate_system":{"subtype":"ellipsoidal","axis":[{"name":"Latitude","abbreviation":"lat","direction":"north","unit":"degree"},{"name":"Longitude","abbreviation":"lon","direction":"east","unit":"degree"}]}},"conversion":{"name":"Equidistant Cylindrical","method":{"name":"Equidistant Cylindrical","id":{"authority":"EPSG","code":1028}},"parameters":[{"name":"Latitude of 1st standard parallel","value":0,"unit":"degree","id":{"authority":"EPSG","code":8823}},{"name":"Latitude of natural origin","value":0,"unit":{"type":"Unit","name":"","conversion_factor":1},"id":{"authority":"EPSG","code":8801}},{"name":"Longitude of natural origin","value":0,"unit":"degree","id":{"authority":"EPSG","code":8802}},{"name":"False easting","value":0,"unit":"metre","id":{"authority":"EPSG","code":8806}},{"name":"False northing","value":0,"unit":"metre","id":{"authority":"EPSG","code":8807}}]},"coordinate_system":{"subtype":"Cartesian","axis":[{"name":"Easting","abbreviation":"","direction":"east","unit":"metre"},{"name":"Northing","abbreviation":"","direction":"north","unit":"metre"}]}},"proj:bbox":[77.21281144352704,17.757789235602306,77.90597708129924,19.342308862718042],"proj:centroid":{"lat":18.54986926024949,"lon":77.56103123535196},"proj:shape":[4622,2013],"proj:transform":[4577133.445008798,20.162955124498982,0,1146178.5021103406,0,-20.162955124498982],"cube:dimensions":{"x":{"type":"spatial","axis":"x","extent":[77.21281144352704,77.90597708129924]},"y":{"type":"spatial","axis":"y","extent":[17.757789235602306,19.342308862718042]}},"ssys:targets":["Mars"],"datetime":"2023-04-18T12:00:00Z","created":"2023-04-20T16:57:02.159Z","updated":"2023-04-20T16:57:02.159Z"},"geometry":{"type":"Polygon","coordinates":[[[77.71977189692257,19.30644581710876],[77.73374362999547,19.199960539096043],[77.74668316427005,19.093404678994183],[77.75971635385247,18.986829362583926],[77.77336800587166,18.880245997622627],[77.78660276686999,18.773714692988722],[77.80061294042936,18.66719068225428],[77.81431877772636,18.560607364148215],[77.82762242510964,18.454052551924853],[77.84085613299511,18.34738696211589],[77.85401697473928,18.240761211757988],[77.86684191221299,18.134180260139566],[77.87895428686141,18.027417629605058],[77.8924163537452,17.920861563926916],[77.90597708129924,17.814316704559253],[77.83409926614429,17.806215920363115],[77.72623115343649,17.794050318695703],[77.61847050469058,17.78183611608723],[77.51196826567741,17.769701970266237],[77.40796323471596,17.757789235602306],[77.40350807415773,17.792522409458623],[77.38982540827715,17.899063427271912],[77.37610163449676,18.00570431460465],[77.36249355451288,18.112355267752143],[77.3490926339429,18.218903213499647],[77.33598708083149,18.325598564713555],[77.32247121215444,18.43221848480228],[77.30846140131688,18.538793274067014],[77.29518013686905,18.64545406438174],[77.28142189883427,18.75202153305719],[77.26758569866713,18.85855465666753],[77.25389822730428,18.965180648111712],[77.23994352496726,19.071695834407794],[77.2259278468269,19.17819850670837],[77.21281144352704,19.279588080081055],[77.26240252797434,19.285865330102165],[77.38089234319978,19.300757103433142],[77.49880775223247,19.31550057912437],[77.61458468848255,19.329897115218557],[77.71497431817204,19.342308862718042],[77.71977189692257,19.30644581710876]]]},"links":[{"rel":"self","type":"application/geo+json","href":"https://stac.astrogeology.usgs.gov/api//collections/mro_ctx_controlled_usgs_dtms/items/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W"},{"rel":"parent","type":"application/json","href":"https://stac.astrogeology.usgs.gov/api//collections/mro_ctx_controlled_usgs_dtms"},{"rel":"collection","type":"application/json","href":"https://stac.astrogeology.usgs.gov/api//collections/mro_ctx_controlled_usgs_dtms"},{"rel":"root","type":"application/json","href":"https://stac.astrogeology.usgs.gov/api/"},{"rel":"thumbnail","href":"https://stac.astrogeology.usgs.gov/api//collections/mro_ctx_controlled_usgs_dtms/items/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W/thumbnail"}],"assets":{"image":{"roles":["data","overview"],"href":"https://astrogeo-ard.s3-us-west-2.amazonaws.com/mars/mro/ctx/controlled/usgs/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W_hillshade.tif","type":"image/tiff; application=geotiff; profile=cloud-optimized","title":"Hillshade","raster:bands":[{"data_type":"uint8","nodata":0,"statistics":{"mean":136.395,"maximum":255,"stddev":33.988,"minimum":1}}],"key":"image"},"thumbnail":{"roles":["thumbnail"],"href":"https://astrogeo-ard.s3-us-west-2.amazonaws.com/mars/mro/ctx/controlled/usgs/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W.jpg","type":"image/jpeg","title":"Thumbnail","key":"thumbnail"},"orthoimage":{"roles":["data"],"href":"https://astrogeo-ard.s3-us-west-2.amazonaws.com/mars/mro/ctx/controlled/usgs/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W_orthoimage.tif","type":"image/tiff; application=geotiff; profile=cloud-optimized","title":"Orthoimage","raster:bands":[{"data_type":"float32","nodata":-32767,"statistics":{"mean":0.595,"maximum":1.141,"stddev":0.128,"minimum":-0.1}}],"key":"orthoimage"},"qa_metrics":{"roles":["metadata"],"href":"https://astrogeo-ard.s3-us-west-2.amazonaws.com/mars/mro/ctx/controlled/usgs/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W/qa_metrics.txt","type":"text/plain","title":"ASP Generated Quality Assurance Metrics","key":"qa_metrics"},"dtm":{"roles":["data"],"href":"https://astrogeo-ard.s3-us-west-2.amazonaws.com/mars/mro/ctx/controlled/usgs/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W_dtm.tif","type":"image/tiff; application=geotiff; profile=cloud-optimized","title":"Digital Terrain Model","raster:bands":[{"data_type":"float32","nodata":-32767,"statistics":{"mean":-2231.655,"maximum":-1356.129,"stddev":336.754,"minimum":-3070.956}}],"key":"dtm"},"mola_csv":{"roles":["metadata"],"href":"https://astrogeo-ard.s3-us-west-2.amazonaws.com/mars/mro/ctx/controlled/usgs/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W_mola_shots.csv","type":"text/plain","title":"MOLA Spot Observations Used for initial pc-align","key":"mola_csv"},"intersection_error":{"roles":["metadata"],"href":"https://astrogeo-ard.s3-us-west-2.amazonaws.com/mars/mro/ctx/controlled/usgs/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W/P18_007925_1987_XN_18N282W__P19_008650_1987_XI_18N282W_intersection_error.tif","type":"image/tiff; application=geotiff; profile=cloud-optimized","title":"Intersection Error","raster:bands":[{"data_type":"float32","nodata":-32767,"statistics":{"mean":1.464,"maximum":6.359,"stddev":1.026,"minimum":0}}],"key":"intersection_error"}},"bbox":[77.21281144352704,17.757789235602306,77.90597708129924,19.342308862718042],"stac_extensions":["https://stac-extensions.github.io/projection/v1.0.0/schema.json","https://stac-extensions.github.io/datacube/v1.0.0/schema.json","https://raw.githubusercontent.com/thareUSGS/ssys/main/json-schema/schema.json"],"collection":"mro_ctx_controlled_usgs_dtms"} -------------------------------------------------------------------------------- /docs/src/advanced/backends.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Starting in version `3.0.0`, we introduced specific `backend` to abstract mosaicJSON storage. 4 | 5 | #### Read-Write Backends 6 | 7 | - **FileBackend** (default, `file:///`) 8 | 9 | - **S3Backend** (`s3://`) 10 | 11 | - **GCSBackend** (`gs://`) 12 | 13 | - **ABSBackend** (`az://{storageaccount}.blob.core.windows.net/{container}/{key}`) 14 | 15 | - **DynamoDBBackend** (`dynamodb://{region}/{table_name}`). If `region` is not passed, it reads the value of the `AWS_REGION` environment variable. If that environment variable does not exist, it falls back to `us-east-1`. If you choose not to pass a `region`, you still need three `/` before the table name, like so `dynamodb:///{table_name}`. 16 | 17 | - **SQLiteBackend** (`sqlite:///{file.db}:{mosaic_name}`) 18 | 19 | #### Read Only Backends 20 | 21 | Read only backend won't allow `mosaic_def` in there `__init__` method. `.write()` and `.update` methods will raise `NotImplementedError` error. 22 | 23 | - **HttpBackend** (`http://`, `https://`) 24 | - **STACBackend** (`stac+:https://`). Based on [SpatioTemporal Asset Catalog](https://github.com/radiantearth/stac-spec) API. 25 | 26 | #### In-Memory 27 | 28 | If you have a mosaicjson document and want to use the different backend methods you can use the special **MemoryBackend**. 29 | 30 | ```python 31 | with MemoryBackend(mosaic_def=mosaicjson) as mosaic: 32 | img = mosaic.tile(1, 1, 1) 33 | ``` 34 | 35 | ### Abstract Class 36 | 37 | All backends are built from a `MosaicJSONBackend` which is a sub-class or `rio-tiler.mosaic.backend.BaseBackend`. 38 | 39 | ## MosaicBackend 40 | 41 | To ease the usage we added a helper function to use the right backend based on the uri schema: `cogeo_mosaic.backends.MosaicBackend` 42 | 43 | ```python 44 | from cogeo_mosaic.backends import MosaicBackend 45 | 46 | with MosaicBackend("s3://mybucket/amosaic.json") as mosaic: 47 | assert isinstance(mosaic, cogeo_mosaic.backends.s3.S3Backend) 48 | 49 | with MosaicBackend("gs://mybucket/amosaic.json") as mosaic: 50 | assert isinstance(mosaic, cogeo_mosaic.backends.gs.GCSBackend) 51 | 52 | with MosaicBackend("dynamodb://us-east-1/amosaic") as mosaic: 53 | assert isinstance(mosaic, cogeo_mosaic.backends.dynamodb.DynamoDBBackend) 54 | 55 | with MosaicBackend("sqlite:///mosaic.db:amosaic") as mosaic: 56 | assert isinstance(mosaic, cogeo_mosaic.backends.sqlite.SQLiteBackend) 57 | 58 | with MosaicBackend("file:///amosaic.json.gz") as mosaic: 59 | assert isinstance(mosaic, cogeo_mosaic.backends.file.FileBackend) 60 | 61 | with MosaicBackend("amosaic.json.gz") as mosaic: 62 | assert isinstance(mosaic, cogeo_mosaic.backends.file.FileBackend) 63 | 64 | # Read-Only 65 | with MosaicBackend("https://mosaic.com/amosaic.json.gz") as mosaic: 66 | assert isinstance(mosaic, cogeo_mosaic.backends.web.HttpBackend) 67 | 68 | with MosaicBackend("stac+https://my-stac.api/search", {"collections": ["satellite"]}, 10, 12) as mosaic: 69 | assert isinstance(mosaic, cogeo_mosaic.backends.stac.STACBackend) 70 | 71 | # In Memory (write) 72 | # You can pass either None or ':memory:' to define an in-memory backend 73 | with MosaicBackend(":memory:", mosaic_def=mosaic) as mosaic: 74 | assert isinstance(mosaic, cogeo_mosaic.backends.memory.MemoryBackend) 75 | 76 | with MosaicBackend(None, mosaic_def=mosaic) as mosaic: 77 | assert isinstance(mosaic, cogeo_mosaic.backends.memory.MemoryBackend) 78 | ``` 79 | 80 | ## GCS Backend 81 | 82 | The GCS Backend allows read and write operations from Google Cloud Storage. 83 | 84 | When using this backend is necessary to set the appropriate roles and IAM 85 | permissions to let cogeo-mosaic access the files. For example: 86 | 87 | * Read-only bucket - IAM Role `roles/storage.objectViewer`. It is possible 88 | to restrict the read-only operation to a single bucket by using the 89 | following condition: `resource.type == "storage.googleapis.com/Object" 90 | && resource.name.startsWith("projects/_/buckets/mybucket")` 91 | 92 | ## ABS Backend 93 | 94 | The ABS Backend allows read and write operations from/to Azure Blob Storage. 95 | 96 | The backend uses DefaultAzureCredential for authorization, see 97 | [here](https://learn.microsoft.com/en-us/python/api/overview/azure/identity-readme?view=azure-python#defaultazurecredential) 98 | for details on how to authenticate. 99 | 100 | When using this backend is necessary to set the appropriate roles and IAM 101 | permissions to let cogeo-mosaic access the files. See the Azure standard 102 | roles; Storage Blob Data Contributor/Owner/Reader. 103 | 104 | ## STAC Backend 105 | 106 | The STACBackend is a **read-only** backend, meaning it can't be used to write a file. This backend will POST to the input url looking for STAC items which will then be used to create the mosaicJSON in memory. 107 | 108 | ```python 109 | import datetime 110 | import morecantile 111 | from cogeo_mosaic.backends.stac import STACBackend 112 | 113 | 114 | geojson = { 115 | "type": "FeatureCollection", 116 | "features": [ 117 | { 118 | "type": "Feature", 119 | "properties": {}, 120 | "geometry": { 121 | "type": "Polygon", 122 | "coordinates": [ 123 | [ 124 | [ 125 | 30.810813903808594, 126 | 29.454247067148533 127 | ], 128 | [ 129 | 30.88600158691406, 130 | 29.454247067148533 131 | ], 132 | [ 133 | 30.88600158691406, 134 | 29.51879923863822 135 | ], 136 | [ 137 | 30.810813903808594, 138 | 29.51879923863822 139 | ], 140 | [ 141 | 30.810813903808594, 142 | 29.454247067148533 143 | ] 144 | ] 145 | ] 146 | } 147 | } 148 | ] 149 | } 150 | 151 | 152 | date_min="2019-01-01" 153 | date_max="2019-12-11" 154 | 155 | start = datetime.datetime.strptime(date_min, "%Y-%m-%d").strftime("%Y-%m-%dT00:00:00Z") 156 | end = datetime.datetime.strptime(date_max, "%Y-%m-%d").strftime("%Y-%m-%dT23:59:59Z") 157 | 158 | query = { 159 | "collections": ["sentinel-s2-l2a-cogs"], 160 | "datetime": f"{start}/{end}", 161 | "query": { 162 | "eo:cloud_cover": { 163 | "lt": 5 164 | } 165 | }, 166 | "intersects": geojson["features"][0]["geometry"], 167 | "limit": 1000, 168 | "fields": { 169 | 'include': ['id', 'properties.datetime', 'properties.data_coverage'], 170 | 'exclude': ['assets'] 171 | } 172 | } 173 | 174 | with STACBackend( 175 | "https://earth-search.aws.element84.com/v0/search", 176 | query=query, 177 | minzoom=8, 178 | maxzoom=15, 179 | ) as mosaic: 180 | print(mosaic.mosaic_def.dict(exclude={"tiles"})) 181 | ``` 182 | 183 | #### Mosaic TileMatrixSet vs Backend TileMatrixSet 184 | 185 | If using a TileMatrixSet other than the default `WebMercatorQuad`, user HAS TO pass mosaic min/max zoom (different than the backend min/max zoom). 186 | 187 | ```python 188 | with STACBackend( 189 | "https://earth-search.aws.element84.com/v0/search", 190 | query=query, 191 | tms=morecantile.tms.get("WGS1984Quad"), 192 | minzoom=8, 193 | maxzoom=12, 194 | mosaic_options={ 195 | "tilematrixset": morecantile.tms.get("WebMercatorQuad"), # This is Optional and will be set by default to `WebMercatorQuad` TMS 196 | "minzoom": 8, 197 | "maxzoom": 14, 198 | }, 199 | ) as mosaic: 200 | print(mosaic.mosaic_def.dict(exclude={"tiles"})) 201 | ``` 202 | 203 | #### Specification 204 | 205 | The STACBackend rely on Spec version 1.0.0alpha. 206 | 207 | #### Paggination 208 | 209 | The returned object from the POST requests might not represent the whole results and thus 210 | we need to use the paggination. 211 | 212 | You can limit the pagination by using `max_items` or `limit` stac-api options. 213 | 214 | - Limit the total result to 1000 items 215 | 216 | ```python 217 | with STACBackend( 218 | "https://earth-search.aws.element84.com/v0/search", 219 | query={}, 220 | minzoom=8, 221 | maxzoom=15, 222 | stac_api_options={"max_items": 1000}, 223 | ) as mosaic: 224 | print(mosaic.mosaic_def.dict(exclude={"tiles"})) 225 | ``` 226 | 227 | - Limit the size of each POST result 228 | 229 | ```python 230 | with STACBackend( 231 | "https://earth-search.aws.element84.com/v0/search", 232 | query={}, 233 | minzoom=8, 234 | maxzoom=15, 235 | stac_api_options={"limit": 100}, 236 | ) as mosaic: 237 | print(mosaic.mosaic_def.dict(exclude={"tiles"})) 238 | ``` 239 | Warnings: trying to run the previous example will results in fetching the whole collection. 240 | 241 | 242 | #### Tile's asset 243 | 244 | MosaicJSON tile asset is defined using `accessor` option. By default the backend will try to construct or retrieve the Item url 245 | 246 | ```python 247 | def default_stac_accessor(feature: Dict): 248 | """Return feature identifier.""" 249 | link = list(filter(lambda link: link["rel"] == "self", feature["links"])) 250 | if link: 251 | return link[0]["href"] 252 | 253 | link = list(filter(lambda link: link["rel"] == "root", feature["links"])) 254 | if link: 255 | return os.path.join( 256 | link[0]["href"], 257 | "collections", 258 | feature["collection"], 259 | "items", 260 | feature["id"], 261 | ) 262 | 263 | # Fall back to the item ID 264 | return feature["id"] 265 | ``` 266 | 267 | This default accessor function rely on the `self` or `root` link to be present. 268 | 269 | It's let to the user to built a Mosaic Tiler which will understand the asset. 270 | 271 | #### Custom accessor 272 | 273 | Accessor HAVE to be a callable which take a GeoJSON feature as input. 274 | 275 | Here is an example of an accessor that will return the ulr for asset `B01` 276 | 277 | ```python 278 | with STACBackend( 279 | "https://earth-search.aws.element84.com/v0/search", 280 | query={}, 281 | minzoom=8, 282 | maxzoom=15, 283 | stac_api_options={"limit": 100}, 284 | mosaic_options={"accessor": lambda x: x["assets"]["B01"]["href"]}, 285 | ) as mosaic: 286 | print(mosaic.mosaic_def.dict(exclude={"tiles"})) 287 | ``` 288 | -------------------------------------------------------------------------------- /cogeo_mosaic/backends/base.py: -------------------------------------------------------------------------------- 1 | """cogeo_mosaic.backend.base: base Backend class.""" 2 | 3 | import abc 4 | import itertools 5 | import warnings 6 | from threading import Lock 7 | from typing import Any, Dict, List, Optional, Sequence, Type, Union 8 | 9 | import attr 10 | from cachetools import TTLCache, cached 11 | from cachetools.keys import hashkey 12 | from morecantile import Tile, TileMatrixSet 13 | from rasterio.crs import CRS 14 | from rasterio.warp import transform, transform_bounds 15 | from rio_tiler.constants import WEB_MERCATOR_TMS 16 | from rio_tiler.io import BaseReader, MultiBandReader, MultiBaseReader, Reader 17 | from rio_tiler.mosaic.backend import BaseBackend 18 | from rio_tiler.types import BBox 19 | from rio_tiler.utils import CRS_to_uri 20 | 21 | from cogeo_mosaic.backends.utils import get_hash 22 | from cogeo_mosaic.cache import cache_config 23 | from cogeo_mosaic.models import Info 24 | from cogeo_mosaic.mosaic import MosaicJSON 25 | from cogeo_mosaic.utils import bbox_union 26 | 27 | 28 | def _convert_to_mosaicjson(value: Union[Dict, MosaicJSON]): 29 | if value is not None: 30 | return MosaicJSON(**dict(value)) 31 | 32 | 33 | @attr.s 34 | class MosaicJSONBackend(BaseBackend): 35 | """Base Class for cogeo-mosaic backend storage. 36 | 37 | Attributes: 38 | input (str): mosaic path. 39 | mosaic_def (MosaicJSON, optional): mosaicJSON document. 40 | tms (morecantile.TileMatrixSet, optional): TileMatrixSet grid definition. Defaults to `WebMercatorQuad`. 41 | minzoom (int): mosaic Min zoom level. Defaults to tms or mosaic minzoom. 42 | maxzoom (int): mosaic Max zoom level. Defaults to tms or mosaic maxzoom. 43 | reader (rio_tiler.io.BaseReader): Dataset reader. Defaults to `rio_tiler.io.Reader`. 44 | reader_options (dict): Options to forward to the reader config. 45 | bounds (tuple): mosaic bounds (left, bottom, right, top). **READ ONLY attribute**. Defaults to `(-180, -90, 180, 90)`. 46 | crs (rasterio.crs.CRS): mosaic crs in which its bounds is defined. **READ ONLY attribute**. Defaults to WGS84. 47 | geographic_crs (rasterio.crs.CRS, optional): CRS to use as geographic coordinate system. **READ ONLY attribute**. Defaults to WGS84. 48 | 49 | """ 50 | 51 | input: str = attr.ib() 52 | mosaic_def: MosaicJSON = attr.ib(default=None, converter=_convert_to_mosaicjson) 53 | tms: TileMatrixSet = attr.ib(default=WEB_MERCATOR_TMS) 54 | 55 | minzoom: int = attr.ib(default=None) 56 | maxzoom: int = attr.ib(default=None) 57 | 58 | reader: Union[ 59 | Type[BaseReader], 60 | Type[MultiBaseReader], 61 | Type[MultiBandReader], 62 | ] = attr.ib(default=Reader) 63 | reader_options: Dict = attr.ib(factory=dict) 64 | 65 | bounds: BBox = attr.ib(init=False) 66 | crs: CRS = attr.ib(init=False) 67 | 68 | _backend_name: str 69 | _file_byte_size: Optional[int] = 0 70 | 71 | def __attrs_post_init__(self): 72 | """Post Init: if not passed in init, try to read from self.input.""" 73 | self.mosaic_def = self.mosaic_def or self._read() 74 | self.bounds = self.mosaic_def.bounds 75 | 76 | # in order to keep support for old mosaic document we assume the default TMS to be WebMercatorQuad 77 | mosaic_tms = self.mosaic_def.tilematrixset or WEB_MERCATOR_TMS 78 | 79 | # By mosaic definition, its `bounds` is defined using the mosaic's TMS 80 | # Geographic CRS. 81 | self.crs = mosaic_tms.rasterio_geographic_crs 82 | 83 | # if we open the Mosaic with a TMS which is not the mosaic TMS 84 | # the min/max zoom will default to the TMS (read) zooms 85 | # except if min/max zoom are passed by the user 86 | if self.minzoom is None: 87 | self.minzoom = ( 88 | self.mosaic_def.minzoom if mosaic_tms == self.tms else self.tms.minzoom 89 | ) 90 | 91 | if self.maxzoom is None: 92 | self.maxzoom = ( 93 | self.mosaic_def.maxzoom if mosaic_tms == self.tms else self.tms.maxzoom 94 | ) 95 | 96 | @abc.abstractmethod 97 | def _read(self) -> MosaicJSON: 98 | """Fetch mosaic definition""" 99 | 100 | @abc.abstractmethod 101 | def write(self, overwrite: bool = True): 102 | """Upload new MosaicJSON to backend.""" 103 | 104 | def update( 105 | self, 106 | features: Sequence[Dict], 107 | add_first: bool = True, 108 | quiet: bool = False, 109 | **kwargs, 110 | ): 111 | """Update existing MosaicJSON on backend.""" 112 | new_mosaic = MosaicJSON.from_features( 113 | features, 114 | self.mosaic_def.minzoom, 115 | self.mosaic_def.maxzoom, 116 | quadkey_zoom=self.quadkey_zoom, 117 | tilematrixset=self.mosaic_def.tilematrixset, 118 | quiet=quiet, 119 | **kwargs, 120 | ) 121 | 122 | for quadkey, new_assets in new_mosaic.tiles.items(): 123 | mosaic_tms = self.mosaic_def.tilematrixset or WEB_MERCATOR_TMS 124 | tile = mosaic_tms.quadkey_to_tile(quadkey) 125 | assets = self.assets_for_tile(*tile) 126 | assets = [*new_assets, *assets] if add_first else [*assets, *new_assets] 127 | 128 | # add custom sorting algorithm (e.g based on path name) 129 | self.mosaic_def.tiles[quadkey] = assets 130 | 131 | bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds) 132 | 133 | if self.mosaic_def.mosaicjson != new_mosaic.mosaicjson: 134 | warnings.warn( 135 | f"Updating `mosaicjson` version from {self.mosaic_def.mosaicjson} to {new_mosaic.mosaicjson}" 136 | ) 137 | self.mosaic_def.mosaicjson = new_mosaic.mosaicjson 138 | 139 | self.mosaic_def._increase_version() 140 | self.mosaic_def.bounds = bounds 141 | self.mosaic_def.center = ( 142 | (bounds[0] + bounds[2]) / 2, 143 | (bounds[1] + bounds[3]) / 2, 144 | self.mosaic_def.minzoom, 145 | ) 146 | self.bounds = bounds 147 | self.write(overwrite=True) 148 | 149 | def assets_for_tile(self, x: int, y: int, z: int, **kwargs: Any) -> List[str]: 150 | """Retrieve assets for tile.""" 151 | mosaic_tms = self.mosaic_def.tilematrixset or WEB_MERCATOR_TMS 152 | if self.tms == mosaic_tms: 153 | return self.get_assets(x, y, z, **kwargs) 154 | 155 | # If TMS are different, then use Tile's geographic coordinates 156 | # and `assets_for_bbox` to get the assets 157 | xmin, ymin, xmax, ymax = self.tms.bounds(x, y, z) 158 | return self.assets_for_bbox( 159 | xmin, 160 | ymin, 161 | xmax, 162 | ymax, 163 | coord_crs=self.tms.rasterio_geographic_crs, 164 | **kwargs, 165 | ) 166 | 167 | def assets_for_point( 168 | self, 169 | lng: float, 170 | lat: float, 171 | coord_crs: Optional[CRS] = None, 172 | **kwargs: Any, 173 | ) -> List[str]: 174 | """Retrieve assets for point.""" 175 | mosaic_tms = self.mosaic_def.tilematrixset or WEB_MERCATOR_TMS 176 | 177 | # default coord_crs should be the TMS's geographic CRS 178 | coord_crs = coord_crs or self.tms.rasterio_geographic_crs 179 | 180 | # If coord_crs is not the same as the mosaic's geographic CRS 181 | # we reproject the coordinates 182 | if coord_crs != mosaic_tms.rasterio_geographic_crs: 183 | xs, ys = transform( 184 | coord_crs, mosaic_tms.rasterio_geographic_crs, [lng], [lat] 185 | ) 186 | lng, lat = xs[0], ys[0] 187 | 188 | # Find the tile index using geographic coordinates 189 | tile = mosaic_tms.tile(lng, lat, self.quadkey_zoom) 190 | 191 | return self.get_assets(tile.x, tile.y, tile.z, **kwargs) 192 | 193 | def assets_for_bbox( 194 | self, 195 | xmin: float, 196 | ymin: float, 197 | xmax: float, 198 | ymax: float, 199 | coord_crs: Optional[CRS] = None, 200 | **kwargs, 201 | ) -> List[str]: 202 | """Retrieve assets for bbox.""" 203 | mosaic_tms = self.mosaic_def.tilematrixset or WEB_MERCATOR_TMS 204 | 205 | # default coord_crs should be the TMS's geographic CRS 206 | coord_crs = coord_crs or self.tms.rasterio_geographic_crs 207 | 208 | # If coord_crs is not the same as the mosaic's geographic CRS 209 | # we reproject the bounding box 210 | if coord_crs != mosaic_tms.rasterio_geographic_crs: 211 | xmin, ymin, xmax, ymax = transform_bounds( 212 | coord_crs, 213 | mosaic_tms.rasterio_geographic_crs, 214 | xmin, 215 | ymin, 216 | xmax, 217 | ymax, 218 | ) 219 | 220 | tl_tile = mosaic_tms.tile(xmin, ymax, self.quadkey_zoom) 221 | br_tile = mosaic_tms.tile(xmax, ymin, self.quadkey_zoom) 222 | 223 | tiles = [ 224 | (x, y, self.quadkey_zoom) 225 | for x in range(tl_tile.x, br_tile.x + 1) 226 | for y in range(tl_tile.y, br_tile.y + 1) 227 | ] 228 | 229 | return list( 230 | dict.fromkeys( 231 | itertools.chain.from_iterable( 232 | [self.get_assets(*t, **kwargs) for t in tiles] 233 | ) 234 | ) 235 | ) 236 | 237 | @cached( # type: ignore 238 | TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl), 239 | key=lambda self, x, y, z, **kwargs: hashkey( 240 | self.input, x, y, z, self.mosaicid, **kwargs 241 | ), 242 | lock=Lock(), 243 | ) 244 | def get_assets(self, x: int, y: int, z: int, reverse: bool = False) -> List[str]: 245 | """Find assets.""" 246 | quadkeys = self.find_quadkeys(Tile(x=x, y=y, z=z), self.quadkey_zoom) 247 | assets = list( 248 | dict.fromkeys( 249 | itertools.chain.from_iterable( 250 | [self.mosaic_def.tiles.get(qk, []) for qk in quadkeys] 251 | ) 252 | ) 253 | ) 254 | if self.mosaic_def.asset_prefix: 255 | assets = [self.mosaic_def.asset_prefix + asset for asset in assets] 256 | 257 | if reverse: 258 | assets = list(reversed(assets)) 259 | 260 | return assets 261 | 262 | def find_quadkeys(self, tile: Tile, quadkey_zoom: int) -> List[str]: 263 | """ 264 | Find quadkeys at desired zoom for tile 265 | 266 | Attributes 267 | ---------- 268 | tile: morecantile.Tile 269 | Input tile to use when searching for quadkeys 270 | quadkey_zoom: int 271 | Zoom level 272 | 273 | Returns 274 | ------- 275 | list 276 | List[str] of quadkeys 277 | 278 | """ 279 | mosaic_tms = self.mosaic_def.tilematrixset or WEB_MERCATOR_TMS 280 | 281 | # get parent 282 | if tile.z > quadkey_zoom: 283 | depth = tile.z - quadkey_zoom 284 | for _ in range(depth): 285 | tile = mosaic_tms.parent(tile)[0] 286 | 287 | return [mosaic_tms.quadkey(*tile)] 288 | 289 | # get child 290 | elif tile.z < quadkey_zoom: 291 | depth = quadkey_zoom - tile.z 292 | 293 | tiles = [tile] 294 | for _ in range(depth): 295 | tiles = sum([mosaic_tms.children(t) for t in tiles], []) 296 | 297 | tiles = list(filter(lambda t: t.z == quadkey_zoom, tiles)) 298 | return [mosaic_tms.quadkey(*tile) for tile in tiles] 299 | 300 | else: 301 | return [mosaic_tms.quadkey(*tile)] 302 | 303 | def info(self, quadkeys: bool = False) -> Info: # type: ignore 304 | """Mosaic info.""" 305 | return Info( 306 | bounds=self.bounds, 307 | crs=CRS_to_uri(self.crs) or self.crs.to_wkt(), 308 | name=self.mosaic_def.name if self.mosaic_def.name else "mosaic", 309 | quadkeys=[] if not quadkeys else self._quadkeys, 310 | mosaic_tilematrixset=repr(self.mosaic_def.tilematrixset or WEB_MERCATOR_TMS), 311 | mosaic_minzoom=self.mosaic_def.minzoom, 312 | mosaic_maxzoom=self.mosaic_def.maxzoom, 313 | ) 314 | 315 | @property 316 | def mosaicid(self) -> str: 317 | """Return sha224 id of the mosaicjson document.""" 318 | return get_hash(**self.mosaic_def.model_dump(exclude_none=True)) 319 | 320 | @property 321 | def _quadkeys(self) -> List[str]: 322 | """Return the list of quadkey tiles.""" 323 | return list(self.mosaic_def.tiles) 324 | 325 | @property 326 | def quadkey_zoom(self) -> int: 327 | """Return Quadkey zoom property.""" 328 | return self.mosaic_def.quadkey_zoom or self.mosaic_def.minzoom 329 | -------------------------------------------------------------------------------- /cogeo_mosaic/mosaic.py: -------------------------------------------------------------------------------- 1 | """cogeo_mosaic.mosaic MosaicJSON models and helper functions.""" 2 | 3 | import os 4 | import re 5 | import sys 6 | import warnings 7 | from contextlib import ExitStack 8 | from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple 9 | 10 | import click 11 | import morecantile 12 | from pydantic import BaseModel, Field, field_validator, model_validator 13 | from rio_tiler.types import BBox 14 | from shapely import linearrings, polygons, total_bounds 15 | from shapely.strtree import STRtree 16 | from supermorecado import burnTiles 17 | 18 | from cogeo_mosaic.errors import MosaicError, MultipleDataTypeError 19 | from cogeo_mosaic.utils import _intersect_percent, get_footprints 20 | 21 | WEB_MERCATOR_TMS = morecantile.tms.get("WebMercatorQuad") 22 | 23 | 24 | def default_accessor(feature: Dict) -> str: 25 | """Return specific feature identifier.""" 26 | return feature["properties"]["path"] 27 | 28 | 29 | def default_filter( 30 | tile: morecantile.Tile, 31 | dataset: Sequence[Dict], 32 | geoms: Sequence[polygons], 33 | minimum_tile_cover: float = None, 34 | tile_cover_sort: bool = False, 35 | maximum_items_per_tile: Optional[int] = None, 36 | ) -> List: 37 | """Filter and/or sort dataset per intersection coverage.""" 38 | indices = list(range(len(dataset))) 39 | 40 | if minimum_tile_cover or tile_cover_sort: 41 | tile_geom = polygons(WEB_MERCATOR_TMS.feature(tile)["geometry"]["coordinates"][0]) 42 | int_pcts = _intersect_percent(tile_geom, geoms) 43 | 44 | if minimum_tile_cover: 45 | if minimum_tile_cover > 1.0: 46 | raise MosaicError("`minimum_tile_cover` HAS TO be between 0 and 1.") 47 | 48 | indices = [ind for ind in indices if int_pcts[ind] > minimum_tile_cover] 49 | 50 | if tile_cover_sort: 51 | # https://stackoverflow.com/a/9764364 52 | _, indices = zip(*sorted(zip(int_pcts, indices), reverse=True)) # type: ignore 53 | 54 | if maximum_items_per_tile: 55 | indices = indices[:maximum_items_per_tile] 56 | 57 | return [dataset[ind] for ind in indices] 58 | 59 | 60 | class MosaicJSON(BaseModel, validate_assignment=True): 61 | """MosaicJSON model. 62 | 63 | Based on https://github.com/developmentseed/mosaicjson-spec 64 | 65 | """ 66 | 67 | mosaicjson: str 68 | name: Optional[str] = None 69 | description: Optional[str] = None 70 | version: str = "1.0.0" 71 | attribution: Optional[str] = None 72 | minzoom: int = Field(0, ge=0, le=30) 73 | maxzoom: int = Field(30, ge=0, le=30) 74 | quadkey_zoom: Optional[int] = None 75 | bounds: BBox = Field(default=(-180, -90, 180, 90)) 76 | center: Optional[Tuple[float, float, int]] = None 77 | tiles: Dict[str, List[str]] 78 | tilematrixset: Optional[morecantile.TileMatrixSet] = None 79 | asset_type: Optional[str] = None 80 | asset_prefix: Optional[str] = None 81 | data_type: Optional[str] = None 82 | colormap: Optional[Dict[int, Tuple[int, int, int, int]]] = None 83 | layers: Optional[Dict] = None 84 | 85 | @field_validator("tilematrixset") 86 | def parse_tms(cls, value) -> Optional[morecantile.TileMatrixSet]: 87 | """Parse TMS.""" 88 | if value: 89 | value = morecantile.TileMatrixSet.model_validate(value) 90 | assert value.is_quadtree, f"{value.id} TMS does not support quadtree." 91 | 92 | return value 93 | 94 | @model_validator(mode="after") 95 | def compute_center(self): 96 | """Compute center if it does not exist.""" 97 | bounds = self.bounds 98 | if not self.center: 99 | self.center = ( 100 | (bounds[0] + bounds[2]) / 2, 101 | (bounds[1] + bounds[3]) / 2, 102 | self.minzoom, 103 | ) 104 | return self 105 | 106 | def _increase_version(self): 107 | """Increment mosaicjson document version.""" 108 | version = list(map(int, self.version.split("."))) 109 | version[-1] += 1 110 | new_version = ".".join(map(str, version)) 111 | self.version = new_version 112 | 113 | @classmethod 114 | def _create_mosaic( 115 | cls, 116 | features: Sequence[Dict], 117 | minzoom: int, 118 | maxzoom: int, 119 | quadkey_zoom: Optional[int] = None, 120 | accessor: Callable[[Dict], str] = default_accessor, 121 | asset_filter: Callable = default_filter, 122 | version: str = "0.0.3", 123 | tilematrixset: Optional[morecantile.TileMatrixSet] = None, 124 | asset_type: Optional[str] = None, 125 | asset_prefix: Optional[str] = None, 126 | data_type: Optional[str] = None, 127 | colormap: Optional[Dict[int, Tuple[int, int, int, int]]] = None, 128 | layers: Optional[Dict] = None, 129 | quiet: bool = True, 130 | **kwargs, 131 | ): 132 | """Create mosaic definition content. 133 | 134 | Attributes: 135 | features (list): List of GeoJSON features. 136 | minzoom (int): Force mosaic min-zoom. 137 | maxzoom (int): Force mosaic max-zoom. 138 | quadkey_zoom (int): Force mosaic quadkey zoom (optional). 139 | accessor (callable): Function called on each feature to get its identifier (default is feature["properties"]["path"]). 140 | asset_filter (callable): Function to filter features. 141 | version (str): mosaicJSON definition version (default: 0.0.2). 142 | quiet (bool): Mask processing steps (default is True). 143 | kwargs (any): Options forwarded to `asset_filter` 144 | 145 | Returns: 146 | mosaic_definition (MosaicJSON): Mosaic definition. 147 | 148 | Examples: 149 | >>> MosaicJSON._create_mosaic([], 12, 14) 150 | 151 | """ 152 | tms = tilematrixset or WEB_MERCATOR_TMS 153 | 154 | assert tms.is_quadtree, f"{tms.id} TMS does not support quadtree." 155 | 156 | quadkey_zoom = quadkey_zoom or minzoom 157 | 158 | if not quiet: 159 | click.echo(f"Get quadkey list for zoom: {quadkey_zoom}", err=True) 160 | 161 | dataset_geoms = polygons( 162 | [linearrings(feat["geometry"]["coordinates"][0]) for feat in features] 163 | ) 164 | 165 | bounds = tuple(total_bounds(dataset_geoms)) 166 | 167 | burntiles = burnTiles(tms=tms) 168 | tiles = [morecantile.Tile(*t) for t in burntiles.burn(features, quadkey_zoom)] 169 | 170 | mosaic_definition: Dict[str, Any] = { 171 | "mosaicjson": version, 172 | "minzoom": minzoom, 173 | "maxzoom": maxzoom, 174 | "quadkey_zoom": quadkey_zoom, 175 | "bounds": bounds, 176 | "center": ( 177 | (bounds[0] + bounds[2]) / 2, 178 | (bounds[1] + bounds[3]) / 2, 179 | minzoom, 180 | ), 181 | "tiles": {}, 182 | "version": "1.0.0", 183 | } 184 | 185 | mosaic_003 = { 186 | "tilematrixset": tilematrixset, 187 | "asset_type": asset_type, 188 | "asset_prefix": asset_prefix, 189 | "data_type": data_type, 190 | "colormap": colormap, 191 | "layers": layers, 192 | } 193 | for k, v in mosaic_003.items(): 194 | if v is not None: 195 | mosaic_definition[k] = v 196 | 197 | if not quiet: 198 | click.echo("Feed Quadkey index", err=True) 199 | 200 | # Create tree and find assets that overlap each tile 201 | tree = STRtree(dataset_geoms) 202 | 203 | with ExitStack() as ctx: 204 | fout = ctx.enter_context(open(os.devnull, "w")) if quiet else sys.stderr 205 | with click.progressbar( # type: ignore 206 | tiles, file=fout, show_percent=True, label="Iterate over quadkeys" 207 | ) as bar: 208 | for tile in bar: 209 | quadkey = tms.quadkey(tile) 210 | tile_geom = polygons( 211 | tms.feature(tile, geographic_crs=tms.geographic_crs)["geometry"][ 212 | "coordinates" 213 | ][0] 214 | ) 215 | 216 | # Find intersections from rtree 217 | intersections_idx = sorted( 218 | tree.query(tile_geom, predicate="intersects") 219 | ) 220 | if len(intersections_idx) == 0: 221 | continue 222 | 223 | intersect_dataset, intersect_geoms = zip( 224 | *[ 225 | (features[idx], dataset_geoms[idx]) 226 | for idx in intersections_idx 227 | ] 228 | ) 229 | 230 | dataset = asset_filter( 231 | tile, intersect_dataset, intersect_geoms, **kwargs 232 | ) 233 | 234 | if dataset: 235 | assets = [accessor(f) for f in dataset] 236 | if asset_prefix: 237 | assets = [ 238 | re.sub(rf"^{asset_prefix}", "", asset) 239 | if asset.startswith(asset_prefix) 240 | else asset 241 | for asset in assets 242 | ] 243 | 244 | mosaic_definition["tiles"][quadkey] = assets 245 | 246 | return cls(**mosaic_definition) 247 | 248 | @classmethod 249 | def from_urls( 250 | cls, 251 | urls: Sequence[str], 252 | minzoom: Optional[int] = None, 253 | maxzoom: Optional[int] = None, 254 | max_threads: int = 20, 255 | tilematrixset: Optional[morecantile.TileMatrixSet] = None, 256 | quiet: bool = True, 257 | **kwargs, 258 | ): 259 | """Create mosaicjson from COG urls. 260 | 261 | Attributes: 262 | urls (list): List of COGs. 263 | tilematrixset: (morecantile.TileMatrixSet), optional (default: "WebMercatorQuad") 264 | minzoom (int): Force mosaic min-zoom. 265 | maxzoom (int): Force mosaic max-zoom. 266 | max_threads (int): Max threads to use (default: 20). 267 | quiet (bool): Mask processing steps (default is True). 268 | kwargs (any): Options forwarded to `MosaicJSON._create_mosaic` 269 | 270 | Returns: 271 | mosaic_definition (MosaicJSON): Mosaic definition. 272 | 273 | 274 | Raises: 275 | Exception: If COGs don't have the same datatype 276 | 277 | Examples: 278 | >>> MosaicJSON.from_urls(["1.tif", "2.tif"]) 279 | 280 | """ 281 | features = get_footprints( 282 | urls, max_threads=max_threads, tms=tilematrixset, quiet=quiet 283 | ) 284 | 285 | if minzoom is None: 286 | data_minzoom = {feat["properties"]["minzoom"] for feat in features} 287 | if len(data_minzoom) > 1: 288 | warnings.warn( 289 | "Multiple MinZoom, Assets different minzoom values", UserWarning 290 | ) 291 | 292 | minzoom = max(data_minzoom) 293 | 294 | if maxzoom is None: 295 | data_maxzoom = {feat["properties"]["maxzoom"] for feat in features} 296 | if len(data_maxzoom) > 1: 297 | warnings.warn( 298 | "Multiple MaxZoom, Assets have multiple resolution values", 299 | UserWarning, 300 | ) 301 | 302 | maxzoom = max(data_maxzoom) 303 | 304 | datatype = {feat["properties"]["datatype"] for feat in features} 305 | if len(datatype) > 1: 306 | raise MultipleDataTypeError("Dataset should have the same data type") 307 | 308 | return cls._create_mosaic( 309 | features, 310 | minzoom=minzoom, 311 | maxzoom=maxzoom, 312 | tilematrixset=tilematrixset, 313 | quiet=quiet, 314 | **kwargs, 315 | ) 316 | 317 | @classmethod 318 | def from_features( 319 | cls, features: Sequence[Dict], minzoom: int, maxzoom: int, **kwargs 320 | ): 321 | """Create mosaicjson from a set of GeoJSON Features. 322 | 323 | Attributes: 324 | features (list): List of GeoJSON features. 325 | minzoom (int): Force mosaic min-zoom. 326 | maxzoom (int): Force mosaic max-zoom. 327 | kwargs (any): Options forwarded to `MosaicJSON._create_mosaic` 328 | 329 | Returns: 330 | mosaic_definition (MosaicJSON): Mosaic definition. 331 | 332 | Examples: 333 | >>> MosaicJSON.from_features([{}, {}], 12, 14) 334 | 335 | """ 336 | return cls._create_mosaic(features, minzoom, maxzoom, **kwargs) 337 | -------------------------------------------------------------------------------- /docs/src/examples/Create_a_Dynamic_StacBackend.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Create a Dynamic STAC backend\n", 8 | "\n", 9 | "By default cogeo-mosaic backends were meant to handle writing and reading mosaicjson either from a file or from a database.\n", 10 | "\n", 11 | "While this is fine for most use cases, some users could want something more `dynamic`. In this Notebook we will show how to create a Dynamic mosaic backend based on STAC api.\n" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "\n", 19 | "## Requirements\n", 20 | "\n", 21 | "To be able to run this notebook you'll need the following requirements:\n", 22 | "- cogeo-mosaic" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 1, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "# Uncomment this line if you need to install the dependencies\n", 32 | "# !pip install cogeo-mosaic" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 1, 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "from typing import Any, Dict, Tuple, Type, Optional, List\n", 42 | "\n", 43 | "import attr\n", 44 | "import morecantile\n", 45 | "from rasterio.crs import CRS\n", 46 | "from rio_tiler.constants import WEB_MERCATOR_TMS, WGS84_CRS\n", 47 | "from rio_tiler.io import BaseReader\n", 48 | "from rio_tiler.io import STACReader\n", 49 | "\n", 50 | "from cogeo_mosaic.backends.base import MosaicJSONBackend\n", 51 | "from cogeo_mosaic.backends.stac import _fetch, default_stac_accessor\n", 52 | "from cogeo_mosaic.mosaic import MosaicJSON\n", 53 | "\n", 54 | "\n", 55 | "@attr.s\n", 56 | "class DynamicStacBackend(MosaicJSONBackend):\n", 57 | " \"\"\"Like a STAC backend but dynamic\"\"\"\n", 58 | "\n", 59 | " # input should be the STAC-API url\n", 60 | " input: str = attr.ib()\n", 61 | "\n", 62 | " # Addition required attribute (STAC Query)\n", 63 | " query: Dict = attr.ib(factory=dict)\n", 64 | "\n", 65 | " tms: morecantile.TileMatrixSet = attr.ib(default=WEB_MERCATOR_TMS)\n", 66 | "\n", 67 | " minzoom: int = attr.ib(default=None)\n", 68 | " maxzoom: int = attr.ib(default=None)\n", 69 | "\n", 70 | " reader: Type[BaseReader] = attr.ib(default=STACReader)\n", 71 | " reader_options: Dict = attr.ib(factory=dict)\n", 72 | "\n", 73 | " bounds: Tuple[float, float, float, float] = attr.ib(\n", 74 | " default=(-180, -90, 180, 90)\n", 75 | " )\n", 76 | " crs: CRS = attr.ib(default=WGS84_CRS)\n", 77 | "\n", 78 | " # STAC API related options\n", 79 | " # max_items | next_link_key | limit\n", 80 | " stac_api_options: Dict = attr.ib(factory=dict)\n", 81 | "\n", 82 | " # The reader is read-only, we can't pass mosaic_def to the init method\n", 83 | " mosaic_def: MosaicJSON = attr.ib(init=False)\n", 84 | "\n", 85 | " _backend_name = \"DynamicSTAC\"\n", 86 | "\n", 87 | " def __attrs_post_init__(self):\n", 88 | " \"\"\"Post Init.\"\"\"\n", 89 | " self.minzoom = self.minzoom if self.minzoom is not None else self.tms.minzoom\n", 90 | " self.maxzoom = self.maxzoom if self.maxzoom is not None else self.tms.maxzoom\n", 91 | " \n", 92 | " # Construct a FAKE/Empty mosaicJSON\n", 93 | " # mosaic_def has to be defined. As we do for the DynamoDB and SQLite backend\n", 94 | " self.mosaic_def = MosaicJSON(\n", 95 | " mosaicjson=\"0.0.3\",\n", 96 | " name=\"it's fake but it's ok\",\n", 97 | " bounds=self.bounds,\n", 98 | " minzoom=self.minzoom,\n", 99 | " maxzoom=self.maxzoom,\n", 100 | " tiles={} # we set `tiles` to an empty list.\n", 101 | " )\n", 102 | "\n", 103 | " def write(self, overwrite: bool = True):\n", 104 | " \"\"\"This method is not used but is required by the abstract class.\"\"\"\n", 105 | " raise NotImplementedError\n", 106 | "\n", 107 | " def update(self):\n", 108 | " \"\"\"We overwrite the default method.\"\"\"\n", 109 | " raise NotImplementedError\n", 110 | "\n", 111 | " def _read(self) -> MosaicJSON:\n", 112 | " \"\"\"This method is not used but is required by the abstract class.\"\"\"\n", 113 | " pass\n", 114 | "\n", 115 | " def assets_for_tile(self, x: int, y: int, z: int) -> List[str]:\n", 116 | " \"\"\"Retrieve assets for tile.\"\"\"\n", 117 | " bounds = self.tms.bounds(x, y, z)\n", 118 | " geom = {\n", 119 | " \"type\": \"Polygon\",\n", 120 | " \"coordinates\": [\n", 121 | " [\n", 122 | " [bounds[0], bounds[3]],\n", 123 | " [bounds[0], bounds[1]],\n", 124 | " [bounds[2], bounds[1]],\n", 125 | " [bounds[2], bounds[3]],\n", 126 | " [bounds[0], bounds[3]],\n", 127 | " ]\n", 128 | " ],\n", 129 | " }\n", 130 | " return self.get_assets(geom)\n", 131 | "\n", 132 | " def assets_for_point(self, lng: float, lat: float, **kwargs: Any) -> List[str]:\n", 133 | " \"\"\"Retrieve assets for point.\n", 134 | "\n", 135 | " Note: some API only accept Polygon.\n", 136 | " \"\"\"\n", 137 | " EPSILON = 1e-14\n", 138 | " geom = {\n", 139 | " \"type\": \"Polygon\",\n", 140 | " \"coordinates\": [\n", 141 | " [\n", 142 | " [lng - EPSILON, lat + EPSILON],\n", 143 | " [lng - EPSILON, lat - EPSILON],\n", 144 | " [lng + EPSILON, lat - EPSILON],\n", 145 | " [lng + EPSILON, lat + EPSILON],\n", 146 | " [lng - EPSILON, lat + EPSILON],\n", 147 | " ]\n", 148 | " ],\n", 149 | " }\n", 150 | " return self.get_assets(geom)\n", 151 | "\n", 152 | " def get_assets(self, geom) -> List[str]:\n", 153 | " \"\"\"Send query to the STAC-API and retrieve assets.\"\"\"\n", 154 | " query = self.query.copy()\n", 155 | " query[\"intersects\"] = geom\n", 156 | "\n", 157 | " features = _fetch(\n", 158 | " self.input,\n", 159 | " query,\n", 160 | " **self.stac_api_options,\n", 161 | " )\n", 162 | " return [default_stac_accessor(f) for f in features]\n", 163 | "\n", 164 | " @property\n", 165 | " def _quadkeys(self) -> List[str]:\n", 166 | " return []\n" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 2, 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "## Base Query for sat-api\n", 176 | "# - limit of 5 items per page (we will stop at page 1)\n", 177 | "# - less than 25% of cloud\n", 178 | "# - more than 75% of data coverage\n", 179 | "# - `sentinel-s2-l2a-cogs` collection\n", 180 | "query = {\n", 181 | " \"collections\": [\"sentinel-s2-l2a-cogs\"],\n", 182 | " \"limit\": 5,\n", 183 | " \"query\": {\n", 184 | " \"eo:cloud_cover\": {\n", 185 | " \"lt\": 25\n", 186 | " },\n", 187 | " \"sentinel:data_coverage\": {\n", 188 | " \"gt\": 75\n", 189 | " }\n", 190 | " },\n", 191 | " \"fields\": {\n", 192 | " 'include': ['id'],\n", 193 | " 'exclude': ['assets', 'geometry']\n", 194 | " }\n", 195 | "}" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 3, 201 | "metadata": {}, 202 | "outputs": [ 203 | { 204 | "name": "stdout", 205 | "output_type": "stream", 206 | "text": [ 207 | "bounds=(-180.0, -90.0, 180.0, 90.0) crs='http://www.opengis.net/def/crs/EPSG/0/4326' name=\"it's fake but it's ok\" quadkeys=[] mosaic_tilematrixset=\" MosaicJSON: # type: ignore 95 | """Get Mosaic definition info.""" 96 | meta = self._fetch_dynamodb(self._metadata_quadkey) 97 | if not meta: 98 | raise MosaicNotFoundError( 99 | f"Mosaic {self.mosaic_name} not found in table {self.table_name}" 100 | ) 101 | 102 | # Numeric values are loaded from DynamoDB as Decimal types 103 | # Convert maxzoom, minzoom, quadkey_zoom to int 104 | for key in ["minzoom", "maxzoom", "quadkey_zoom"]: 105 | if meta.get(key): 106 | meta[key] = int(meta[key]) 107 | 108 | # Convert bounds, center to float 109 | for key in ["bounds", "center"]: 110 | if meta.get(key): 111 | meta[key] = list(map(float, meta[key])) 112 | 113 | # Create pydantic class 114 | # For now, a tiles key must exist 115 | meta["tiles"] = {} 116 | return MosaicJSON(**meta) 117 | 118 | def write(self, overwrite: bool = False, **kwargs: Any): 119 | """Write mosaicjson document to AWS DynamoDB. 120 | 121 | Args: 122 | overwrite (bool): delete old mosaic items inthe Table. 123 | **kwargs (any): Options forwarded to `dynamodb.create_table` 124 | 125 | Raises: 126 | MosaicExistsError: If mosaic already exists in the Table. 127 | 128 | """ 129 | if not self._table_exists(): 130 | self._create_table(**kwargs) 131 | 132 | if self._mosaic_exists(): 133 | if not overwrite: 134 | raise MosaicExistsError( 135 | f"Mosaic already exists in {self.table_name}, use `overwrite=True`." 136 | ) 137 | self.delete() 138 | 139 | items: List[Dict[str, Any]] = [] 140 | 141 | # Create Metadata item 142 | # Note: `parse_float=Decimal` is required because DynamoDB requires all numbers to be 143 | # in Decimal type (ref: https://blog.ruanbekker.com/blog/2019/02/05/convert-float-to-decimal-data-types-for-boto3-dynamodb-using-python/) 144 | meta = json.loads( 145 | self.mosaic_def.model_dump_json(exclude={"tiles"}), parse_float=Decimal 146 | ) 147 | items.append( 148 | {"quadkey": self._metadata_quadkey, "mosaicId": self.mosaic_name, **meta} 149 | ) 150 | 151 | # Create Tile items 152 | for quadkey, assets in self.mosaic_def.tiles.items(): 153 | items.append( 154 | {"mosaicId": self.mosaic_name, "quadkey": quadkey, "assets": assets} 155 | ) 156 | 157 | self._write_items(items) 158 | 159 | def update( 160 | self, 161 | features: Sequence[Dict], 162 | add_first: bool = True, 163 | quiet: bool = False, 164 | **kwargs, 165 | ): 166 | """Update existing MosaicJSON on backend.""" 167 | logger.debug(f"Updating {self.mosaic_name}...") 168 | 169 | new_mosaic = MosaicJSON.from_features( 170 | features, 171 | self.mosaic_def.minzoom, 172 | self.mosaic_def.maxzoom, 173 | quadkey_zoom=self.quadkey_zoom, 174 | tilematrixset=self.mosaic_def.tilematrixset, 175 | quiet=quiet, 176 | **kwargs, 177 | ) 178 | 179 | bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds) 180 | 181 | if self.mosaic_def.mosaicjson != new_mosaic.mosaicjson: 182 | warnings.warn( 183 | f"Updating `mosaicjson` version from {self.mosaic_def.mosaicjson} to {new_mosaic.mosaicjson}" 184 | ) 185 | self.mosaic_def.mosaicjson = new_mosaic.mosaicjson 186 | 187 | self.mosaic_def._increase_version() 188 | self.mosaic_def.bounds = bounds 189 | self.mosaic_def.center = ( 190 | (bounds[0] + bounds[2]) / 2, 191 | (bounds[1] + bounds[3]) / 2, 192 | self.mosaic_def.minzoom, 193 | ) 194 | self.bounds = bounds 195 | 196 | items: List[Dict[str, Any]] = [] 197 | 198 | # Create Metadata item 199 | # Note: `parse_float=Decimal` is required because DynamoDB requires all numbers to be 200 | # in Decimal type (ref: https://blog.ruanbekker.com/blog/2019/02/05/convert-float-to-decimal-data-types-for-boto3-dynamodb-using-python/) 201 | meta = json.loads( 202 | self.mosaic_def.model_dump_json(exclude={"tiles"}), parse_float=Decimal 203 | ) 204 | items.append( 205 | {"quadkey": self._metadata_quadkey, "mosaicId": self.mosaic_name, **meta} 206 | ) 207 | 208 | # Create Tile items 209 | for quadkey, new_assets in new_mosaic.tiles.items(): 210 | mosaic_tms = self.mosaic_def.tilematrixset or WEB_MERCATOR_TMS 211 | tile = mosaic_tms.quadkey_to_tile(quadkey) 212 | assets = self.assets_for_tile(*tile) 213 | assets = [*new_assets, *assets] if add_first else [*assets, *new_assets] 214 | items.append( 215 | {"mosaicId": self.mosaic_name, "quadkey": quadkey, "assets": assets} 216 | ) 217 | 218 | self._write_items(items) 219 | 220 | @cached( # type: ignore 221 | TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl), 222 | key=lambda self, x, y, z: hashkey(self.input, x, y, z, self.mosaicid), 223 | lock=Lock(), 224 | ) 225 | def get_assets(self, x: int, y: int, z: int) -> List[str]: 226 | """Find assets.""" 227 | quadkeys = self.find_quadkeys(Tile(x=x, y=y, z=z), self.quadkey_zoom) 228 | assets = list( 229 | dict.fromkeys( 230 | itertools.chain.from_iterable( 231 | [self._fetch_dynamodb(qk).get("assets", []) for qk in quadkeys] 232 | ) 233 | ) 234 | ) 235 | if self.mosaic_def.asset_prefix: 236 | assets = [self.mosaic_def.asset_prefix + asset for asset in assets] 237 | 238 | return assets 239 | 240 | @property 241 | def _quadkeys(self) -> List[str]: 242 | """Return the list of quadkey tiles.""" 243 | resp = self.table.query( 244 | KeyConditionExpression=Key("mosaicId").eq(self.mosaic_name), 245 | ProjectionExpression="quadkey", 246 | ) 247 | return [ 248 | item["quadkey"] 249 | for item in resp["Items"] 250 | if item["quadkey"] != self._metadata_quadkey 251 | ] 252 | 253 | def _create_table(self, billing_mode: str = "PAY_PER_REQUEST", **kwargs: Any): 254 | """Create DynamoDB Table. 255 | 256 | Args: 257 | billing_mode (str): DynamoDB billing mode (default set to PER_REQUEST). 258 | **kwargs (any): Options forwarded to `dynamodb.create_table` 259 | 260 | """ 261 | logger.debug(f"Creating {self.table_name} Table.") 262 | 263 | # Define schema for primary key 264 | # Non-keys don't need a schema 265 | attr_defs = [ 266 | {"AttributeName": "mosaicId", "AttributeType": "S"}, 267 | {"AttributeName": "quadkey", "AttributeType": "S"}, 268 | ] 269 | key_schema = [ 270 | {"AttributeName": "mosaicId", "KeyType": "HASH"}, 271 | {"AttributeName": "quadkey", "KeyType": "RANGE"}, 272 | ] 273 | 274 | # Note: errors if table already exists 275 | try: 276 | self.client.create_table( 277 | AttributeDefinitions=attr_defs, 278 | TableName=self.table.table_name, 279 | KeySchema=key_schema, 280 | BillingMode=billing_mode, 281 | **kwargs, 282 | ) 283 | 284 | # If outside try/except block, could wait forever if unable to 285 | # create table 286 | self.table.wait_until_exists() 287 | except self.table.meta.client.exceptions.ResourceNotFoundException: 288 | warnings.warn("Unable to create table.") 289 | return 290 | 291 | def _write_items(self, items: List[Dict]): 292 | with self.table.batch_writer() as batch: 293 | with click.progressbar( 294 | items, 295 | length=len(items), 296 | show_percent=True, 297 | label=f"Uploading mosaic {self.table_name}:{self.mosaic_name} to DynamoDB", 298 | ) as progitems: 299 | for item in progitems: 300 | batch.put_item(item) 301 | 302 | def _fetch_dynamodb(self, quadkey: str) -> Dict: 303 | try: 304 | return self.table.get_item( 305 | Key={"mosaicId": self.mosaic_name, "quadkey": quadkey} 306 | ).get("Item", {}) 307 | except ClientError as e: 308 | status_code = e.response["ResponseMetadata"]["HTTPStatusCode"] 309 | exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError) 310 | raise exc(e.response["Error"]["Message"]) from e 311 | 312 | def _table_exists(self) -> bool: 313 | """Check if the Table already exists.""" 314 | try: 315 | _ = self.table.table_status 316 | return True 317 | except self.table.meta.client.exceptions.ResourceNotFoundException: 318 | return False 319 | 320 | def _mosaic_exists(self) -> bool: 321 | """Check if the mosaic already exists in the Table.""" 322 | item = self.table.get_item( 323 | Key={"mosaicId": self.mosaic_name, "quadkey": self._metadata_quadkey} 324 | ).get("Item", {}) 325 | return bool(item) 326 | 327 | def delete(self): 328 | """Delete all items for a specific mosaic in the dynamoDB Table.""" 329 | logger.debug(f"Deleting all items for mosaic {self.mosaic_name}...") 330 | 331 | quadkey_list = self._quadkeys + [self._metadata_quadkey] 332 | with self.table.batch_writer() as batch_writer: 333 | for item in quadkey_list: 334 | batch_writer.delete_item( 335 | Key={"mosaicId": self.mosaic_name, "quadkey": item} 336 | ) 337 | -------------------------------------------------------------------------------- /docs/src/examples/Create_a_Dynamic_RtreeBackend.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Create a Dynamic RTree backend\n", 8 | "\n", 9 | "By default cogeo-mosaic backends were meant to handle writing and reading mosaicjson either from a file or from a database.\n", 10 | "\n", 11 | "While this is fine for most use cases, some users could want something more `dynamic`. In this Notebook we will show how to create a Dynamic mosaic backend based on RTree (https://rtree.readthedocs.io/en/latest/tutorial.html#using-rtree-as-a-cheapo-spatial-database).\n" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "\n", 19 | "## Requirements\n", 20 | "\n", 21 | "To be able to run this notebook you'll need the following requirements:\n", 22 | "- cogeo-mosaic\n", 23 | "- rtree\n", 24 | "- shapely\n", 25 | "- tqdm" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 2, 31 | "metadata": { 32 | "scrolled": true 33 | }, 34 | "outputs": [], 35 | "source": [ 36 | "# Uncomment this line if you need to install the dependencies\n", 37 | "# !pip install cogeo-mosaic rtree shapely tqdm" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 3, 43 | "metadata": { 44 | "scrolled": true 45 | }, 46 | "outputs": [], 47 | "source": [ 48 | "import httpx\n", 49 | "import pickle\n", 50 | "\n", 51 | "from tqdm.notebook import tqdm\n", 52 | "\n", 53 | "from rtree import index" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "## 1. Create the rtree Index\n", 61 | "\n", 62 | "Ref: https://azure.microsoft.com/en-us/services/open-datasets/catalog/naip/?tab=data-access#AzureNotebooks\n", 63 | "\n", 64 | "Azure is hosting a RTree index (tile_index) and a binary file with all the Naip geometry (tiles.p)\n", 65 | "\n", 66 | "binary: https://naipeuwest.blob.core.windows.net/naip-index/rtree/tiles.p\n", 67 | "Rtree: https://naipeuwest.blob.core.windows.net/naip-index/rtree/tile_index.dat and https://naipeuwest.blob.core.windows.net/naip-index/rtree/tile_index.idx\n", 68 | "\n", 69 | "Sadly the Rtree contains only the Indexes, which then has to be used to retrieve the path and geometry in `tiles.p`\n", 70 | "\n", 71 | "For this Demo we need to store the information directly in the Rtree object \n" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 4, 77 | "metadata": { 78 | "scrolled": true 79 | }, 80 | "outputs": [ 81 | { 82 | "name": "stderr", 83 | "output_type": "stream", 84 | "text": [ 85 | "/var/folders/l9/xz620xfs2s5_38f8ybq4m59w0000gn/T/ipykernel_31305/2877602215.py:11: UserWarning: Unpickling a shapely <2.0 geometry object. Please save the pickle again as this compatibility may be removed in a future version of shapely.\n", 86 | " tile_index = pickle.load(f)\n" 87 | ] 88 | } 89 | ], 90 | "source": [ 91 | "# Download geometry file\n", 92 | "url = \"https://naipeuwest.blob.core.windows.net/naip-index/rtree/tiles.p\"\n", 93 | "with httpx.stream(\"GET\", url) as r:\n", 94 | " r.raise_for_status()\n", 95 | " with open(\"tiles.p\", \"wb\") as f:\n", 96 | " for chunk in r.iter_bytes(chunk_size=8192):\n", 97 | " f.write(chunk)\n", 98 | "\n", 99 | "# Load tile index and create rtree index\n", 100 | "with open(\"tiles.p\", \"rb\") as f:\n", 101 | " tile_index = pickle.load(f)" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 5, 107 | "metadata": { 108 | "scrolled": true 109 | }, 110 | "outputs": [ 111 | { 112 | "data": { 113 | "application/vnd.jupyter.widget-view+json": { 114 | "model_id": "4952ab6b82cc4f6a80c8f99142c31800", 115 | "version_major": 2, 116 | "version_minor": 0 117 | }, 118 | "text/plain": [ 119 | " 0%| | 0/1018253 [00:00 MosaicJSON:\n", 210 | " \"\"\"This method is not used but is required by the abstract class.\"\"\"\n", 211 | " pass\n", 212 | "\n", 213 | " def assets_for_tile(self, x: int, y: int, z: int, **kwargs) -> List[str]:\n", 214 | " \"\"\"Retrieve assets for tile.\"\"\"\n", 215 | " bbox = self.tms.bounds(x, y, z)\n", 216 | " return self.get_assets(bbox)\n", 217 | "\n", 218 | " def assets_for_point(self, lng: float, lat: float, **kwargs) -> List[str]:\n", 219 | " \"\"\"Retrieve assets for point.\"\"\"\n", 220 | " EPSILON = 1e-14\n", 221 | " bbox = (lng - EPSILON, lat - EPSILON, lng + EPSILON, lat + EPSILON)\n", 222 | " return self.get_assets(bbox)\n", 223 | "\n", 224 | " def get_assets(self, bbox) -> List[str]:\n", 225 | " \"\"\"Find assets.\"\"\"\n", 226 | " assets = [n.object for n in self.index.intersection(bbox, objects=True)]\n", 227 | " return self.asset_filter(assets)\n", 228 | "\n", 229 | " @property\n", 230 | " def _quadkeys(self) -> List[str]:\n", 231 | " return []\n" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 10, 237 | "metadata": { 238 | "scrolled": true 239 | }, 240 | "outputs": [ 241 | { 242 | "name": "stdout", 243 | "output_type": "stream", 244 | "text": [ 245 | "['https://naipeuwest.blob.core.windows.net/naip/v002/md/2013/md_100cm_2013/38077/m_3807724_nw_18_1_20130924.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/md/2011/md_100cm_2011/38077/m_3807724_nw_18_1_20110629.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/md/2015/md_100cm_2015/38077/m_3807724_nw_18_1_20150814.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/va/2011/va_100cm_2011/38077/m_3807724_nw_18_1_20110530.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/md/2017/md_100cm_2017/38077/m_3807724_nw_18_1_20170716.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/md/2018/md_060cm_2018/38077/m_3807724_nw_18_060_20181025.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/md/2018/md_060cm_2018/38077/m_3807724_ne_18_060_20181019.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/md/2015/md_100cm_2015/38077/m_3807724_ne_18_1_20150814.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/md/2011/md_100cm_2011/38077/m_3807724_ne_18_1_20110629.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/md/2013/md_100cm_2013/38077/m_3807724_ne_18_1_20130924.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/va/2011/va_100cm_2011/38077/m_3807724_ne_18_1_20110530.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/md/2017/md_100cm_2017/38077/m_3807724_ne_18_1_20170716.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/va/2012/va_100cm_2012/38077/m_3807724_ne_18_1_20120511.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/va/2012/va_100cm_2012/38077/m_3807724_nw_18_1_20120511.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/va/2014/va_100cm_2014/38077/m_3807724_nw_18_1_20140927.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/va/2014/va_100cm_2014/38077/m_3807724_ne_18_1_20140927.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/va/2016/va_100cm_2016/38077/m_3807724_ne_18_1_20160718.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/va/2016/va_100cm_2016/38077/m_3807724_nw_18_1_20160718.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/va/2018/va_060cm_2018/38077/m_3807724_ne_18_060_20181019.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/va/2018/va_060cm_2018/38077/m_3807724_nw_18_060_20181025.tif']\n" 246 | ] 247 | } 248 | ], 249 | "source": [ 250 | "# Get assets for a Tile requests\n", 251 | "with DynamicRtreeBackend(\"naip\") as mosaic:\n", 252 | " print(mosaic.assets_for_tile(4684, 6278, 14))" 253 | ] 254 | }, 255 | { 256 | "cell_type": "markdown", 257 | "metadata": {}, 258 | "source": [ 259 | "The naip dataset has couple overlapping years, to create an optimized mosaic we need to filter the assets.\n", 260 | "\n", 261 | "Here is an example of filter function which takes the latest data and highest resolution first." 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": 11, 267 | "metadata": { 268 | "scrolled": true 269 | }, 270 | "outputs": [], 271 | "source": [ 272 | "import pathlib\n", 273 | "\n", 274 | "def latest_naip_asset(assets: List[str]) -> List[str]:\n", 275 | "\n", 276 | " def get_info(asset) -> Dict:\n", 277 | " parts = pathlib.Path(asset).parts\n", 278 | " capture_date = parts[-1].split(\"_\")[-1].rstrip(\".tif\")\n", 279 | " resolution = int(parts[-3].split(\"_\")[1].rstrip(\"cm\"))\n", 280 | " fname_parts = parts[-1].split(\"_\")\n", 281 | " quadrangle = f\"{fname_parts[1]}_{fname_parts[2]}_{fname_parts[3]}\"\n", 282 | "\n", 283 | " return {\n", 284 | " \"path\": asset,\n", 285 | " \"capture_date\": capture_date,\n", 286 | " \"quadrangle\": quadrangle,\n", 287 | " \"resolution\": resolution\n", 288 | " }\n", 289 | "\n", 290 | " asset_info = [get_info(f) for f in assets]\n", 291 | "\n", 292 | " # Sort by resolution and by dates\n", 293 | " asset_info = sorted(\n", 294 | " asset_info, key=lambda item: (item[\"capture_date\"], -item[\"resolution\"]),\n", 295 | " reverse=True\n", 296 | " )\n", 297 | "\n", 298 | " quad = []\n", 299 | " out_dataset = []\n", 300 | " for d in asset_info:\n", 301 | " q = d[\"quadrangle\"]\n", 302 | " if q not in quad:\n", 303 | " out_dataset.append(d[\"path\"])\n", 304 | " quad.append(q)\n", 305 | "\n", 306 | " return out_dataset\n" 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "execution_count": 12, 312 | "metadata": { 313 | "scrolled": true 314 | }, 315 | "outputs": [ 316 | { 317 | "name": "stdout", 318 | "output_type": "stream", 319 | "text": [ 320 | "['https://naipeuwest.blob.core.windows.net/naip/v002/md/2018/md_060cm_2018/38077/m_3807724_nw_18_060_20181025.tif', 'https://naipeuwest.blob.core.windows.net/naip/v002/md/2018/md_060cm_2018/38077/m_3807724_ne_18_060_20181019.tif']\n" 321 | ] 322 | } 323 | ], 324 | "source": [ 325 | "with DynamicRtreeBackend(\"naip\", asset_filter=latest_naip_asset) as mosaic:\n", 326 | " print(mosaic.assets_for_tile(4684, 6278, 14))" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": null, 332 | "metadata": {}, 333 | "outputs": [], 334 | "source": [] 335 | } 336 | ], 337 | "metadata": { 338 | "kernelspec": { 339 | "display_name": "python3.13 (3.13.7)", 340 | "language": "python", 341 | "name": "python3" 342 | }, 343 | "language_info": { 344 | "codemirror_mode": { 345 | "name": "ipython", 346 | "version": 3 347 | }, 348 | "file_extension": ".py", 349 | "mimetype": "text/x-python", 350 | "name": "python", 351 | "nbconvert_exporter": "python", 352 | "pygments_lexer": "ipython3", 353 | "version": "3.13.7" 354 | } 355 | }, 356 | "nbformat": 4, 357 | "nbformat_minor": 2 358 | } 359 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | ## Unreleased 2 | 3 | ## 9.0.2 (2025-12-15) 4 | 5 | * add support for python 3.14 6 | 7 | ## 9.0.1 (2025-11-25) 8 | 9 | * update morecantile requirements to `>=7.0` 10 | 11 | ## 9.0.0 (2025-11-21) 12 | 13 | * update rio-tiler requirement to `>=8.0,<9.0` 14 | * rename `cogeo_mosaic.backends.base.BaseBackend` to `cogeo_mosaic.backends.base.MosaicJSONBackend` **breaking change** 15 | * `cogeo_mosaic.backends.base.MosaicJSONBackend` subclass `rio_tiler.mosaic.backend.BaseBackend` 16 | * remove `.center` property from `MosaicJSONBackend` **breaking change** 17 | * `reverse` option should be passed via `search_options` **breaking change** 18 | 19 | ```python 20 | # before 21 | with MemoryBackend(mosaic_def=...) as mosaic: 22 | ptsR = mosaic.point(-73, 45, reverse=True) 23 | 24 | # now 25 | with MemoryBackend(mosaic_def=...) as mosaic: 26 | ptsR = mosaic.point(-73, 45, search_options={"reverse": True}) 27 | ``` 28 | 29 | ## 8.2.0 (2025-05-06) 30 | 31 | * allow `kwargs` to be forwarded to backend's `get_assets` method from `assets_for_tile`, `assets_for_point` and `assets_for_bbox` methods 32 | 33 | ## 8.1.0 (2025-02-13) 34 | 35 | * add `lock=Lock()` to cachetools `cached` instance to avoid thread corruption 36 | * remove python 3.8 support 37 | * add python 3.13 support 38 | 39 | ## 8.0.0 (2024-10-21) 40 | 41 | * remove deprecated methods 42 | 43 | * update morecantile requirement to `>=5.0,<7.0` 44 | 45 | * update rio-tiler requirement to `>=7.0,<8.0` 46 | 47 | * update `Info` model 48 | 49 | ```python 50 | # before 51 | class Info(BaseModel): 52 | bounds: BBox = Field(default=(-180, -90, 180, 90)) 53 | center: Optional[Tuple[float, float, int]] = None 54 | minzoom: int = Field(0, ge=0, le=30) 55 | maxzoom: int = Field(30, ge=0, le=30) 56 | name: Optional[str] = None 57 | quadkeys: List[str] = [] 58 | tilematrixset: Optional[str] = None 59 | 60 | # now 61 | class Info(BaseModel): 62 | bounds: BBox = Field(default=(-180, -90, 180, 90)) 63 | crs: str 64 | center: Optional[Tuple[float, float, int]] = None 65 | name: Optional[str] = None 66 | quadkeys: List[str] = [] 67 | mosaic_tilematrixset: Optional[str] = None 68 | mosaic_minzoom: int = Field(0, ge=0, le=30) 69 | mosaic_maxzoom: int = Field(30, ge=0, le=30) 70 | ``` 71 | 72 | ## 7.2.0 (2024-10-04) 73 | 74 | * update BaseBackend to use a default coord_crs from the tms (author @AndrewAnnex, https://github.com/developmentseed/cogeo-mosaic/pull/234) 75 | * add python 3.12 support 76 | * Add tms parameter to cli for MosaicJSON (co-author @AndrewAnnex, https://github.com/developmentseed/cogeo-mosaic/pull/233) 77 | 78 | ## 7.1.0 (2023-12-06) 79 | 80 | * Automatically remove/add `asset_prefix` in Mosaic Backends 81 | 82 | ## 7.0.1 (2023-10-17) 83 | 84 | * add `py.typed` file (https://peps.python.org/pep-0561) 85 | 86 | ## 7.0.0 (2023-07-26) 87 | 88 | * update `morecantile` requirement to `>=5.0,<6.0` 89 | * update `rio-tiler` requirement to `>=6.0,<7.0` 90 | * update `pydantic` requirement to `~=2.0` 91 | 92 | ## 6.2.0 (2023-07-11) 93 | 94 | * add `coord_crs` to `MosaicBackend.point()` method 95 | 96 | ## 6.1.0 (2023-07-11) 97 | 98 | * add `tilematrixset` in `MosaicBackend.info()` response 99 | 100 | ## 6.0.1 (2023-07-11) 101 | 102 | * fix `HttpBackend` post_init method 103 | 104 | ## 6.0.0 (2023-07-10) 105 | 106 | * update `morecantile>=4.1,<5.0` and `rio-tiler>=5.0,<6.0` requirements 107 | 108 | * replace `supermercado` with [`supermorecado`](https://github.com/developmentseed/supermorecado) to burn geometries as tiles for different TMS 109 | 110 | * update MosaicJSON models to `0.0.3` specification (adds `tilematrixset`, `asset_type`, `asset_prefix`, `data_type`, `colormap` and `layers` attributes) 111 | 112 | * allow Mosaic creation using other TileMatrixSet (default is still `WebMercatorQuad`) 113 | 114 | * add `tms` support to MosaicBackend to read tile in other TMS than the mosaic TileMatrixSet 115 | 116 | ```python 117 | # Before 118 | # Mosaic and output Tile in WebMercatorQuad 119 | with MosaicBackend("mosaic.json") as mosaic: 120 | img, _ = mosaic.tile(0, 0, 0) 121 | 122 | # Now 123 | # Mosaic in WebMercatorQuad (default), output tile in WGS84 124 | WGS1984Quad = morecantile.tms.get("WGS1984Quad") 125 | with MosaicBackend("mosaic.json", tms=WGS1984Quad) as mosaic: 126 | img, _ = mosaic.tile(0, 0, 0) 127 | ``` 128 | 129 | ## 5.1.1 (2023-02-06) 130 | 131 | * Clip dataset bounds with of TMS bbox (author @lseelenbinder, https://github.com/developmentseed/cogeo-mosaic/pull/200) 132 | 133 | ## 5.1.0 (2023-01-20) 134 | 135 | * use `az://` prefix for private Azure Blob Storage Backend. 136 | 137 | ## 5.0.0 (2022-11-21) 138 | 139 | * switch from pygeos to shapely>=2.0 140 | 141 | ## 4.2.2 (2022-11-19) 142 | 143 | * remove useless file in package 144 | 145 | ## 4.2.1 (2022-11-15) 146 | 147 | * add python 3.11 support 148 | 149 | ## 4.2.0 (2022-10-24) 150 | 151 | * remove python 3.7 support 152 | * add python 3.10 support 153 | * switch to hatch build-system 154 | * update rio-tiler dependency to >=4.0.0a0 155 | 156 | ## 4.1.1 (2022-10-21) 157 | 158 | * Add Azure Blob Storage backend (author @christoe, https://github.com/developmentseed/cogeo-mosaic/pull/191) 159 | 160 | ## 4.1.0 (2022-02-22) 161 | 162 | * remove `mercantile` and switch to morecantile>=3.1 163 | 164 | ## 4.0.0 (2021-11-30) 165 | 166 | * no change since `4.0.0a2` 167 | 168 | ## 4.0.0a2 (2021-11-22) 169 | 170 | * update rio-tiler requirement (`>=3.0.0a6`) and update backend reader type information 171 | 172 | ## 4.0.0a1 (2021-11-18) 173 | 174 | * update rio-tiler requirement (`>=3.0.0a5`) 175 | * fix `MosaicBackend` to match Backend input names. 176 | 177 | ## 4.0.0a0 (2021-10-20) 178 | 179 | * update morecantile requirement to >= 3.0 180 | * update rio-tiler requirement to >= 3.0 and update Backend's properties 181 | * switch from `requests` to `httpx` 182 | * add `BaseBackend.assets_for_bbox()` method (https://github.com/developmentseed/cogeo-mosaic/pull/184) 183 | 184 | **breaking changes** 185 | 186 | * remove `BaseBackend.metadata()` method (can be replaced by `BaseBackend.mosaic_def.dict(exclude={"tiles"})`) 187 | * remove `cogeo_mosaic.models.Metadata` model 188 | * remove python 3.6 support 189 | * `BaseBackend.path` -> `BaseBackend.input` attribute (`input` was added in rio-tiler BaseReader) 190 | 191 | ## 3.0.2 (2021-07-08) 192 | 193 | * Add Google Cloud Storage (`gs://...`) mosaic backend (author @AndreaGiardini, https://github.com/developmentseed/cogeo-mosaic/pull/179) 194 | 195 | ## 3.0.1 (2021-06-22) 196 | 197 | * Make sure to pass an openned file to click.Progressbar (https://github.com/developmentseed/cogeo-mosaic/pull/178) 198 | 199 | ## 3.0.0 (2021-05-19) 200 | 201 | * update rio-tiler version dependencies 202 | * update pygeos dependency to >=0.10 which fix https://github.com/developmentseed/cogeo-mosaic/issues/81 203 | 204 | ## 3.0.0rc2 (2021-02-25) 205 | 206 | **breaking** 207 | 208 | * `gzip` is now only applied if the path endswith `.gz` 209 | * remove `backend_options` attribute in base backends. This attribute was used to pass optional `gzip` option and/or STAC related options 210 | * STAC backends has additional attributes (`stac_api_options` and `mosaic_options`) 211 | 212 | 213 | ## 3.0.0rc1 (2021-02-11) 214 | 215 | * add `SQLite` backend (https://github.com/developmentseed/cogeo-mosaic/pull/148) 216 | * fix cached responsed after updating a mosaic (https://github.com/developmentseed/cogeo-mosaic/pull/148/files#r557020660) 217 | * update mosaicJSON.bounds type definition to match rio-tiler BaseReader definition (https://github.com/developmentseed/cogeo-mosaic/issues/158) 218 | * add default bounds/minzoom/maxzoom values matching the mosaicjson default in the backends (https://github.com/developmentseed/cogeo-mosaic/pull/162) 219 | * raise an error when trying to pass `mosaic_def` in read-only backend (https://github.com/developmentseed/cogeo-mosaic/pull/162) 220 | * add `MemoryBackend` (https://github.com/developmentseed/cogeo-mosaic/pull/163) 221 | 222 | **breaking** 223 | 224 | * Updated the backends `.point()` methods to return a list in form of `[(asset1, values)]` (https://github.com/developmentseed/cogeo-mosaic/pull/168) 225 | 226 | ## 3.0.0b1 (2020-12-18) 227 | 228 | * remove `overview` command (https://github.com/developmentseed/cogeo-mosaic/issues/71#issuecomment-748265645) 229 | * remove `rio-cogeo` dependencies 230 | * update rio-tiler version (`2.0.0rc4`) 231 | 232 | ## 3.0.0a19 (2020-12-14) 233 | 234 | * Update to remove all calls to `rio_tiler.mercator` functions. 235 | 236 | ## 3.0.0a18 (2020-11-24) 237 | 238 | * update Backend base class for rio-tiler 2.0.0rc3 (add `.feature()` method) 239 | 240 | ## 3.0.0a17 (2020-11-09) 241 | 242 | * update for rio-tiler 2.0rc and add backend output models 243 | 244 | ## 3.0.0a16 (2020-10-26) 245 | 246 | * raise `MosaicNotFoundError` when mosaic doesn't exists in the DynamoDB table. 247 | 248 | ## 3.0.0a15 (2020-10-22) 249 | 250 | * fix typo in DynamoDB backend (https://github.com/developmentseed/cogeo-mosaic/pull/134) 251 | * rename `cogeo_mosaic/backends/http.py` -> `cogeo_mosaic/backends/web.py` to avoid conflicts (author @kylebarron, https://github.com/developmentseed/cogeo-mosaic/pull/133) 252 | 253 | ## 3.0.0a14 (2020-10-22) 254 | 255 | * add logger (`cogeo_mosaic.logger.logger`) 256 | * Update STACBackend to better handler paggination (ref: https://github.com/developmentseed/cogeo-mosaic/pull/125) 257 | * with change from #125, `stac_next_link_key` has be specified if you know the STAC API is using the latest specs: 258 | 259 | ```python 260 | with MosaicBackend( 261 | f"stac+{stac_endpoint}", 262 | query.copy(), 263 | 11, 264 | 14, 265 | backend_options={ 266 | "accessor": lambda feature: feature["id"], 267 | "stac_next_link_key": "next", 268 | } 269 | ) as mosaic: 270 | ``` 271 | 272 | * add `to-geojson` CLI to create a GeoJSON from a mosaicJSON document (#128) 273 | * refactor internal cache (https://github.com/developmentseed/cogeo-mosaic/pull/131) 274 | * add progressbar for iterating over quadkeys when creating a mosaic (author @kylebarron, https://github.com/developmentseed/cogeo-mosaic/pull/130) 275 | 276 | ### Breaking changes 277 | 278 | * refactored DynamoDB backend to store multiple mosaics in one table (https://github.com/developmentseed/cogeo-mosaic/pull/127) 279 | - new path schema `dynamodb://{REGION}?/{TABLE}:{MOSAIC}` 280 | 281 | * renamed exception `MosaicExists` to `MosaicExistsError` 282 | * renamed option `fetch_quadkeys` to `quadkeys` in DynamoDBBackend.info() method 283 | * add `quadkeys` option in `Backends.info()` to return (or not) the list of quadkeys (https://github.com/developmentseed/cogeo-mosaic/pull/129) 284 | * moves `get_assets` to the base Backend (https://github.com/developmentseed/cogeo-mosaic/pull/131) 285 | * remove multi_level mosaic support (https://github.com/developmentseed/cogeo-mosaic/issues/122) 286 | 287 | ## 3.0.0a13 (2020-10-13) 288 | 289 | * add TMS in BaseBackend to align with rio-tiler BaseBackend. 290 | 291 | ## 3.0.0a12 (2020-10-07) 292 | 293 | * remove pkg_resources (https://github.com/pypa/setuptools/issues/510) 294 | * raise error when `minimum_tile_cover` is > 1 (https://github.com/developmentseed/cogeo-mosaic/issues/117) 295 | * fix wrong indices sorting in default_filter (https://github.com/developmentseed/cogeo-mosaic/issues/118) 296 | 297 | Note: We changed the versioning scheme to {major}.{minor}.{path}{pre}{prenum} 298 | 299 | ## 3.0a11 (2020-09-21) 300 | 301 | * Raise Exception when trying to overwrite a mosaic (#112) 302 | * Add `reverse` option in `.tile` and `.point` to get values from assets in reversed order. 303 | 304 | ## 3.0a10 (2020-08-24) 305 | 306 | * Allow PointOutsideBounds exception for `point` method (#108) 307 | 308 | ## 3.0a9 (2020-08-24) 309 | 310 | * BaseBackend.center returns value from the mosaic definition (#105) 311 | 312 | ## 3.0a8 (2020-08-21) 313 | 314 | * BaseBackend is now a subclass of rio-tiler.io.base.BaseReader (add minzoom, maxzoom, bounds properties and info method) 315 | * use `attr` to define backend classes 316 | 317 | ### Breaking changes 318 | * `backend_options` is now used to pass options (*kwargs) to the `_read` method 319 | 320 | ## 3.0a7 (2020-07-31) 321 | 322 | * update to rio-tiler 2.0b5 323 | 324 | ### Breaking changes 325 | * 'value' -> 'values' in MosaicBackend.point output (#98) 326 | 327 | ## 3.0a6 (2020-07-31) 328 | 329 | * Use environement variable to set/disable cache (#93, autho @geospatial-jeff) 330 | * Allow Threads configuration for overview command (author @kylebarron) 331 | * add --in-memory/--no-in-memory to control temporary files creation for `overview` function. 332 | * allow pixel_selection method options for `overview` function. 333 | * update to rio-tiler 2.0b4 334 | * use new COGReader and STACReader to add .tile and .point methods directly in the backends 335 | 336 | ### Breaking changes 337 | * backend.tile -> backend.assets_for_tile 338 | * backend.point -> backend.assets_for_point 339 | 340 | ## 3.0a5 (2020-06-29) 341 | 342 | * remove FTP from supported backend (#87, author @geospatial-jeff) 343 | * add backend CRUD exceptions (#86, author @geospatial-jeff) 344 | 345 | ## 3.0a4 (2020-06-25) 346 | 347 | * add STACBackend (#82) 348 | * fix backends caching and switch to TTL cache (#83) 349 | 350 | ## 3.0a3 (2020-05-01) 351 | 352 | * add Upload CLI (#74, author @kylebarron) 353 | * fix boto3 dynamodb exception (#75) 354 | 355 | ## 3.0a2 (2020-05-01) 356 | 357 | * Better mosaicJSON model testing and default center from bounds (#73, author @geospatial-jeff) 358 | 359 | ## 3.0a1 (2020-05-01) 360 | 361 | This is a major version, meaning a lot of refactoring was done and may lead to breaking changes. 362 | 363 | * add quadkey_zoom option in CLI (#41, author @kylebarron) 364 | * use R-tree from pygeos for testing intersections (#43, author @kylebarron) 365 | 366 | ### Breaking changes 367 | * added BackendStorage for dynamodb, s3, file and http (with @kylebarron) 368 | * added MosaicJSON pydantic model for internal mosaicjson representation (with @kylebarron and @geospatial-jeff) 369 | 370 | ## 2.0.1 (2020-01-28) 371 | 372 | * Bug fix, use pygeos from pypi instead of git repo 373 | 374 | ## 2.0.0 (2020-01-28) - Major refactor 375 | 376 | * remove stack related code (lambda handler, serverless) 377 | * switch to pygeos (#24) 378 | * bug fixes 379 | * add `last` pixel_method 380 | 381 | ## 1.0.0 (2019-12-13) 382 | 383 | * add tif output 384 | * fix overview creation 385 | * add other Web templates 386 | 387 | ## 0.3.0 (2019-11-07) 388 | 389 | * use aws lambda layer 390 | * add `update_mosaic` utility function 391 | * add `/tiles/point` endpoint to get points values from a mosaic 392 | * add logs for mosaic creation 393 | * add custom pixel methods 394 | * add custom color maps 395 | 396 | ### Breaking changes 397 | * rename `/mosaic/info/` to `/mosaic//info` 398 | 399 | ## 0.2.0 (2019-09-30) 400 | 401 | * update for lambda-proxy~=5.0 (#15) 402 | * add `minimum_tile_cover` option for mosaic creation (#16) 403 | * add `tile_cover_sort` option (#16) 404 | * add verbosity for cli 405 | 406 | ## 0.1.0 (2019-09-05) 407 | 408 | * add /create.html endpoint (#14) 409 | * update to remotepixel/amazonlinux docker image 410 | --------------------------------------------------------------------------------