├── docs ├── docs │ ├── index.md │ ├── release-notes.md │ ├── contributing.md │ └── usage.md └── mkdocs.yml ├── tests ├── __init__.py ├── fixtures │ ├── protomaps(vector)ODbL_firenze.pmtiles │ ├── usgs-mt-whitney-8-15-webp-512.pmtiles │ └── stamen_toner(raster)CC-BY+ODbL_z3.pmtiles ├── test_io.py └── test_reader.py ├── CHANGES.md ├── aiopmtiles ├── __init__.py ├── aiopmtiles.py └── io.py ├── .github ├── codecov.yml └── workflows │ ├── deploy_mkdocs.yml │ └── ci.yml ├── .bumpversion.cfg ├── CONTRIBUTING.md ├── .pre-commit-config.yaml ├── LICENSE ├── .gitignore ├── pyproject.toml └── README.md /docs/docs/index.md: -------------------------------------------------------------------------------- 1 | ../../README.md -------------------------------------------------------------------------------- /docs/docs/release-notes.md: -------------------------------------------------------------------------------- 1 | ../../CHANGES.md -------------------------------------------------------------------------------- /docs/docs/contributing.md: -------------------------------------------------------------------------------- 1 | ../../CONTRIBUTING.md -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """aiopmtiles tests suite.""" 2 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | 2 | ## 0.1.0 (TMB) 3 | 4 | Initial release. 5 | -------------------------------------------------------------------------------- /aiopmtiles/__init__.py: -------------------------------------------------------------------------------- 1 | """pmtiles""" 2 | 3 | __version__ = "0.1.0" 4 | 5 | from .aiopmtiles import Reader # noqa 6 | -------------------------------------------------------------------------------- /tests/fixtures/protomaps(vector)ODbL_firenze.pmtiles: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/developmentseed/aiopmtiles/HEAD/tests/fixtures/protomaps(vector)ODbL_firenze.pmtiles -------------------------------------------------------------------------------- /tests/fixtures/usgs-mt-whitney-8-15-webp-512.pmtiles: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/developmentseed/aiopmtiles/HEAD/tests/fixtures/usgs-mt-whitney-8-15-webp-512.pmtiles -------------------------------------------------------------------------------- /tests/fixtures/stamen_toner(raster)CC-BY+ODbL_z3.pmtiles: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/developmentseed/aiopmtiles/HEAD/tests/fixtures/stamen_toner(raster)CC-BY+ODbL_z3.pmtiles -------------------------------------------------------------------------------- /.github/codecov.yml: -------------------------------------------------------------------------------- 1 | comment: off 2 | 3 | coverage: 4 | status: 5 | project: 6 | default: 7 | target: auto 8 | threshold: 5 9 | -------------------------------------------------------------------------------- /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.1.0 3 | commit = True 4 | tag = True 5 | tag_name = {new_version} 6 | 7 | [bumpversion:file:aiopmtiles/__init__.py] 8 | search = __version__ = "{current_version}" 9 | replace = __version__ = "{new_version}" 10 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Issues and pull requests are more than welcome. 4 | 5 | **dev install** 6 | 7 | ```bash 8 | $ git clone https://github.com/developmentseed/aiopmtiles.git 9 | $ cd aiopmtiles 10 | $ python -m pip install -e .["test","dev","aws","gcp"] 11 | ``` 12 | 13 | You can then run the tests with the following command: 14 | 15 | ```sh 16 | python -m pytest --cov aiopmtiles --cov-report term-missing 17 | ``` 18 | 19 | **pre-commit** 20 | 21 | This repo is set to use `pre-commit` to run *isort*, *flake8*, *pydocstring*, *black* ("uncompromising Python code formatter") and mypy when committing new code. 22 | 23 | ```bash 24 | $ pre-commit install 25 | ``` 26 | -------------------------------------------------------------------------------- /.github/workflows/deploy_mkdocs.yml: -------------------------------------------------------------------------------- 1 | name: Publish docs via GitHub Pages 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | # Only rebuild website when docs have changed 9 | - 'README.md' 10 | - 'docs/**' 11 | - 'mkdocs.yml' 12 | - 'rio_stac/**.py' 13 | - .github/workflows/deploy_mkdocs.yml 14 | 15 | jobs: 16 | build: 17 | name: Deploy docs 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Checkout main 21 | uses: actions/checkout@v5 22 | 23 | - name: Set up Python 3.12 24 | uses: actions/setup-python@v6 25 | with: 26 | python-version: 3.12 27 | 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install --upgrade pip 31 | python -m pip install -e .["doc"] 32 | 33 | - name: Deploy docs 34 | run: mkdocs gh-deploy -f docs/mkdocs.yml --force 35 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/abravalheri/validate-pyproject 3 | rev: v0.12.1 4 | hooks: 5 | - id: validate-pyproject 6 | 7 | - repo: https://github.com/psf/black 8 | rev: 22.12.0 9 | hooks: 10 | - id: black 11 | language_version: python 12 | 13 | - repo: https://github.com/PyCQA/isort 14 | rev: 5.12.0 15 | hooks: 16 | - id: isort 17 | language_version: python 18 | 19 | - repo: https://github.com/charliermarsh/ruff-pre-commit 20 | rev: v0.0.238 21 | hooks: 22 | - id: ruff 23 | args: ["--fix"] 24 | 25 | - repo: https://github.com/pre-commit/mirrors-mypy 26 | rev: v0.991 27 | hooks: 28 | - id: mypy 29 | language_version: python 30 | # No reason to run if only tests have changed. They intentionally break typing. 31 | exclude: tests/.* 32 | additional_dependencies: 33 | - types-aiofiles 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Development Seed 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/test_io.py: -------------------------------------------------------------------------------- 1 | """test IO FileSystem.""" 2 | 3 | import os 4 | 5 | import pytest 6 | 7 | from aiopmtiles.io import ( 8 | FileSystem, 9 | GcsFileSystem, 10 | HttpFileSystem, 11 | LocalFileSystem, 12 | S3FileSystem, 13 | ) 14 | 15 | FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures") 16 | VECTOR_PMTILES = os.path.join(FIXTURES_DIR, "protomaps(vector)ODbL_firenze.pmtiles") 17 | 18 | 19 | @pytest.mark.parametrize( 20 | "url,fs", 21 | [ 22 | ("myfile.pmtiles", LocalFileSystem), 23 | ("file:///myfile.pmtiles", LocalFileSystem), 24 | ("s3://bucket/myfile.pmtiles", S3FileSystem), 25 | ("gs://bucket/myfile.pmtiles", GcsFileSystem), 26 | ("http://url.io/myfile.pmtiles", HttpFileSystem), 27 | ("https://url.io/myfile.pmtiles", HttpFileSystem), 28 | ], 29 | ) 30 | def test_create_from_filepath(url, fs): 31 | """Test Filesystem creation from url.""" 32 | assert isinstance(FileSystem.create_from_filepath(url), fs) 33 | 34 | 35 | def test_bad_schema(): 36 | """Should raise ValueError.""" 37 | with pytest.raises(ValueError): 38 | FileSystem.create_from_filepath("something://myfile.pmtiles") 39 | 40 | 41 | @pytest.mark.asyncio 42 | async def test_local_fs(): 43 | """Test LocalFilesSytem.""" 44 | async with LocalFileSystem(VECTOR_PMTILES) as fs: 45 | assert not fs.file.closed 46 | magic_bytes = await fs.get(0, 6) 47 | assert magic_bytes.decode() == "PMTiles" 48 | assert fs.file.closed 49 | -------------------------------------------------------------------------------- /docs/mkdocs.yml: -------------------------------------------------------------------------------- 1 | # Project Information 2 | site_name: 'aiopmtiles' 3 | site_description: 'Async Version of Python PMTiles Reader.' 4 | 5 | # Repository 6 | repo_name: 'developmentseed/aiopmtiles' 7 | repo_url: 'http://github.com/developmentseed/aiopmtiles' 8 | edit_uri: 'blob/main/docs/src/' 9 | site_url: 'https://developmentseed.org/aiopmtiles/' 10 | 11 | # Social links 12 | extra: 13 | social: 14 | - icon: 'fontawesome/brands/github' 15 | link: 'https://github.com/developmentseed' 16 | - icon: 'fontawesome/brands/twitter' 17 | link: 'https://twitter.com/developmentseed' 18 | 19 | # Layout 20 | nav: 21 | - Home: 'index.md' 22 | - Usage: 'usage.md' 23 | - Development - Contributing: 'contributing.md' 24 | - Release Notes: 'release-notes.md' 25 | 26 | plugins: 27 | - search 28 | 29 | # Theme 30 | theme: 31 | icon: 32 | logo: 'material/home' 33 | repo: 'fontawesome/brands/github' 34 | name: 'material' 35 | language: 'en' 36 | palette: 37 | primary: 'red' 38 | accent: 'light red' 39 | font: 40 | text: 'Nunito Sans' 41 | code: 'Fira Code' 42 | 43 | # These extensions are chosen to be a superset of Pandoc's Markdown. 44 | # This way, I can write in Pandoc's Markdown and have it be supported here. 45 | # https://pandoc.org/MANUAL.html 46 | markdown_extensions: 47 | - admonition 48 | - attr_list 49 | - codehilite: 50 | guess_lang: false 51 | - def_list 52 | - footnotes 53 | - pymdownx.arithmatex 54 | - pymdownx.betterem 55 | - pymdownx.caret: 56 | insert: false 57 | - pymdownx.details 58 | - pymdownx.emoji 59 | - pymdownx.escapeall: 60 | hardbreak: true 61 | nbsp: true 62 | - pymdownx.magiclink: 63 | hide_protocol: true 64 | repo_url_shortener: true 65 | - pymdownx.smartsymbols 66 | - pymdownx.superfences 67 | - pymdownx.tasklist: 68 | custom_checkbox: true 69 | - pymdownx.tilde 70 | - toc: 71 | permalink: true 72 | -------------------------------------------------------------------------------- /tests/test_reader.py: -------------------------------------------------------------------------------- 1 | """Test Reader.""" 2 | 3 | import os 4 | 5 | import pytest 6 | 7 | from aiopmtiles import Reader 8 | from aiopmtiles.io import LocalFileSystem 9 | 10 | FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures") 11 | VECTOR_PMTILES = os.path.join(FIXTURES_DIR, "protomaps(vector)ODbL_firenze.pmtiles") 12 | RASTER_PMTILES = os.path.join(FIXTURES_DIR, "usgs-mt-whitney-8-15-webp-512.pmtiles") 13 | V2_PMTILES = os.path.join(FIXTURES_DIR, "stamen_toner(raster)CC-BY+ODbL_z3.pmtiles") 14 | 15 | 16 | @pytest.mark.asyncio 17 | async def test_reader_vector(): 18 | """Test Reader with Vector PMTiles.""" 19 | async with Reader(VECTOR_PMTILES) as src: 20 | assert isinstance(src.fs, LocalFileSystem) 21 | assert src._header 22 | assert src._header_offset == 0 23 | assert src._header_length == 127 24 | assert src.bounds 25 | assert src.minzoom == 0 26 | assert src.maxzoom == 14 27 | assert src.center[2] == 0 28 | assert src.is_vector 29 | assert src.tile_compression.name == "GZIP" 30 | assert not src.fs.file.closed 31 | assert src.tile_type.name == "MVT" 32 | 33 | metadata = await src.metadata() 34 | assert "attribution" in metadata 35 | assert "tilestats" in metadata 36 | 37 | assert src.fs.file.closed 38 | 39 | 40 | @pytest.mark.asyncio 41 | async def test_reader_raster(): 42 | """Test Reader with raster PMTiles.""" 43 | async with Reader(RASTER_PMTILES) as src: 44 | assert isinstance(src.fs, LocalFileSystem) 45 | assert src._header 46 | assert src.bounds 47 | assert src.minzoom == 8 48 | assert src.maxzoom == 15 49 | assert src.center[2] == 12 50 | assert not src.is_vector 51 | assert src.tile_compression.name == "NONE" 52 | assert not src.fs.file.closed 53 | assert src.tile_type.name == "WEBP" 54 | 55 | metadata = await src.metadata() 56 | assert "attribution" in metadata 57 | assert "type" in metadata 58 | 59 | assert src.fs.file.closed 60 | 61 | 62 | @pytest.mark.asyncio 63 | async def test_reader_bad_spec(): 64 | """Should raise an error if not spec == 3.""" 65 | with pytest.raises(AssertionError): 66 | async with Reader(V2_PMTILES) as src: 67 | pass 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "aiopmtiles" 3 | description = "Async version of protomaps/PMTiles." 4 | readme = "README.md" 5 | requires-python = ">=3.11" 6 | license = {file = "LICENSE"} 7 | authors = [ 8 | {name = "Vincent Sarago", email = "vincent@developmentseed.com"}, 9 | ] 10 | classifiers = [ 11 | "Intended Audience :: Information Technology", 12 | "Intended Audience :: Science/Research", 13 | "License :: OSI Approved :: BSD License", 14 | "Programming Language :: Python :: 3.11", 15 | "Programming Language :: Python :: 3.12", 16 | "Programming Language :: Python :: 3.13", 17 | "Topic :: Scientific/Engineering :: GIS", 18 | ] 19 | dynamic = ["version"] 20 | dependencies = [ 21 | "pmtiles", 22 | "httpx", 23 | "aiofiles", 24 | "aiocache", 25 | ] 26 | 27 | [project.optional-dependencies] 28 | aws = [ 29 | "aioboto3" 30 | ] 31 | gcp = [ 32 | "gcloud-aio-auth", 33 | "gcloud-aio-storage" 34 | ] 35 | test = [ 36 | "pytest", 37 | "pytest-cov", 38 | "pytest-asyncio", 39 | ] 40 | dev = [ 41 | "pre-commit", 42 | ] 43 | doc = [ 44 | "mkdocs", 45 | "mkdocs-material", 46 | "pygments", 47 | "pdocs", 48 | ] 49 | 50 | [project.urls] 51 | Source = "https://github.com/developmentseed/aiopmtiles" 52 | Documentation = "https://developmentseed.org/aiopmtiles/" 53 | 54 | [build-system] 55 | requires = ["flit>=3.2,<4"] 56 | build-backend = "flit_core.buildapi" 57 | 58 | [tool.flit.module] 59 | name = "aiopmtiles" 60 | 61 | [tool.flit.sdist] 62 | exclude = [ 63 | "tests/", 64 | "docs/", 65 | ".github/", 66 | "CHANGES.md", 67 | "CONTRIBUTING.md", 68 | ] 69 | 70 | [tool.coverage.run] 71 | branch = true 72 | parallel = true 73 | 74 | [tool.coverage.report] 75 | exclude_lines = [ 76 | "no cov", 77 | "if __name__ == .__main__.:", 78 | "if TYPE_CHECKING:", 79 | ] 80 | 81 | [tool.isort] 82 | profile = "black" 83 | known_first_party = ["aiopmtiles"] 84 | known_third_party = ["pmtiles"] 85 | default_section = "THIRDPARTY" 86 | 87 | [tool.mypy] 88 | no_strict_optional = true 89 | 90 | [tool.ruff] 91 | select = [ 92 | "D1", # pydocstyle errors 93 | "E", # pycodestyle errors 94 | "W", # pycodestyle warnings 95 | "C", # flake8-comprehensions 96 | "B", # flake8-bugbear 97 | ] 98 | ignore = [ 99 | "E501", # line too long, handled by black 100 | "B008", # do not perform function calls in argument defaults 101 | "B905", # ignore zip() without an explicit strict= parameter, only support with python >3.10 102 | ] 103 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | # On every pull request, but only on push to main 4 | on: 5 | push: 6 | branches: 7 | - main 8 | tags: 9 | - '*' 10 | pull_request: 11 | env: 12 | LATEST_PY_VERSION: '3.13' 13 | 14 | jobs: 15 | tests: 16 | runs-on: ubuntu-latest 17 | strategy: 18 | matrix: 19 | python-version: 20 | - '3.11' 21 | - '3.12' 22 | - '3.13' 23 | 24 | steps: 25 | - uses: actions/checkout@v5 26 | - name: Set up Python ${{ matrix.python-version }} 27 | uses: actions/setup-python@v6 28 | with: 29 | python-version: ${{ matrix.python-version }} 30 | 31 | - name: Install aiopmtiles 32 | run: | 33 | python -m pip install --upgrade pip 34 | python -m pip install .["test","aws","gcp"] 35 | 36 | - name: run pre-commit 37 | if: ${{ matrix.python-version == env.LATEST_PY_VERSION }} 38 | run: | 39 | python -m pip install pre-commit 40 | pre-commit run --all-files 41 | 42 | - name: Run test 43 | run: python -m pytest --cov aiopmtiles --cov-report xml --cov-report term-missing 44 | 45 | - name: Upload Results 46 | if: ${{ matrix.python-version == env.LATEST_PY_VERSION }} 47 | uses: codecov/codecov-action@v1 48 | with: 49 | file: ./coverage.xml 50 | flags: unittests 51 | name: ${{ matrix.python-version }} 52 | fail_ci_if_error: false 53 | 54 | publish: 55 | needs: [tests] 56 | runs-on: ubuntu-latest 57 | if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' 58 | steps: 59 | - uses: actions/checkout@v5 60 | - name: Set up Python 61 | uses: actions/setup-python@v6 62 | with: 63 | python-version: ${{ env.LATEST_PY_VERSION }} 64 | 65 | - name: Install dependencies 66 | run: | 67 | python -m pip install --upgrade pip 68 | python -m pip install flit 69 | python -m pip install . 70 | 71 | - name: Set tag version 72 | id: tag 73 | # https://stackoverflow.com/questions/58177786/get-the-current-pushed-tag-in-github-actions 74 | run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} 75 | 76 | - name: Set module version 77 | id: module 78 | # https://stackoverflow.com/questions/58177786/get-the-current-pushed-tag-in-github-actions 79 | run: echo ::set-output name=version::$(python -c 'from importlib.metadata import version; print(version("aiopmtiles"))') 80 | 81 | - name: Build and publish 82 | if: steps.tag.outputs.tag == steps.module.outputs.version 83 | env: 84 | FLIT_USERNAME: ${{ secrets.PYPI_USERNAME }} 85 | FLIT_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 86 | run: flit publish 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aiopmtiles 2 | 3 |

4 | Async Version of Python PMTiles Reader. 5 |

6 |

7 | 8 | Test 9 | 10 | 11 | Coverage 12 | 13 | 14 | Package version 15 | 16 | 17 | Downloads 18 | 19 | 20 | Downloads 21 | 22 |

23 | 24 | --- 25 | 26 | **Documentation**: https://developmentseed.org/aiopmtiles/ 27 | 28 | **Source Code**: https://github.com/developmentseed/aiopmtiles 29 | 30 | --- 31 | 32 | `aiopmtiles` is a python `Async I/O` version of the great [PMTiles](https://github.com/protomaps/PMTiles) python reader. 33 | 34 | The [**PMTiles**](https://github.com/protomaps/PMTiles) format is a *Cloud-optimized + compressed single-file tile archives for vector and raster maps*. 35 | 36 | ## Installation 37 | 38 | ```bash 39 | $ python -m pip install pip -U 40 | 41 | # From Pypi 42 | $ python -m pip install aiopmtiles 43 | 44 | # Or from source 45 | $ python -m pip install git+http://github.com/developmentseed/aiopmtiles 46 | ``` 47 | 48 | ## Example 49 | 50 | ```python 51 | 52 | from aiopmtiles import Reader 53 | 54 | async with Reader("https://r2-public.protomaps.com/protomaps-sample-datasets/cb_2018_us_zcta510_500k.pmtiles") as src: 55 | # PMTiles Metadata 56 | meta = src.metadata 57 | 58 | # Spatial Metadata 59 | bounds = src.bounds 60 | minzoom, maxzoom = src.minzoom, src.maxzoom 61 | 62 | # Is the data a Vector Tile Archive 63 | assert src.is_vector 64 | 65 | # PMTiles tiles type 66 | tile_type = src._header["tile_type"] 67 | 68 | # Tile Compression 69 | comp = src.tile_compression 70 | 71 | # Get Tile 72 | data = await src.get_tile(0, 0, 0) 73 | ``` 74 | 75 | ## Contribution & Development 76 | 77 | See [CONTRIBUTING.md](https://github.com/developmentseed/aiopmtiles/blob/main/CONTRIBUTING.md) 78 | 79 | ## Authors 80 | 81 | See [contributors](https://github.com/developmentseed/aiopmtiles/graphs/contributors) 82 | 83 | ## Changes 84 | 85 | See [CHANGES.md](https://github.com/developmentseed/aiopmtiles/blob/main/CHANGES.md). 86 | 87 | ## License 88 | 89 | See [LICENSE](https://github.com/developmentseed/aiopmtiles/blob/main/LICENSE) 90 | -------------------------------------------------------------------------------- /aiopmtiles/aiopmtiles.py: -------------------------------------------------------------------------------- 1 | """Async version of protomaps/PMTiles.""" 2 | 3 | import gzip 4 | import json 5 | from contextlib import AsyncExitStack 6 | from dataclasses import dataclass, field 7 | from typing import Dict, Optional, Protocol, Tuple 8 | 9 | from aiocache import Cache, cached 10 | from pmtiles.tile import ( 11 | Compression, 12 | TileType, 13 | deserialize_directory, 14 | deserialize_header, 15 | find_tile, 16 | zxy_to_tileid, 17 | ) 18 | 19 | from aiopmtiles.io import FileSystem 20 | 21 | 22 | class _GetBytes(Protocol): 23 | async def __call__(self, offset: int, length: int) -> bytes: 24 | ... 25 | 26 | 27 | @dataclass 28 | class Reader: 29 | """PMTiles Reader.""" 30 | 31 | filepath: str 32 | options: Optional[Dict] = field(default_factory=dict) 33 | 34 | fs: FileSystem = field(init=False) 35 | 36 | _header: Dict = field(init=False) 37 | _header_offset: int = field(default=0, init=False) 38 | _header_length: int = field(default=127, init=False) 39 | 40 | _ctx: AsyncExitStack = field(default_factory=AsyncExitStack) 41 | 42 | async def __aenter__(self): 43 | """Support using with Context Managers.""" 44 | self.fs = await self._ctx.enter_async_context( 45 | FileSystem.create_from_filepath(self.filepath, **self.options) 46 | ) 47 | 48 | header_values = await self._get(self._header_offset, self._header_length) 49 | spec_version = header_values[7] 50 | assert spec_version == 3, "Only Version 3 of PMTiles specification is supported" 51 | 52 | self._header = deserialize_header(header_values) 53 | 54 | return self 55 | 56 | async def __aexit__(self, exc_type, exc_value, traceback): 57 | """Support using with Context Managers.""" 58 | await self._ctx.__aexit__(exc_type, exc_value, traceback) 59 | 60 | @cached( 61 | cache=Cache.MEMORY, 62 | key_builder=lambda f, self, offset, length: f"{self.filepath}-{offset}-{length}", 63 | ) 64 | async def _get(self, offset: int, length: int) -> bytes: 65 | """Get Bytes.""" 66 | return await self.fs.get(offset, length) 67 | 68 | async def metadata(self) -> Dict: 69 | """Return PMTiles Metadata.""" 70 | metadata = await self._get( 71 | self._header["metadata_offset"], 72 | self._header["metadata_length"] - 1, 73 | ) 74 | if self._header["internal_compression"] == Compression.GZIP: 75 | metadata = gzip.decompress(metadata) 76 | 77 | return json.loads(metadata) 78 | 79 | async def get_tile(self, z, x, y) -> Optional[bytes]: 80 | """Get Tile Data.""" 81 | tile_id = zxy_to_tileid(z, x, y) 82 | 83 | dir_offset = self._header["root_offset"] 84 | dir_length = self._header["root_length"] 85 | for _ in range(0, 4): # max depth 86 | directory_values = await self._get(dir_offset, dir_length - 1) 87 | directory = deserialize_directory(directory_values) 88 | 89 | if result := find_tile(directory, tile_id): 90 | if result.run_length == 0: 91 | dir_offset = self._header["leaf_directory_offset"] + result.offset 92 | dir_length = result.length 93 | 94 | else: 95 | data = await self._get( 96 | self._header["tile_data_offset"] + result.offset, 97 | result.length - 1, 98 | ) 99 | return data 100 | 101 | return None 102 | 103 | @property 104 | def minzoom(self) -> int: 105 | """Return minzoom.""" 106 | return self._header["min_zoom"] 107 | 108 | @property 109 | def maxzoom(self) -> int: 110 | """Return maxzoom.""" 111 | return self._header["max_zoom"] 112 | 113 | @property 114 | def bounds(self) -> Tuple[float, float, float, float]: 115 | """Return Archive Bounds.""" 116 | return ( 117 | self._header["min_lon_e7"] / 10000000, 118 | self._header["min_lat_e7"] / 10000000, 119 | self._header["max_lon_e7"] / 10000000, 120 | self._header["max_lat_e7"] / 10000000, 121 | ) 122 | 123 | @property 124 | def center(self) -> Tuple[float, float, int]: 125 | """Return Archive center.""" 126 | return ( 127 | self._header["center_lon_e7"] / 10000000, 128 | self._header["center_lat_e7"] / 10000000, 129 | self._header["center_zoom"], 130 | ) 131 | 132 | @property 133 | def is_vector(self) -> bool: 134 | """Return tile type.""" 135 | return self._header["tile_type"] == TileType.MVT 136 | 137 | @property 138 | def tile_compression(self) -> Compression: 139 | """Return tile compression type.""" 140 | return self._header["tile_compression"] 141 | 142 | @property 143 | def tile_type(self) -> TileType: 144 | """Return tile type.""" 145 | return self._header["tile_type"] 146 | -------------------------------------------------------------------------------- /aiopmtiles/io.py: -------------------------------------------------------------------------------- 1 | """FileSystems for PMTiles Reader.""" 2 | 3 | import abc 4 | from contextlib import AsyncExitStack 5 | from dataclasses import dataclass, field 6 | from typing import Any 7 | from urllib.parse import urlparse 8 | 9 | import aiofiles 10 | import httpx 11 | 12 | try: 13 | import aioboto3 14 | 15 | except ImportError: # pragma: nocover 16 | aioboto3 = None # type: ignore 17 | 18 | try: 19 | from gcloud.aio.storage import Storage as GcpStorage 20 | 21 | except ImportError: # pragma: nocover 22 | GcpStorage = None # type: ignore 23 | 24 | 25 | @dataclass 26 | class FileSystem(abc.ABC): 27 | """Filesystem base class""" 28 | 29 | filepath: str 30 | ctx: AsyncExitStack = field(default_factory=AsyncExitStack, init=False) 31 | 32 | @abc.abstractmethod 33 | async def get(self, offset: int, length: int) -> bytes: 34 | """Perform a range request""" 35 | ... 36 | 37 | @abc.abstractmethod 38 | async def __aenter__(self): 39 | """Async context management""" 40 | ... 41 | 42 | async def __aexit__(self, exc_type, exc_val, exc_tb): 43 | """async context management""" 44 | await self.ctx.aclose() 45 | 46 | @classmethod 47 | def create_from_filepath(cls, filepath: str, **kwargs: Any) -> "FileSystem": 48 | """Instantiate the appropriate filesystem based on filepath scheme""" 49 | parsed = urlparse(filepath) 50 | 51 | if parsed.scheme in {"http", "https"}: 52 | return HttpFileSystem(filepath, **kwargs) 53 | 54 | elif parsed.scheme == "s3": 55 | return S3FileSystem(filepath, **kwargs) 56 | 57 | elif parsed.scheme == "gs": 58 | return GcsFileSystem(filepath, **kwargs) 59 | 60 | elif parsed.scheme == "file": 61 | return LocalFileSystem(filepath, **kwargs) 62 | 63 | # Invalid Scheme 64 | elif parsed.scheme: 65 | raise ValueError(f"'{parsed.scheme}' is not supported") 66 | 67 | # fallback to LocalFileSystem 68 | else: 69 | return LocalFileSystem(filepath, **kwargs) 70 | 71 | 72 | @dataclass 73 | class LocalFileSystem(FileSystem): 74 | """Local (disk) filesystem""" 75 | 76 | file: Any = field(init=False) 77 | 78 | async def get(self, offset: int, length: int) -> bytes: 79 | """Perform a range request""" 80 | await self.file.seek(offset) 81 | return await self.file.read(length + 1) 82 | 83 | async def __aenter__(self): 84 | """Async context management""" 85 | self.file = await self.ctx.enter_async_context( 86 | aiofiles.open(self.filepath, "rb") 87 | ) 88 | return self 89 | 90 | 91 | @dataclass 92 | class HttpFileSystem(FileSystem): 93 | """HTTP filesystem""" 94 | 95 | client: httpx.AsyncClient = field(init=False) 96 | 97 | async def get(self, offset: int, length: int) -> bytes: 98 | """Perform a range request""" 99 | range_header = {"Range": f"bytes={offset}-{offset + length}"} 100 | resp = await self.client.get(self.filepath, headers=range_header) 101 | resp.raise_for_status() 102 | return resp.content 103 | 104 | async def __aenter__(self): 105 | """Async context management""" 106 | self.client = await self.ctx.enter_async_context(httpx.AsyncClient()) 107 | return self 108 | 109 | 110 | @dataclass 111 | class S3FileSystem(FileSystem): 112 | """S3 filesystem""" 113 | 114 | request_payer: bool = False 115 | 116 | _session: Any = field(init=False) 117 | _resource: Any = field(init=False) 118 | _obj: Any = field(init=False) 119 | 120 | def __post_init__(self): 121 | """Check for dependency.""" 122 | assert aioboto3 is not None, "'aioboto3' must be installed to use S3 FileSystem" 123 | 124 | async def get(self, offset: int, length: int) -> bytes: 125 | """Perform a range request""" 126 | kwargs = {} 127 | if self.request_payer: 128 | kwargs["RequestPayer"] = self.request_payer 129 | 130 | req = await self._obj.get(Range=f"bytes={offset}-{offset + length}", **kwargs) 131 | 132 | return await req["Body"].read() 133 | 134 | async def __aenter__(self): 135 | """Async context management""" 136 | parsed = urlparse(self.filepath) 137 | self._session = aioboto3.Session() 138 | self._resource = await self.ctx.enter_async_context( 139 | self._session.resource("s3") 140 | ) 141 | self._obj = await self._resource.Object(parsed.netloc, parsed.path.strip("/")) 142 | return self 143 | 144 | 145 | @dataclass 146 | class GcsFileSystem(FileSystem): 147 | """GCS filesystem""" 148 | 149 | _client: GcpStorage = field(init=False) 150 | _bucket: str = field(init=False) 151 | _obj: str = field(init=False) 152 | 153 | def __post_init__(self): 154 | """Check for dependency.""" 155 | assert ( 156 | GcpStorage is not None 157 | ), "'gcloud-aio-storage' must be installed to use GCS FileSystem" 158 | 159 | async def get(self, offset: int, length: int) -> bytes: 160 | """Perform a range request""" 161 | headers = {"Range": f"bytes={offset}-{offset + length}"} 162 | return await self._client.download(self._bucket, self._obj, headers=headers) 163 | 164 | async def __aenter__(self): 165 | """Async context management""" 166 | parsed_path = urlparse(self.filepath) 167 | self._client = await self.ctx.enter_async_context(GcpStorage()) 168 | self._bucket = parsed_path.netloc 169 | self._obj = parsed_path.path.strip("/") 170 | return self 171 | -------------------------------------------------------------------------------- /docs/docs/usage.md: -------------------------------------------------------------------------------- 1 | Create a simple FastAPI application to serve tiles from PMTiles 2 | 3 | ```python 4 | from typing import Dict 5 | 6 | from fastapi import FastAPI, Path, Query 7 | from pmtiles.tile import Compression 8 | from starlette.middleware.cors import CORSMiddleware 9 | from starlette.requests import Request 10 | from starlette.responses import Response 11 | 12 | from aiopmtiles import Reader 13 | 14 | 15 | app = FastAPI() 16 | 17 | 18 | app.add_middleware( 19 | CORSMiddleware, 20 | allow_origins=["*"], 21 | allow_credentials=True, 22 | allow_methods=["GET"], 23 | allow_headers=["*"], 24 | ) 25 | 26 | @app.get("/metadata") 27 | async def metadata(url: str = Query(..., description="PMTiles archive URL.")): 28 | """get Metadata.""" 29 | async with Reader(url) as src: 30 | return await src.metadata() 31 | 32 | @app.get("/tiles/{z}/{x}/{y}", response_class=Response) 33 | async def tiles( 34 | z: int = Path(ge=0, le=30, description="TMS tiles's zoom level"), 35 | x: int = Path(description="TMS tiles's column"), 36 | y: int = Path(description="TMS tiles's row"), 37 | url: str = Query(..., description="PMTiles archive URL."), 38 | ): 39 | """get Tile.""" 40 | headers: Dict[str, str] = {} 41 | 42 | async with Reader(url) as src: 43 | data = await src.get_tile(z, x, y) 44 | if src.header["internal_compression"] == Compression.GZIP: 45 | headers["Content-Encoding"] = "gzip" 46 | 47 | return Response(data, media_type="application/x-protobuf", headers=headers) 48 | 49 | @app.get("/tilejson.json") 50 | async def tilejson( 51 | request: Request, 52 | url: str = Query(..., description="PMTiles archive URL."), 53 | ): 54 | """get TileJSON.""" 55 | async with Reader(url) as src: 56 | tilejson = { 57 | "tilejson": "3.0.0", 58 | "name": "pmtiles", 59 | "version": "1.0.0", 60 | "scheme": "xyz", 61 | "tiles": [ 62 | str(request.url_for("tiles", z="{z}", x="{x}", y="{y}")) + f"?url={url}" 63 | ], 64 | "minzoom": src.minzoom, 65 | "maxzoom": src.maxzoom, 66 | "bounds": src.bounds, 67 | "center": src.center, 68 | } 69 | 70 | # If Vector Tiles then we can try to add more metadata 71 | if src.is_vector: 72 | if vector_layers := meta.get("vector_layers"): 73 | tilejson["vector_layers"] = vector_layers 74 | 75 | return tilejson 76 | 77 | 78 | @app.get("/style.json") 79 | async def stylejson( 80 | request: Request, 81 | url: str = Query(..., description="PMTiles archive URL."), 82 | ): 83 | """get StyleJSON.""" 84 | tiles_url = str(request.url_for("tiles", z="{z}", x="{x}", y="{y}")) + f"?url={url}" 85 | 86 | async with Reader(url) as src: 87 | if src.is_vector: 88 | style_json = { 89 | "version": 8, 90 | "sources": { 91 | "pmtiles": { 92 | "type": "vector", 93 | "scheme": "xyz", 94 | "tiles": [tiles_url], 95 | "minzoom": src.minzoom, 96 | "maxzoom": src.maxzoom, 97 | "bounds": src.bounds, 98 | }, 99 | }, 100 | "layers": [], 101 | "center": [src.center[0], src.center[1]], 102 | "zoom": src.center[2], 103 | } 104 | 105 | meta = await src.metadata() 106 | if vector_layers := meta.get("vector_layers"): 107 | for layer in vector_layers: 108 | layer_id = layer["id"] 109 | if layer_id == "mask": 110 | style_json["layers"].append( 111 | { 112 | "id": f"{layer_id}_fill", 113 | "type": "fill", 114 | "source": "pmtiles", 115 | "source-layer": layer_id, 116 | "filter": ["==", ["geometry-type"], "Polygon"], 117 | "paint": { 118 | 'fill-color': 'black', 119 | 'fill-opacity': 0.8 120 | }, 121 | } 122 | ) 123 | 124 | else: 125 | style_json["layers"].append( 126 | { 127 | "id": f"{layer_id}_fill", 128 | "type": "fill", 129 | "source": "pmtiles", 130 | "source-layer": layer_id, 131 | "filter": ["==", ["geometry-type"], "Polygon"], 132 | "paint": { 133 | 'fill-color': 'rgba(200, 100, 240, 0.4)', 134 | 'fill-outline-color': '#000' 135 | }, 136 | } 137 | ) 138 | 139 | style_json["layers"].append( 140 | { 141 | "id": f"{layer_id}_stroke", 142 | "source": 'pmtiles', 143 | "source-layer": layer_id, 144 | "type": 'line', 145 | "filter": ["==", ["geometry-type"], "LineString"], 146 | "paint": { 147 | 'line-color': '#000', 148 | 'line-width': 1, 149 | 'line-opacity': 0.75 150 | } 151 | } 152 | ) 153 | style_json["layers"].append( 154 | { 155 | "id": f"{layer_id}_point", 156 | "source": 'pmtiles', 157 | "source-layer": layer_id, 158 | "type": 'circle', 159 | "filter": ["==", ["geometry-type"], "Point"], 160 | "paint": { 161 | 'circle-color': '#000', 162 | 'circle-radius': 2.5, 163 | 'circle-opacity': 0.75 164 | } 165 | } 166 | ) 167 | 168 | else: 169 | style_json = { 170 | "sources": { 171 | "pmtiles": { 172 | "type": "raster", 173 | "scheme": "xyz", 174 | "tiles": [tiles_url], 175 | "minzoom": src.minzoom, 176 | "maxzoom": src.maxzoom, 177 | "bounds": src.bounds, 178 | }, 179 | }, 180 | "layers": [ 181 | { 182 | "id": "raster", 183 | "type": "raster", 184 | "source": "pmtiles", 185 | }, 186 | ], 187 | "center": [src.center[0], src.center[1]], 188 | "zoom": src.center[2], 189 | } 190 | 191 | return style_json 192 | ``` 193 | --------------------------------------------------------------------------------