├── .github ├── dependabot.yml └── workflows │ ├── ci.yml │ └── dependabot.yml ├── .gitignore ├── .pre-commit-config.yaml ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── clippy.toml ├── fixtures ├── 0_0_0.png ├── 2_2_2.png ├── 3_4_5.png ├── leaf.pmtiles ├── protomaps(vector)ODbL_firenze.pmtiles └── stamen_toner(raster)CC-BY+ODbL_z3.pmtiles ├── justfile └── src ├── async_reader.rs ├── backend_aws_s3.rs ├── backend_http.rs ├── backend_mmap.rs ├── backend_s3.rs ├── cache.rs ├── directory.rs ├── error.rs ├── header.rs ├── lib.rs ├── tile.rs └── writer.rs /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | 4 | # Maintain dependencies for GitHub Actions 5 | - package-ecosystem: github-actions 6 | directory: "/" 7 | schedule: 8 | interval: weekly 9 | groups: 10 | all-actions-version-updates: 11 | applies-to: version-updates 12 | patterns: 13 | - "*" 14 | all-actions-security-updates: 15 | applies-to: security-updates 16 | patterns: 17 | - "*" 18 | 19 | # Update Rust dependencies 20 | - package-ecosystem: cargo 21 | directory: "/" 22 | schedule: 23 | interval: daily 24 | time: "02:00" 25 | open-pull-requests-limit: 10 26 | groups: 27 | all-cargo-version-updates: 28 | applies-to: version-updates 29 | patterns: 30 | - "*" 31 | all-cargo-security-updates: 32 | applies-to: security-updates 33 | patterns: 34 | - "*" 35 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | release: 9 | types: [ published ] 10 | workflow_dispatch: 11 | 12 | defaults: 13 | run: 14 | shell: bash 15 | 16 | jobs: 17 | test: 18 | name: Test 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | - if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' 23 | uses: Swatinem/rust-cache@v2 24 | - uses: taiki-e/install-action@v2 25 | with: { tool: just } 26 | - name: Ensure this crate has not yet been published 27 | run: just check-if-published 28 | - run: just ci-test 29 | - name: Check semver 30 | uses: obi1kenobi/cargo-semver-checks-action@v2 31 | with: 32 | feature-group: only-explicit-features 33 | features: __all_non_conflicting 34 | 35 | test-msrv: 36 | name: Test MSRV 37 | runs-on: ubuntu-latest 38 | steps: 39 | - uses: actions/checkout@v4 40 | - if: github.event_name != 'release' && github.event_name != 'workflow_dispatch' 41 | uses: Swatinem/rust-cache@v2 42 | - uses: taiki-e/install-action@v2 43 | with: { tool: just } 44 | - name: Read MSRV 45 | id: msrv 46 | run: echo "value=$(just get-msrv)" >> $GITHUB_OUTPUT 47 | - name: Install MSRV Rust ${{ steps.msrv.outputs.value }} 48 | uses: dtolnay/rust-toolchain@stable 49 | with: 50 | toolchain: ${{ steps.msrv.outputs.value }} 51 | - run: just ci_mode=0 ci-test-msrv # Ignore warnings in MSRV 52 | 53 | coverage: 54 | name: Code Coverage 55 | if: github.event_name != 'release' 56 | runs-on: ubuntu-latest 57 | steps: 58 | - uses: actions/checkout@v4 59 | - uses: Swatinem/rust-cache@v2 60 | - uses: taiki-e/install-action@v2 61 | with: { tool: 'just,cargo-llvm-cov' } 62 | - name: Generate code coverage 63 | run: just ci-coverage 64 | - name: Upload coverage to Codecov 65 | uses: codecov/codecov-action@v5 66 | with: 67 | token: ${{ secrets.CODECOV_TOKEN }} 68 | files: target/llvm-cov/codecov.info 69 | fail_ci_if_error: true 70 | 71 | # This job checks if any of the previous jobs failed or were canceled. 72 | # This approach also allows some jobs to be skipped if they are not needed. 73 | ci-passed: 74 | if: always() 75 | needs: [ test, test-msrv ] 76 | runs-on: ubuntu-latest 77 | steps: 78 | - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} 79 | run: exit 1 80 | 81 | release: 82 | # Some dependencies of the `ci-passed` job might be skipped, but we still want to run if the `ci-passed` job succeeded. 83 | if: always() && startsWith(github.ref, 'refs/tags/') && needs.ci-passed.result == 'success' 84 | name: Publish to crates.io 85 | needs: [ ci-passed ] 86 | runs-on: ubuntu-latest 87 | steps: 88 | - uses: actions/checkout@v4 89 | - name: Publish to crates.io 90 | run: cargo publish 91 | env: 92 | CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} 93 | -------------------------------------------------------------------------------- /.github/workflows/dependabot.yml: -------------------------------------------------------------------------------- 1 | name: Dependabot auto-merge 2 | on: pull_request 3 | 4 | permissions: write-all 5 | 6 | jobs: 7 | dependabot: 8 | runs-on: ubuntu-latest 9 | if: github.actor == 'dependabot[bot]' 10 | steps: 11 | - name: Dependabot metadata 12 | id: metadata 13 | uses: dependabot/fetch-metadata@v2 14 | with: 15 | github-token: ${{ secrets.GITHUB_TOKEN }} 16 | - name: Approve Dependabot PRs 17 | if: steps.metadata.outputs.update-type == 'version-update:semver-patch' 18 | run: gh pr review --approve "$PR_URL" 19 | env: 20 | PR_URL: ${{ github.event.pull_request.html_url }} 21 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 22 | - name: Enable auto-merge for Dependabot PRs 23 | if: steps.metadata.outputs.update-type == 'version-update:semver-patch' 24 | run: gh pr merge --auto --squash "$PR_URL" 25 | env: 26 | PR_URL: ${{ github.event.pull_request.html_url }} 27 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | /.idea 4 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v5.0.0 6 | hooks: 7 | - id: check-added-large-files 8 | - id: check-executables-have-shebangs 9 | - id: check-json 10 | - id: check-shebang-scripts-are-executable 11 | exclude: '.+\.rs' # would be triggered by #![some_attribute] 12 | - id: check-symlinks 13 | - id: check-toml 14 | - id: check-yaml 15 | args: [ --allow-multiple-documents ] 16 | - id: destroyed-symlinks 17 | - id: end-of-file-fixer 18 | - id: mixed-line-ending 19 | args: [ --fix=lf ] 20 | - id: trailing-whitespace 21 | 22 | - repo: https://github.com/Lucas-C/pre-commit-hooks 23 | rev: v1.5.5 24 | hooks: 25 | - id: forbid-tabs 26 | - id: remove-tabs 27 | 28 | - repo: local 29 | hooks: 30 | - id: cargo-fmt 31 | name: Rust Format 32 | description: "Automatically format Rust code with cargo fmt" 33 | entry: sh -c "cargo fmt --all" 34 | language: rust 35 | pass_filenames: false 36 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "pmtiles" 3 | version = "0.15.0" 4 | edition = "2024" 5 | authors = ["Luke Seelenbinder ", "Yuri Astrakhan "] 6 | license = "MIT OR Apache-2.0" 7 | description = "Implementation of the PMTiles v3 spec with multiple sync and async backends." 8 | repository = "https://github.com/stadiamaps/pmtiles-rs" 9 | keywords = ["pmtiles", "gis", "geo"] 10 | categories = ["science::geo"] 11 | rust-version = "1.85" 12 | 13 | [features] 14 | default = [] 15 | aws-s3-async = ["__async-aws-s3"] 16 | http-async = ["__async", "dep:reqwest"] 17 | iter-async = ["__async", "dep:async-stream", "dep:futures-util"] 18 | mmap-async-tokio = ["__async", "dep:fmmap", "fmmap?/tokio"] 19 | s3-async-native = ["__async-s3", "__async-s3-nativetls"] 20 | s3-async-rustls = ["__async-s3", "__async-s3-rustls"] 21 | tilejson = ["dep:tilejson", "dep:serde", "dep:serde_json"] 22 | write = ["dep:countio", "dep:flate2"] 23 | 24 | # Forward some of the common features to reqwest dependency 25 | reqwest-default = ["reqwest?/default"] 26 | reqwest-native-tls = ["reqwest?/native-tls"] 27 | reqwest-rustls-tls = ["reqwest?/rustls-tls"] 28 | reqwest-rustls-tls-native-roots = ["reqwest?/rustls-tls-native-roots"] 29 | reqwest-rustls-tls-webpki-roots = ["reqwest?/rustls-tls-webpki-roots"] 30 | 31 | #### These features are for the internal usage only 32 | # This is a list of features we use in docs.rs and other places where we want everything. 33 | # This list excludes these conflicting features: s3-async-native 34 | __all_non_conflicting = [ 35 | "aws-s3-async", 36 | "http-async", 37 | "iter-async", 38 | "mmap-async-tokio", 39 | "s3-async-rustls", 40 | "tilejson", 41 | "write", 42 | ] 43 | __async = ["dep:tokio", "async-compression/tokio"] 44 | __async-s3 = ["__async", "dep:rust-s3"] 45 | __async-s3-nativetls = ["rust-s3?/use-tokio-native-tls"] 46 | __async-s3-rustls = ["rust-s3?/tokio-rustls-tls"] 47 | __async-aws-s3 = ["__async", "dep:aws-sdk-s3"] 48 | 49 | [dependencies] 50 | # TODO: determine how we want to handle compression in async & sync environments 51 | async-compression = { version = "0.4", features = ["gzip"] } 52 | async-stream = { version = "0.3", optional = true } 53 | aws-sdk-s3 = { version = "1.49.0", optional = true } 54 | bytes = "1" 55 | countio = { version = "0.2.19", optional = true } 56 | fast_hilbert = "2.0.1" 57 | flate2 = { version = "1", optional = true } 58 | fmmap = { version = "0.4", default-features = false, optional = true } 59 | futures-util = { version = "0.3", optional = true } 60 | reqwest = { version = "0.12.4", default-features = false, optional = true } 61 | rust-s3 = { version = "0.35.1", optional = true, default-features = false, features = ["fail-on-err"] } 62 | serde = { version = "1", optional = true } 63 | serde_json = { version = "1", optional = true } 64 | thiserror = "2" 65 | tilejson = { version = "0.4", optional = true } 66 | tokio = { version = "1", default-features = false, features = ["io-util"], optional = true } 67 | varint-rs = "2" 68 | 69 | [dev-dependencies] 70 | flate2 = "1" 71 | fmmap = { version = "0.4", features = ["tokio"] } 72 | reqwest = { version = "0.12.4", features = ["rustls-tls-webpki-roots"] } 73 | tempfile = "3.13.0" 74 | tokio = { version = "1", features = ["test-util", "macros", "rt"] } 75 | 76 | [package.metadata.docs.rs] 77 | features = ["__all_non_conflicting"] 78 | 79 | [lints.rust] 80 | unsafe_code = "forbid" 81 | unused_qualifications = "warn" 82 | 83 | [lints.clippy] 84 | pedantic = { level = "warn", priority = -1 } 85 | missing_errors_doc = "allow" 86 | module_name_repetitions = "allow" 87 | panic_in_result_fn = "warn" 88 | similar_names = "allow" 89 | todo = "warn" 90 | unwrap_used = "warn" 91 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | Copyright 2022 Stadia Maps, Inc. 179 | 180 | Licensed under the Apache License, Version 2.0 (the "License"); 181 | you may not use this file except in compliance with the License. 182 | You may obtain a copy of the License at 183 | 184 | http://www.apache.org/licenses/LICENSE-2.0 185 | 186 | Unless required by applicable law or agreed to in writing, software 187 | distributed under the License is distributed on an "AS IS" BASIS, 188 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 | See the License for the specific language governing permissions and 190 | limitations under the License. 191 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Stadia Maps, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `PMTiles` (for Rust) 2 | 3 | [![GitHub repo](https://img.shields.io/badge/github-stadiamaps/pmtiles--rs-8da0cb?logo=github)](https://github.com/stadiamaps/pmtiles-rs) 4 | [![crates.io version](https://img.shields.io/crates/v/pmtiles)](https://crates.io/crates/pmtiles) 5 | [![docs.rs status](https://img.shields.io/docsrs/pmtiles)](https://docs.rs/pmtiles) 6 | [![crates.io license](https://img.shields.io/crates/l/pmtiles)](https://github.com/stadiamaps/pmtiles-rs/blob/main/LICENSE-APACHE) 7 | [![CI build status](https://github.com/stadiamaps/pmtiles-rs/actions/workflows/ci.yml/badge.svg)](https://github.com/stadiamaps/pmtiles-rs/actions) 8 | [![Codecov](https://img.shields.io/codecov/c/github/stadiamaps/pmtiles-rs)](https://app.codecov.io/gh/stadiamaps/pmtiles-rs) 9 | 10 | This crate implements the [PMTiles v3 spec](https://github.com/protomaps/PMTiles/blob/master/spec/v3/spec.md), 11 | originally created by Brandon Liu for Protomaps. 12 | 13 | ## Features 14 | 15 | - Opening and validating `PMTile` archives 16 | - Querying tiles 17 | - Backends supported: 18 | - Async `mmap` (Tokio) for local files 19 | - Async `http` and `https` (Reqwest + Tokio) for URLs 20 | - Async `s3` (Rust-S3 + Tokio) for S3-compatible buckets 21 | - Creating `PMTile` archives 22 | 23 | ## Plans & TODOs 24 | 25 | - [ ] Documentation and example code 26 | - [ ] Support conversion to and from `MBTiles` + `x/y/z` 27 | - [ ] Support additional backends (sync `mmap` and `http` at least) 28 | - [ ] Support additional async styles (e.g., `async-std`) 29 | 30 | PRs welcome! 31 | 32 | ## Usage examples 33 | 34 | ### Reading from a local `PMTiles` file 35 | 36 | ```rust,no_run 37 | use bytes::Bytes; 38 | use pmtiles::{AsyncPmTilesReader, TileCoord}; 39 | 40 | async fn get_tile(z: u8, x: u32, y: u32) -> Option { 41 | let file = "example.pmtiles"; 42 | // Use `new_with_cached_path` for better performance 43 | let reader = AsyncPmTilesReader::new_with_path(file).await.unwrap(); 44 | let coord = TileCoord::new(z, x, y).unwrap(); 45 | reader.get_tile(coord).await.unwrap() 46 | } 47 | ``` 48 | 49 | ### Reading from a URL with a simple directory cache 50 | 51 | This example uses a simple hashmap-based cache to optimize reads from a `PMTiles` source. The same caching is available for all other methods. Note that `HashMapCache` is a rudimentary cache without eviction. You may want to build a more sophisticated cache for production use by implementing the `DirectoryCache` trait. 52 | 53 | ```rust,no_run 54 | use bytes::Bytes; 55 | use pmtiles::{AsyncPmTilesReader, HashMapCache, TileCoord}; 56 | use pmtiles::reqwest::Client; // Re-exported Reqwest crate 57 | 58 | async fn get_tile(z: u8, x: u32, y: u32) -> Option { 59 | let cache = HashMapCache::default(); 60 | let client = Client::builder().use_rustls_tls().build().unwrap(); 61 | let url = "https://protomaps.github.io/PMTiles/protomaps(vector)ODbL_firenze.pmtiles"; 62 | let reader = AsyncPmTilesReader::new_with_cached_url(cache, client, url).await.unwrap(); 63 | let coord = TileCoord::new(z, x, y).unwrap(); 64 | reader.get_tile(coord).await.unwrap() 65 | } 66 | ``` 67 | 68 | ### Reading from an S3 bucket with a directory cache 69 | 70 | AWS client configuration is fairly none-trivial to document here. See AWS SDK [documentation](https://crates.io/crates/aws-sdk-s3) for more details. 71 | 72 | ```rust,no_run 73 | use bytes::Bytes; 74 | use pmtiles::{AsyncPmTilesReader, HashMapCache, TileCoord}; 75 | use pmtiles::aws_sdk_s3::Client; // Re-exported AWS SDK S3 client 76 | 77 | async fn get_tile(client: Client, z: u8, x: u32, y: u32) -> Option { 78 | let cache = HashMapCache::default(); 79 | let bucket = "https://s3.example.com".to_string(); 80 | let key = "example.pmtiles".to_string(); 81 | let reader = AsyncPmTilesReader::new_with_cached_client_bucket_and_path(cache, client, bucket, key).await.unwrap(); 82 | let coord = TileCoord::new(z, x, y).unwrap(); 83 | reader.get_tile(coord).await.unwrap() 84 | } 85 | ``` 86 | 87 | ### Writing to a `PMTiles` file 88 | 89 | ```rust,no_run 90 | use pmtiles::{PmTilesWriter, TileType, TileCoord}; 91 | use std::fs::File; 92 | 93 | let file = File::create("example.pmtiles").unwrap(); 94 | let mut writer = PmTilesWriter::new(TileType::Mvt).create(file).unwrap(); 95 | let coord = TileCoord::new(0, 0, 0).unwrap(); 96 | writer.add_tile(coord, &[/*...*/]).unwrap(); 97 | writer.finalize().unwrap(); 98 | ``` 99 | 100 | ## Development 101 | 102 | * This project is easier to develop with [just](https://github.com/casey/just#readme), a modern alternative to `make`. 103 | Install it with `cargo install just`. 104 | * To get a list of available commands, run `just`. 105 | * To run tests, use `just test`. 106 | 107 | ## License 108 | 109 | Licensed under either of 110 | 111 | * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or ) 112 | * MIT license ([LICENSE-MIT](LICENSE-MIT) or ) 113 | at your option. 114 | 115 | ### Test Data License 116 | 117 | Some `PMTile` fixtures copied from official [PMTiles repository](https://github.com/protomaps/PMTiles/commit/257b41dd0497e05d1d686aa92ce2f742b6251644). 118 | 119 | ### Contribution 120 | 121 | Unless you explicitly state otherwise, any contribution intentionally 122 | submitted for inclusion in the work by you, as defined in the 123 | Apache-2.0 license, shall be dual licensed as above, without any 124 | additional terms or conditions. 125 | -------------------------------------------------------------------------------- /clippy.toml: -------------------------------------------------------------------------------- 1 | allow-unwrap-in-tests = true 2 | avoid-breaking-exported-api = false 3 | -------------------------------------------------------------------------------- /fixtures/0_0_0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stadiamaps/pmtiles-rs/a41a63a6b0e73d4a8ff4f621805ce9aee4a49bac/fixtures/0_0_0.png -------------------------------------------------------------------------------- /fixtures/2_2_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stadiamaps/pmtiles-rs/a41a63a6b0e73d4a8ff4f621805ce9aee4a49bac/fixtures/2_2_2.png -------------------------------------------------------------------------------- /fixtures/3_4_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stadiamaps/pmtiles-rs/a41a63a6b0e73d4a8ff4f621805ce9aee4a49bac/fixtures/3_4_5.png -------------------------------------------------------------------------------- /fixtures/leaf.pmtiles: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stadiamaps/pmtiles-rs/a41a63a6b0e73d4a8ff4f621805ce9aee4a49bac/fixtures/leaf.pmtiles -------------------------------------------------------------------------------- /fixtures/protomaps(vector)ODbL_firenze.pmtiles: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stadiamaps/pmtiles-rs/a41a63a6b0e73d4a8ff4f621805ce9aee4a49bac/fixtures/protomaps(vector)ODbL_firenze.pmtiles -------------------------------------------------------------------------------- /fixtures/stamen_toner(raster)CC-BY+ODbL_z3.pmtiles: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stadiamaps/pmtiles-rs/a41a63a6b0e73d4a8ff4f621805ce9aee4a49bac/fixtures/stamen_toner(raster)CC-BY+ODbL_z3.pmtiles -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env just --justfile 2 | 3 | main_crate := 'pmtiles' 4 | features_flag := '--features __all_non_conflicting' 5 | 6 | # if running in CI, treat warnings as errors by setting RUSTFLAGS and RUSTDOCFLAGS to '-D warnings' unless they are already set 7 | # Use `CI=true just ci-test` to run the same tests as in GitHub CI. 8 | # Use `just env-info` to see the current values of RUSTFLAGS and RUSTDOCFLAGS 9 | ci_mode := if env('CI', '') != '' {'1'} else {''} 10 | export RUSTFLAGS := env('RUSTFLAGS', if ci_mode == '1' {'-D warnings'} else {''}) 11 | export RUSTDOCFLAGS := env('RUSTDOCFLAGS', if ci_mode == '1' {'-D warnings'} else {''}) 12 | export RUST_BACKTRACE := env('RUST_BACKTRACE', if ci_mode == '1' {'1'} else {''}) 13 | 14 | @_default: 15 | {{just_executable()}} --list 16 | 17 | # Build the project 18 | build: 19 | cargo build --workspace --all-targets {{features_flag}} 20 | 21 | # Quick compile without building a binary 22 | check: 23 | cargo check --workspace --all-targets {{features_flag}} 24 | @echo "-------------- Checking individual crate features" 25 | cargo check --workspace --all-targets --features aws-s3-async 26 | cargo check --workspace --all-targets --features http-async 27 | cargo check --workspace --all-targets --features iter-async 28 | cargo check --workspace --all-targets --features mmap-async-tokio 29 | cargo check --workspace --all-targets --features s3-async-native 30 | cargo check --workspace --all-targets --features s3-async-rustls 31 | cargo check --workspace --all-targets --features tilejson 32 | cargo check --workspace --all-targets --features write 33 | 34 | # Verify that the current version of the crate is not the same as the one published on crates.io 35 | check-if-published package=main_crate: 36 | #!/usr/bin/env bash 37 | set -euo pipefail 38 | LOCAL_VERSION="$({{just_executable()}} get-crate-field version {{package}})" 39 | echo "Detected crate {{package}} version: '$LOCAL_VERSION'" 40 | PUBLISHED_VERSION="$(cargo search --quiet {{package}} | grep "^{{package}} =" | sed -E 's/.* = "(.*)".*/\1/')" 41 | echo "Published crate version: '$PUBLISHED_VERSION'" 42 | if [ "$LOCAL_VERSION" = "$PUBLISHED_VERSION" ]; then 43 | echo "ERROR: The current crate version has already been published." 44 | exit 1 45 | else 46 | echo "The current crate version has not yet been published." 47 | fi 48 | 49 | # Generate code coverage report to upload to codecov.io 50 | ci-coverage: env-info && \ 51 | (coverage '--codecov --output-path target/llvm-cov/codecov.info') 52 | # ATTENTION: the full file path above is used in the CI workflow 53 | mkdir -p target/llvm-cov 54 | 55 | # Run all tests as expected by CI 56 | ci-test: env-info test-fmt clippy check test test-doc && assert-git-is-clean 57 | 58 | # Run minimal subset of tests to ensure compatibility with MSRV 59 | ci-test-msrv: env-info test 60 | 61 | # Clean all build artifacts 62 | clean: 63 | cargo clean 64 | rm -f Cargo.lock 65 | 66 | # Run cargo clippy to lint the code 67 | clippy *args: 68 | cargo clippy --workspace --all-targets {{features_flag}} {{args}} 69 | cargo clippy --workspace --all-targets --features s3-async-native {{args}} 70 | 71 | # Generate code coverage report. Will install `cargo llvm-cov` if missing. 72 | coverage *args='--no-clean --open': (cargo-install 'cargo-llvm-cov') 73 | cargo llvm-cov --workspace --all-targets {{features_flag}} --include-build-script {{args}} 74 | 75 | # Build and open code documentation 76 | docs *args='--open': 77 | DOCS_RS=1 cargo doc --no-deps {{args}} --workspace {{features_flag}} 78 | 79 | # Print environment info 80 | env-info: 81 | @echo "Running {{if ci_mode == '1' {'in CI mode'} else {'in dev mode'} }} on {{os()}} / {{arch()}}" 82 | {{just_executable()}} --version 83 | rustc --version 84 | cargo --version 85 | rustup --version 86 | @echo "RUSTFLAGS='$RUSTFLAGS'" 87 | @echo "RUSTDOCFLAGS='$RUSTDOCFLAGS'" 88 | 89 | # Reformat all code `cargo fmt`. If nightly is available, use it for better results 90 | fmt: 91 | #!/usr/bin/env bash 92 | set -euo pipefail 93 | if rustup component list --toolchain nightly | grep rustfmt &> /dev/null; then 94 | echo 'Reformatting Rust code using nightly Rust fmt to sort imports' 95 | cargo +nightly fmt --all -- --config imports_granularity=Module,group_imports=StdExternalCrate 96 | else 97 | echo 'Reformatting Rust with the stable cargo fmt. Install nightly with `rustup install nightly` for better results' 98 | cargo fmt --all 99 | fi 100 | 101 | # Get any package's field from the metadata 102 | get-crate-field field package=main_crate: (assert-cmd 'jq') 103 | cargo metadata --format-version 1 | jq -e -r '.packages | map(select(.name == "{{package}}")) | first | .{{field}} | select(. != null)' 104 | 105 | # Get the minimum supported Rust version (MSRV) for the crate 106 | get-msrv package=main_crate: (get-crate-field 'rust_version' package) 107 | 108 | # Find the minimum supported Rust version (MSRV) using cargo-msrv extension, and update Cargo.toml 109 | msrv: (cargo-install 'cargo-msrv') 110 | cargo msrv find --write-msrv --ignore-lockfile {{features_flag}} 111 | 112 | # Check semver compatibility with prior published version. Install it with `cargo install cargo-semver-checks` 113 | semver *args: (cargo-install 'cargo-semver-checks') 114 | cargo semver-checks {{features_flag}} {{args}} 115 | 116 | # Run all unit and integration tests 117 | test: 118 | cargo test --workspace --all-targets {{features_flag}} 119 | cargo test --workspace --doc {{features_flag}} 120 | @echo "-------------- Testing individual crate features" 121 | cargo test --workspace --all-targets --features aws-s3-async 122 | cargo test --workspace --all-targets --features http-async 123 | cargo test --workspace --all-targets --features iter-async 124 | cargo test --workspace --all-targets --features mmap-async-tokio 125 | cargo test --workspace --all-targets --features s3-async-native 126 | cargo test --workspace --all-targets --features s3-async-rustls 127 | cargo test --workspace --all-targets --features tilejson 128 | cargo test --workspace --all-targets --features write 129 | 130 | # Test documentation generation 131 | test-doc: (docs '') 132 | 133 | # Test code formatting 134 | test-fmt: 135 | cargo fmt --all -- --check 136 | 137 | # Find unused dependencies. Install it with `cargo install cargo-udeps` 138 | udeps: (cargo-install 'cargo-udeps') 139 | cargo +nightly udeps --workspace --all-targets {{features_flag}} 140 | 141 | # Update all dependencies, including breaking changes. Requires nightly toolchain (install with `rustup install nightly`) 142 | update: 143 | cargo +nightly -Z unstable-options update --breaking 144 | cargo update 145 | 146 | # Ensure that a certain command is available 147 | [private] 148 | assert-cmd command: 149 | @if ! type {{command}} > /dev/null; then \ 150 | echo "Command '{{command}}' could not be found. Please make sure it has been installed on your computer." ;\ 151 | exit 1 ;\ 152 | fi 153 | 154 | # Make sure the git repo has no uncommitted changes 155 | [private] 156 | assert-git-is-clean: 157 | @if [ -n "$(git status --untracked-files --porcelain)" ]; then \ 158 | >&2 echo "ERROR: git repo is no longer clean. Make sure compilation and tests artifacts are in the .gitignore, and no repo files are modified." ;\ 159 | >&2 echo "######### git status ##########" ;\ 160 | git status ;\ 161 | git --no-pager diff ;\ 162 | exit 1 ;\ 163 | fi 164 | 165 | # Check if a certain Cargo command is installed, and install it if needed 166 | [private] 167 | cargo-install $COMMAND $INSTALL_CMD='' *args='': 168 | #!/usr/bin/env bash 169 | set -euo pipefail 170 | if ! command -v $COMMAND > /dev/null; then 171 | if ! command -v cargo-binstall > /dev/null; then 172 | echo "$COMMAND could not be found. Installing it with cargo install ${INSTALL_CMD:-$COMMAND} --locked {{args}}" 173 | cargo install ${INSTALL_CMD:-$COMMAND} --locked {{args}} 174 | else 175 | echo "$COMMAND could not be found. Installing it with cargo binstall ${INSTALL_CMD:-$COMMAND} --locked {{args}}" 176 | cargo binstall ${INSTALL_CMD:-$COMMAND} --locked {{args}} 177 | fi 178 | fi 179 | -------------------------------------------------------------------------------- /src/async_reader.rs: -------------------------------------------------------------------------------- 1 | // FIXME: This seems like a bug - there are lots of u64 to usize conversions in this file, 2 | // so any file larger than 4GB, or an untrusted file with bad data may crash. 3 | #![expect(clippy::cast_possible_truncation)] 4 | 5 | use std::future::Future; 6 | #[cfg(feature = "iter-async")] 7 | use std::sync::Arc; 8 | 9 | #[cfg(feature = "iter-async")] 10 | use async_stream::try_stream; 11 | use bytes::Bytes; 12 | #[cfg(feature = "iter-async")] 13 | use futures_util::stream::BoxStream; 14 | #[cfg(feature = "__async")] 15 | use tokio::io::AsyncReadExt as _; 16 | 17 | use crate::PmtError::UnsupportedCompression; 18 | use crate::header::{HEADER_SIZE, MAX_INITIAL_BYTES}; 19 | use crate::{ 20 | Compression, DirCacheResult, DirEntry, Directory, Header, PmtError, PmtResult, TileId, 21 | }; 22 | #[cfg(feature = "__async")] 23 | use crate::{DirectoryCache, NoCache}; 24 | 25 | pub struct AsyncPmTilesReader { 26 | backend: B, 27 | cache: C, 28 | header: Header, 29 | root_directory: Directory, 30 | } 31 | 32 | impl AsyncPmTilesReader { 33 | /// Creates a new reader from a specified source and validates the provided `PMTiles` archive is valid. 34 | /// 35 | /// Note: Prefer using `new_with_*` methods. 36 | pub async fn try_from_source(backend: B) -> PmtResult { 37 | Self::try_from_cached_source(backend, NoCache).await 38 | } 39 | } 40 | 41 | impl AsyncPmTilesReader { 42 | /// Creates a new cached reader from a specified source and validates the provided `PMTiles` archive is valid. 43 | /// 44 | /// Note: Prefer using `new_with_*` methods. 45 | pub async fn try_from_cached_source(backend: B, cache: C) -> PmtResult { 46 | // Read the first 127 and up to 16,384 bytes to ensure we can initialize the header and root directory. 47 | let mut initial_bytes = backend.read(0, MAX_INITIAL_BYTES).await?; 48 | if initial_bytes.len() < HEADER_SIZE { 49 | return Err(PmtError::InvalidHeader); 50 | } 51 | 52 | let header = Header::try_from_bytes(initial_bytes.split_to(HEADER_SIZE))?; 53 | 54 | let directory_bytes = initial_bytes 55 | .split_off((header.root_offset as usize) - HEADER_SIZE) 56 | .split_to(header.root_length as _); 57 | 58 | let root_directory = 59 | Self::read_compressed_directory(header.internal_compression, directory_bytes).await?; 60 | 61 | Ok(Self { 62 | backend, 63 | cache, 64 | header, 65 | root_directory, 66 | }) 67 | } 68 | 69 | /// Fetches tile data using either [`TileCoord`](crate::TileCoord) or [`TileId`] to locate the tile. 70 | /// 71 | /// ```no_run 72 | /// # async fn test() { 73 | /// # let backend = pmtiles::MmapBackend::try_from("").await.unwrap(); 74 | /// # let reader = pmtiles::AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 75 | /// // Using a tile (z, x, y) coordinate to fetch a tile 76 | /// let coord = pmtiles::TileCoord::new(0, 0, 0).unwrap(); 77 | /// let tile = reader.get_tile(coord).await.unwrap(); 78 | /// // Using a tile ID to fetch a tile 79 | /// let tile_id = pmtiles::TileId::from(coord); 80 | /// let tile = reader.get_tile(tile_id).await.unwrap(); 81 | /// # } 82 | /// ``` 83 | pub async fn get_tile>(&self, tile_id: Id) -> PmtResult> { 84 | let Some(entry) = self.find_tile_entry(tile_id.into()).await? else { 85 | return Ok(None); 86 | }; 87 | 88 | let offset = (self.header.data_offset + entry.offset) as _; 89 | let length = entry.length as _; 90 | 91 | Ok(Some(self.backend.read_exact(offset, length).await?)) 92 | } 93 | 94 | /// Fetches tile bytes from the archive. 95 | /// If the tile is compressed, it will be decompressed. 96 | pub async fn get_tile_decompressed>( 97 | &self, 98 | tile_id: Id, 99 | ) -> PmtResult> { 100 | Ok(if let Some(data) = self.get_tile(tile_id).await? { 101 | Some(Self::decompress(self.header.tile_compression, data).await?) 102 | } else { 103 | None 104 | }) 105 | } 106 | 107 | /// Access header information. 108 | pub fn get_header(&self) -> &Header { 109 | &self.header 110 | } 111 | 112 | /// Gets metadata from the archive. 113 | /// 114 | /// Note: by spec, this should be valid JSON. This method currently returns a [String]. 115 | /// This may change in the future. 116 | pub async fn get_metadata(&self) -> PmtResult { 117 | let offset = self.header.metadata_offset as _; 118 | let length = self.header.metadata_length as _; 119 | let metadata = self.backend.read_exact(offset, length).await?; 120 | 121 | let decompressed_metadata = 122 | Self::decompress(self.header.internal_compression, metadata).await?; 123 | 124 | Ok(String::from_utf8(decompressed_metadata.to_vec())?) 125 | } 126 | 127 | /// Return an async stream over all tile entries in the archive. Directory entries are traversed, and not included in the result. 128 | /// Because this function requires the reader for the duration of the stream, you need to wrap the reader in an `Arc`. 129 | /// 130 | /// ``` 131 | /// # use std::sync::Arc; 132 | /// # use pmtiles::{AsyncPmTilesReader, MmapBackend}; 133 | /// # use futures_util::TryStreamExt as _; 134 | /// #[tokio::main(flavor="current_thread")] 135 | /// async fn main() -> Result<(), pmtiles::PmtError> { 136 | /// let backend = MmapBackend::try_from("fixtures/protomaps(vector)ODbL_firenze.pmtiles").await?; 137 | /// let reader = Arc::new(AsyncPmTilesReader::try_from_source(backend).await?); 138 | /// let mut entries = reader.entries(); 139 | /// while let Some(entry) = entries.try_next().await? { 140 | /// // ... do something with entry ... 141 | /// } 142 | /// Ok(()) 143 | /// } 144 | /// ``` 145 | #[cfg(feature = "iter-async")] 146 | pub fn entries<'a>(self: Arc) -> BoxStream<'a, PmtResult> 147 | where 148 | B: 'a, 149 | C: 'a, 150 | { 151 | Box::pin(try_stream! { 152 | let mut queue = std::collections::VecDeque::new(); 153 | 154 | for entry in &self.root_directory.entries { 155 | queue.push_back(entry.clone()); 156 | } 157 | 158 | while let Some(entry) = queue.pop_front() { 159 | if entry.is_leaf() { 160 | let offset = (self.header.leaf_offset + entry.offset) as _; 161 | let length = entry.length as usize; 162 | let leaf_dir = self.read_directory(offset, length).await?; 163 | // enqueue all entries in the leaf directory 164 | for leaf_entry in leaf_dir.entries { 165 | queue.push_back(leaf_entry); 166 | } 167 | } else { 168 | yield entry; 169 | } 170 | } 171 | }) 172 | } 173 | 174 | #[cfg(feature = "tilejson")] 175 | pub async fn parse_tilejson(&self, sources: Vec) -> PmtResult { 176 | use serde_json::Value; 177 | 178 | let meta = self.get_metadata().await?; 179 | let meta: Value = serde_json::from_str(&meta).map_err(|_| PmtError::InvalidMetadata)?; 180 | let Value::Object(meta) = meta else { 181 | return Err(PmtError::InvalidMetadata); 182 | }; 183 | 184 | let mut tj = self.header.get_tilejson(sources); 185 | for (key, value) in meta { 186 | if let Value::String(v) = value { 187 | if key == "description" { 188 | tj.description = Some(v); 189 | } else if key == "attribution" { 190 | tj.attribution = Some(v); 191 | } else if key == "legend" { 192 | tj.legend = Some(v); 193 | } else if key == "name" { 194 | tj.name = Some(v); 195 | } else if key == "version" { 196 | tj.version = Some(v); 197 | } else if key == "minzoom" || key == "maxzoom" { 198 | // We already have the correct values from the header, so just drop these 199 | // attributes from the metadata silently, don't overwrite known-good values. 200 | } else { 201 | tj.other.insert(key, Value::String(v)); 202 | } 203 | } else if key == "vector_layers" { 204 | if let Ok(v) = serde_json::from_value::>(value) { 205 | tj.vector_layers = Some(v); 206 | } else { 207 | return Err(PmtError::InvalidMetadata); 208 | } 209 | } else { 210 | tj.other.insert(key, value); 211 | } 212 | } 213 | Ok(tj) 214 | } 215 | 216 | /// Recursively locates a tile in the archive. 217 | async fn find_tile_entry(&self, tile_id: TileId) -> PmtResult> { 218 | let entry = self.root_directory.find_tile_id(tile_id); 219 | if let Some(entry) = entry { 220 | if entry.is_leaf() { 221 | return self.find_entry_rec(tile_id, entry, 0).await; 222 | } 223 | } 224 | 225 | Ok(entry.cloned()) 226 | } 227 | 228 | async fn find_entry_rec( 229 | &self, 230 | tile_id: TileId, 231 | entry: &DirEntry, 232 | depth: u8, 233 | ) -> PmtResult> { 234 | // the recursion is done as two functions because it is a bit cleaner, 235 | // and it allows the directory to be cached later without cloning it first. 236 | let offset = (self.header.leaf_offset + entry.offset) as _; 237 | 238 | let entry = match self.cache.get_dir_entry(offset, tile_id).await { 239 | DirCacheResult::NotCached => { 240 | // Cache miss - read from backend 241 | let length = entry.length as _; 242 | let dir = self.read_directory(offset, length).await?; 243 | let entry = dir.find_tile_id(tile_id).cloned(); 244 | self.cache.insert_dir(offset, dir).await; 245 | entry 246 | } 247 | DirCacheResult::NotFound => None, 248 | DirCacheResult::Found(entry) => Some(entry), 249 | }; 250 | 251 | if let Some(ref entry) = entry { 252 | if entry.is_leaf() { 253 | return if depth <= 4 { 254 | Box::pin(self.find_entry_rec(tile_id, entry, depth + 1)).await 255 | } else { 256 | Ok(None) 257 | }; 258 | } 259 | } 260 | 261 | Ok(entry) 262 | } 263 | 264 | async fn read_directory(&self, offset: usize, length: usize) -> PmtResult { 265 | let data = self.backend.read_exact(offset, length).await?; 266 | Self::read_compressed_directory(self.header.internal_compression, data).await 267 | } 268 | 269 | async fn read_compressed_directory( 270 | compression: Compression, 271 | bytes: Bytes, 272 | ) -> PmtResult { 273 | let decompressed_bytes = Self::decompress(compression, bytes).await?; 274 | Directory::try_from(decompressed_bytes) 275 | } 276 | 277 | async fn decompress(compression: Compression, bytes: Bytes) -> PmtResult { 278 | if compression == Compression::None { 279 | return Ok(bytes); 280 | } 281 | 282 | let mut decompressed_bytes = Vec::with_capacity(bytes.len() * 2); 283 | match compression { 284 | Compression::Gzip => { 285 | async_compression::tokio::bufread::GzipDecoder::new(&bytes[..]) 286 | .read_to_end(&mut decompressed_bytes) 287 | .await?; 288 | } 289 | Compression::None => { 290 | return Ok(bytes); 291 | } 292 | v => Err(UnsupportedCompression(v))?, 293 | } 294 | 295 | Ok(Bytes::from(decompressed_bytes)) 296 | } 297 | } 298 | 299 | pub trait AsyncBackend { 300 | /// Reads exactly `length` bytes starting at `offset` 301 | fn read_exact( 302 | &self, 303 | offset: usize, 304 | length: usize, 305 | ) -> impl Future> + Send 306 | where 307 | Self: Sync, 308 | { 309 | async move { 310 | let data = self.read(offset, length).await?; 311 | 312 | if data.len() == length { 313 | Ok(data) 314 | } else { 315 | Err(PmtError::UnexpectedNumberOfBytesReturned( 316 | length, 317 | data.len(), 318 | )) 319 | } 320 | } 321 | } 322 | 323 | /// Reads up to `length` bytes starting at `offset`. 324 | fn read(&self, offset: usize, length: usize) -> impl Future> + Send; 325 | } 326 | 327 | #[cfg(test)] 328 | #[cfg(feature = "mmap-async-tokio")] 329 | mod tests { 330 | use crate::tests::{RASTER_FILE, VECTOR_FILE}; 331 | use crate::{AsyncPmTilesReader, MmapBackend, TileCoord}; 332 | 333 | fn id(z: u8, x: u32, y: u32) -> TileCoord { 334 | TileCoord::new(z, x, y).unwrap() 335 | } 336 | 337 | #[tokio::test] 338 | async fn open_sanity_check() { 339 | let backend = MmapBackend::try_from(RASTER_FILE).await.unwrap(); 340 | AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 341 | } 342 | 343 | async fn compare_tiles(z: u8, x: u32, y: u32, fixture_bytes: &[u8]) { 344 | let backend = MmapBackend::try_from(RASTER_FILE).await.unwrap(); 345 | let tiles = AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 346 | let tile = tiles 347 | .get_tile_decompressed(id(z, x, y)) 348 | .await 349 | .unwrap() 350 | .unwrap(); 351 | 352 | assert_eq!( 353 | tile.len(), 354 | fixture_bytes.len(), 355 | "Expected tile length to match." 356 | ); 357 | assert_eq!(tile, fixture_bytes, "Expected tile to match fixture."); 358 | } 359 | 360 | #[tokio::test] 361 | async fn get_first_tile() { 362 | let fixture_tile = include_bytes!("../fixtures/0_0_0.png"); 363 | compare_tiles(0, 0, 0, fixture_tile).await; 364 | } 365 | 366 | #[tokio::test] 367 | async fn get_another_tile() { 368 | let fixture_tile = include_bytes!("../fixtures/2_2_2.png"); 369 | compare_tiles(2, 2, 2, fixture_tile).await; 370 | } 371 | 372 | #[tokio::test] 373 | async fn get_yet_another_tile() { 374 | let fixture_tile = include_bytes!("../fixtures/3_4_5.png"); 375 | compare_tiles(3, 4, 5, fixture_tile).await; 376 | } 377 | 378 | #[tokio::test] 379 | async fn test_missing_tile() { 380 | let backend = MmapBackend::try_from(VECTOR_FILE).await.unwrap(); 381 | let tiles = AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 382 | 383 | let tile = tiles.get_tile(id(6, 31, 23)).await; 384 | assert!(tile.is_ok()); 385 | assert!(tile.unwrap().is_none()); 386 | } 387 | 388 | #[tokio::test] 389 | async fn test_leaf_tile() { 390 | let backend = MmapBackend::try_from(VECTOR_FILE).await.unwrap(); 391 | let tiles = AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 392 | 393 | let tile = tiles.get_tile(id(12, 2174, 1492)).await; 394 | assert!(tile.is_ok_and(|t| t.is_some())); 395 | } 396 | 397 | #[tokio::test] 398 | async fn test_leaf_tile_compressed() { 399 | let backend = MmapBackend::try_from(VECTOR_FILE).await.unwrap(); 400 | let tiles = AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 401 | let coord = id(12, 2174, 1492); 402 | 403 | let tile = tiles.get_tile(coord).await; 404 | assert!(tile.as_ref().is_ok_and(Option::is_some)); 405 | let tile = tile.unwrap().unwrap(); 406 | 407 | let tile_dec = tiles.get_tile_decompressed(coord).await; 408 | assert!(tile_dec.as_ref().is_ok_and(Option::is_some)); 409 | let tile_dec = tile_dec.unwrap().unwrap(); 410 | 411 | assert!( 412 | tile_dec.len() > tile.len(), 413 | "Decompressed tile should be larger than compressed tile" 414 | ); 415 | } 416 | 417 | #[tokio::test] 418 | async fn test_get_metadata() { 419 | let backend = MmapBackend::try_from(VECTOR_FILE).await.unwrap(); 420 | let tiles = AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 421 | 422 | let metadata = tiles.get_metadata().await.unwrap(); 423 | assert!(!metadata.is_empty()); 424 | } 425 | 426 | #[tokio::test] 427 | #[cfg(feature = "tilejson")] 428 | async fn test_parse_tilejson() { 429 | let backend = MmapBackend::try_from(VECTOR_FILE).await.unwrap(); 430 | let tiles = AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 431 | 432 | let tj = tiles.parse_tilejson(Vec::new()).await.unwrap(); 433 | assert!(tj.attribution.is_some()); 434 | } 435 | 436 | #[tokio::test] 437 | #[cfg(feature = "tilejson")] 438 | async fn test_parse_tilejson2() { 439 | let backend = MmapBackend::try_from(RASTER_FILE).await.unwrap(); 440 | let tiles = AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 441 | 442 | let tj = tiles.parse_tilejson(Vec::new()).await.unwrap(); 443 | assert!(tj.other.is_empty()); 444 | } 445 | 446 | #[tokio::test] 447 | async fn test_martin_675() { 448 | let backend = MmapBackend::try_from("fixtures/leaf.pmtiles") 449 | .await 450 | .unwrap(); 451 | let tiles = AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 452 | // Verify that the test case does contain a leaf directory 453 | assert_ne!(0, tiles.get_header().leaf_length); 454 | for (contents, z, x, y) in [ 455 | (b"0", 0, 0, 0), 456 | (b"1", 1, 0, 0), 457 | (b"2", 1, 0, 1), 458 | (b"3", 1, 1, 1), 459 | (b"4", 1, 1, 0), 460 | ] { 461 | let tile = tiles.get_tile(id(z, x, y)).await.unwrap().unwrap(); 462 | assert_eq!(tile, &contents[..]); 463 | } 464 | } 465 | 466 | #[tokio::test] 467 | #[cfg(feature = "iter-async")] 468 | async fn test_entries() { 469 | use futures_util::TryStreamExt as _; 470 | 471 | let backend = MmapBackend::try_from(VECTOR_FILE).await.unwrap(); 472 | let tiles = AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 473 | 474 | let entries = std::sync::Arc::new(tiles).entries(); 475 | 476 | let all_entries: Vec<_> = entries.try_collect().await.unwrap(); 477 | assert_eq!(all_entries.len(), 108); 478 | } 479 | } 480 | -------------------------------------------------------------------------------- /src/backend_aws_s3.rs: -------------------------------------------------------------------------------- 1 | use aws_sdk_s3::Client; 2 | use bytes::Bytes; 3 | 4 | use crate::{AsyncBackend, AsyncPmTilesReader, DirectoryCache, NoCache, PmtError, PmtResult}; 5 | 6 | impl AsyncPmTilesReader { 7 | /// Creates a new `PMTiles` reader from a client, bucket and key to the 8 | /// archive using the `aws-sdk-s3` backend. 9 | /// 10 | /// Fails if the `bucket` or `key` does not exist or is an invalid 11 | /// archive. Note that S3 requests are made to validate it. 12 | pub async fn new_with_client_bucket_and_path( 13 | client: Client, 14 | bucket: String, 15 | key: String, 16 | ) -> PmtResult { 17 | Self::new_with_cached_client_bucket_and_path(NoCache, client, bucket, key).await 18 | } 19 | } 20 | 21 | impl AsyncPmTilesReader { 22 | /// Creates a new `PMTiles` reader from a client, bucket and key to the 23 | /// archive using the `aws-sdk-s3` backend. Caches using the designated 24 | /// `cache`. 25 | /// 26 | /// Fails if the `bucket` or `key` does not exist or is an invalid 27 | /// archive. 28 | /// (Note: S3 requests are made to validate it.) 29 | pub async fn new_with_cached_client_bucket_and_path( 30 | cache: C, 31 | client: Client, 32 | bucket: String, 33 | key: String, 34 | ) -> PmtResult { 35 | let backend = AwsS3Backend::from(client, bucket, key); 36 | 37 | Self::try_from_cached_source(backend, cache).await 38 | } 39 | } 40 | 41 | pub struct AwsS3Backend { 42 | client: Client, 43 | bucket: String, 44 | key: String, 45 | } 46 | 47 | impl AwsS3Backend { 48 | #[must_use] 49 | pub fn from(client: Client, bucket: String, key: String) -> Self { 50 | Self { 51 | client, 52 | bucket, 53 | key, 54 | } 55 | } 56 | } 57 | 58 | impl AsyncBackend for AwsS3Backend { 59 | async fn read(&self, offset: usize, length: usize) -> PmtResult { 60 | let range_end = offset + length - 1; 61 | let range = format!("bytes={offset}-{range_end}"); 62 | 63 | let obj = self 64 | .client 65 | .get_object() 66 | .bucket(self.bucket.clone()) 67 | .key(self.key.clone()) 68 | .range(range) 69 | .send() 70 | .await 71 | .map_err(Box::new)?; 72 | 73 | let response_bytes = obj 74 | .body 75 | .collect() 76 | .await 77 | .map_err(|e| PmtError::Reading(e.into()))? 78 | .into_bytes(); 79 | 80 | if response_bytes.len() > length { 81 | Err(PmtError::ResponseBodyTooLong(response_bytes.len(), length)) 82 | } else { 83 | Ok(response_bytes) 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/backend_http.rs: -------------------------------------------------------------------------------- 1 | use bytes::Bytes; 2 | use reqwest::header::{HeaderValue, RANGE}; 3 | use reqwest::{Client, IntoUrl, Method, Request, StatusCode, Url}; 4 | 5 | use crate::{AsyncBackend, AsyncPmTilesReader, DirectoryCache, NoCache, PmtError, PmtResult}; 6 | 7 | impl AsyncPmTilesReader { 8 | /// Creates a new `PMTiles` reader from a URL using the Reqwest backend. 9 | /// 10 | /// Fails if `url` does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) 11 | pub async fn new_with_url(client: Client, url: U) -> PmtResult { 12 | Self::new_with_cached_url(NoCache, client, url).await 13 | } 14 | } 15 | 16 | impl AsyncPmTilesReader { 17 | /// Creates a new `PMTiles` reader with cache from a URL using the Reqwest backend. 18 | /// 19 | /// Fails if `url` does not exist or is an invalid archive. (Note: HTTP requests are made to validate it.) 20 | pub async fn new_with_cached_url( 21 | cache: C, 22 | client: Client, 23 | url: U, 24 | ) -> PmtResult { 25 | let backend = HttpBackend::try_from(client, url)?; 26 | 27 | Self::try_from_cached_source(backend, cache).await 28 | } 29 | } 30 | 31 | pub struct HttpBackend { 32 | client: Client, 33 | url: Url, 34 | } 35 | 36 | impl HttpBackend { 37 | pub fn try_from(client: Client, url: U) -> PmtResult { 38 | Ok(HttpBackend { 39 | client, 40 | url: url.into_url()?, 41 | }) 42 | } 43 | } 44 | 45 | impl AsyncBackend for HttpBackend { 46 | async fn read(&self, offset: usize, length: usize) -> PmtResult { 47 | let end = offset + length - 1; 48 | let range = format!("bytes={offset}-{end}"); 49 | let range = HeaderValue::try_from(range)?; 50 | 51 | let mut req = Request::new(Method::GET, self.url.clone()); 52 | req.headers_mut().insert(RANGE, range); 53 | 54 | let response = self.client.execute(req).await?.error_for_status()?; 55 | if response.status() != StatusCode::PARTIAL_CONTENT { 56 | return Err(PmtError::RangeRequestsUnsupported); 57 | } 58 | 59 | let response_bytes = response.bytes().await?; 60 | if response_bytes.len() > length { 61 | Err(PmtError::ResponseBodyTooLong(response_bytes.len(), length)) 62 | } else { 63 | Ok(response_bytes) 64 | } 65 | } 66 | } 67 | 68 | #[cfg(test)] 69 | mod tests { 70 | use super::*; 71 | 72 | static TEST_URL: &str = 73 | "https://protomaps.github.io/PMTiles/protomaps(vector)ODbL_firenze.pmtiles"; 74 | 75 | #[tokio::test] 76 | async fn basic_http_test() { 77 | let client = Client::builder().use_rustls_tls().build().unwrap(); 78 | let backend = HttpBackend::try_from(client, TEST_URL).unwrap(); 79 | 80 | AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/backend_mmap.rs: -------------------------------------------------------------------------------- 1 | use std::io; 2 | use std::path::Path; 3 | 4 | use bytes::{Buf, Bytes}; 5 | use fmmap::tokio::{AsyncMmapFile, AsyncMmapFileExt as _, AsyncOptions}; 6 | 7 | use crate::{AsyncBackend, AsyncPmTilesReader, DirectoryCache, NoCache, PmtError, PmtResult}; 8 | 9 | impl AsyncPmTilesReader { 10 | /// Creates a new `PMTiles` reader from a file path using the async mmap backend. 11 | /// 12 | /// Fails if `path` does not exist or is an invalid archive. 13 | pub async fn new_with_path>(path: P) -> PmtResult { 14 | Self::new_with_cached_path(NoCache, path).await 15 | } 16 | } 17 | 18 | impl AsyncPmTilesReader { 19 | /// Creates a new cached `PMTiles` reader from a file path using the async mmap backend. 20 | /// 21 | /// Fails if `path` does not exist or is an invalid archive. 22 | pub async fn new_with_cached_path>(cache: C, path: P) -> PmtResult { 23 | let backend = MmapBackend::try_from(path).await?; 24 | 25 | Self::try_from_cached_source(backend, cache).await 26 | } 27 | } 28 | 29 | pub struct MmapBackend { 30 | file: AsyncMmapFile, 31 | } 32 | 33 | impl MmapBackend { 34 | pub async fn try_from>(p: P) -> PmtResult { 35 | Ok(Self { 36 | file: AsyncMmapFile::open_with_options(p, AsyncOptions::new().read(true)) 37 | .await 38 | .map_err(|_| PmtError::UnableToOpenMmapFile)?, 39 | }) 40 | } 41 | } 42 | 43 | impl From for PmtError { 44 | fn from(_: fmmap::error::Error) -> Self { 45 | Self::Reading(io::Error::from(io::ErrorKind::UnexpectedEof)) 46 | } 47 | } 48 | 49 | impl AsyncBackend for MmapBackend { 50 | async fn read_exact(&self, offset: usize, length: usize) -> PmtResult { 51 | if self.file.len() >= offset + length { 52 | Ok(self.file.reader(offset)?.copy_to_bytes(length)) 53 | } else { 54 | Err(PmtError::Reading(io::Error::from( 55 | io::ErrorKind::UnexpectedEof, 56 | ))) 57 | } 58 | } 59 | 60 | async fn read(&self, offset: usize, length: usize) -> PmtResult { 61 | let reader = self.file.reader(offset)?; 62 | 63 | let read_length = length.min(reader.len()); 64 | 65 | Ok(self.file.reader(offset)?.copy_to_bytes(read_length)) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/backend_s3.rs: -------------------------------------------------------------------------------- 1 | use bytes::Bytes; 2 | use s3::Bucket; 3 | 4 | use crate::PmtError::ResponseBodyTooLong; 5 | use crate::{AsyncBackend, AsyncPmTilesReader, DirectoryCache, NoCache, PmtResult}; 6 | 7 | impl AsyncPmTilesReader { 8 | /// Creates a new `PMTiles` reader from a bucket and path to the 9 | /// archive using the `rust-s3` backend. 10 | /// 11 | /// Fails if `bucket` or `path` does not exist or is an invalid archive. (Note: S3 requests are made to validate it.) 12 | pub async fn new_with_bucket_path(bucket: Bucket, path: String) -> PmtResult { 13 | Self::new_with_cached_bucket_path(NoCache, bucket, path).await 14 | } 15 | } 16 | 17 | impl AsyncPmTilesReader { 18 | /// Creates a new `PMTiles` reader from a bucket and path to the 19 | /// archive using the `rust-s3` backend with a given `cache` backend. 20 | /// 21 | /// Fails if `bucket` or `path` does not exist or is an invalid archive. 22 | /// Note that S3 requests are made to validate it. 23 | pub async fn new_with_cached_bucket_path( 24 | cache: C, 25 | bucket: Bucket, 26 | path: String, 27 | ) -> PmtResult { 28 | let backend = S3Backend::from(bucket, path); 29 | 30 | Self::try_from_cached_source(backend, cache).await 31 | } 32 | } 33 | 34 | pub struct S3Backend { 35 | bucket: Bucket, 36 | path: String, 37 | } 38 | 39 | impl S3Backend { 40 | #[must_use] 41 | pub fn from(bucket: Bucket, path: String) -> S3Backend { 42 | Self { bucket, path } 43 | } 44 | } 45 | 46 | impl AsyncBackend for S3Backend { 47 | async fn read(&self, offset: usize, length: usize) -> PmtResult { 48 | let response = self 49 | .bucket 50 | .get_object_range( 51 | self.path.as_str(), 52 | offset as _, 53 | Some((offset + length - 1) as _), 54 | ) 55 | .await?; 56 | 57 | let response_bytes = response.bytes(); 58 | 59 | if response_bytes.len() > length { 60 | Err(ResponseBodyTooLong(response_bytes.len(), length)) 61 | } else { 62 | Ok(response_bytes.clone()) 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/cache.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::future::Future; 3 | use std::sync::{Arc, RwLock}; 4 | 5 | use crate::{DirEntry, Directory, TileId}; 6 | 7 | pub enum DirCacheResult { 8 | NotCached, 9 | NotFound, 10 | Found(DirEntry), 11 | } 12 | 13 | impl From> for DirCacheResult { 14 | fn from(entry: Option<&DirEntry>) -> Self { 15 | match entry { 16 | Some(entry) => DirCacheResult::Found(entry.clone()), 17 | None => DirCacheResult::NotFound, 18 | } 19 | } 20 | } 21 | 22 | /// A cache for `PMTiles` directories. 23 | pub trait DirectoryCache { 24 | /// Get a directory from the cache, using the offset as a key. 25 | fn get_dir_entry( 26 | &self, 27 | offset: usize, 28 | tile_id: TileId, 29 | ) -> impl Future + Send; 30 | 31 | /// Insert a directory into the cache, using the offset as a key. 32 | /// Note that the cache must be internally mutable. 33 | fn insert_dir(&self, offset: usize, directory: Directory) -> impl Future + Send; 34 | } 35 | 36 | pub struct NoCache; 37 | 38 | impl DirectoryCache for NoCache { 39 | #[inline] 40 | async fn get_dir_entry(&self, _offset: usize, _tile_id: TileId) -> DirCacheResult { 41 | DirCacheResult::NotCached 42 | } 43 | 44 | #[inline] 45 | async fn insert_dir(&self, _offset: usize, _directory: Directory) {} 46 | } 47 | 48 | /// A simple HashMap-based implementation of a `PMTiles` directory cache. 49 | #[derive(Default)] 50 | pub struct HashMapCache { 51 | pub cache: Arc>>, 52 | } 53 | 54 | impl DirectoryCache for HashMapCache { 55 | async fn get_dir_entry(&self, offset: usize, tile_id: TileId) -> DirCacheResult { 56 | // Panic if the lock is poisoned is not something the user can handle 57 | #[expect(clippy::unwrap_used)] 58 | if let Some(dir) = self.cache.read().unwrap().get(&offset) { 59 | return dir.find_tile_id(tile_id).into(); 60 | } 61 | DirCacheResult::NotCached 62 | } 63 | 64 | async fn insert_dir(&self, offset: usize, directory: Directory) { 65 | // Panic if the lock is poisoned is not something the user can handle 66 | #[expect(clippy::unwrap_used)] 67 | self.cache.write().unwrap().insert(offset, directory); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/directory.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{Debug, Formatter}; 2 | 3 | use bytes::{Buf, Bytes}; 4 | use varint_rs::VarintReader as _; 5 | #[cfg(feature = "write")] 6 | use varint_rs::VarintWriter as _; 7 | 8 | use crate::{PmtError, TileId}; 9 | 10 | #[derive(Default, Clone)] 11 | pub struct Directory { 12 | pub(crate) entries: Vec, 13 | } 14 | 15 | impl Debug for Directory { 16 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 17 | f.write_fmt(format_args!("Directory [entries: {}]", self.entries.len())) 18 | } 19 | } 20 | 21 | impl Directory { 22 | #[cfg(feature = "write")] 23 | pub(crate) fn with_capacity(capacity: usize) -> Self { 24 | Self { 25 | entries: Vec::with_capacity(capacity), 26 | } 27 | } 28 | 29 | #[cfg(feature = "write")] 30 | pub(crate) fn from_entries(entries: Vec) -> Self { 31 | Self { entries } 32 | } 33 | 34 | #[cfg(feature = "write")] 35 | pub(crate) fn entries(&self) -> &[DirEntry] { 36 | &self.entries 37 | } 38 | 39 | #[cfg(feature = "write")] 40 | pub(crate) fn push(&mut self, entry: DirEntry) { 41 | self.entries.push(entry); 42 | } 43 | 44 | /// Find the directory entry for a given tile ID. 45 | #[must_use] 46 | pub fn find_tile_id(&self, tile_id: TileId) -> Option<&DirEntry> { 47 | match self 48 | .entries 49 | .binary_search_by(|e| e.tile_id.cmp(&tile_id.value())) 50 | { 51 | Ok(idx) => self.entries.get(idx), 52 | Err(next_id) => { 53 | // Adapted from JavaScript code at 54 | // https://github.com/protomaps/PMTiles/blob/9c7f298fb42290354b8ed0a9b2f50e5c0d270c40/js/index.ts#L210 55 | if next_id > 0 { 56 | let previous_tile = self.entries.get(next_id - 1)?; 57 | if previous_tile.is_leaf() 58 | || (tile_id.value() - previous_tile.tile_id) 59 | < u64::from(previous_tile.run_length) 60 | { 61 | return Some(previous_tile); 62 | } 63 | } 64 | None 65 | } 66 | } 67 | } 68 | 69 | /// Get an estimated byte size of the directory object. Use this for cache eviction. 70 | #[must_use] 71 | pub fn get_approx_byte_size(&self) -> usize { 72 | self.entries.capacity() * size_of::() 73 | } 74 | } 75 | 76 | impl TryFrom for Directory { 77 | type Error = PmtError; 78 | 79 | fn try_from(buffer: Bytes) -> Result { 80 | let mut buffer = buffer.reader(); 81 | let n_entries = buffer.read_usize_varint()?; 82 | 83 | let mut entries = vec![DirEntry::default(); n_entries]; 84 | 85 | // Read tile IDs 86 | let mut next_tile_id = 0; 87 | for entry in &mut entries { 88 | next_tile_id += buffer.read_u64_varint()?; 89 | entry.tile_id = next_tile_id; 90 | } 91 | 92 | // Read Run Lengths 93 | for entry in &mut entries { 94 | entry.run_length = buffer.read_u32_varint()?; 95 | } 96 | 97 | // Read Lengths 98 | for entry in &mut entries { 99 | entry.length = buffer.read_u32_varint()?; 100 | } 101 | 102 | // Read Offsets 103 | let mut last_entry: Option<&DirEntry> = None; 104 | for entry in &mut entries { 105 | let offset = buffer.read_u64_varint()?; 106 | entry.offset = if offset == 0 { 107 | let e = last_entry.ok_or(PmtError::InvalidEntry)?; 108 | e.offset + u64::from(e.length) 109 | } else { 110 | offset - 1 111 | }; 112 | last_entry = Some(entry); 113 | } 114 | 115 | Ok(Directory { entries }) 116 | } 117 | } 118 | 119 | #[cfg(feature = "write")] 120 | impl crate::writer::WriteTo for Directory { 121 | fn write_to(&self, writer: &mut W) -> std::io::Result<()> { 122 | // Write number of entries 123 | writer.write_usize_varint(self.entries.len())?; 124 | 125 | // Write tile IDs 126 | let mut last_tile_id = 0; 127 | for entry in &self.entries { 128 | writer.write_u64_varint(entry.tile_id - last_tile_id)?; 129 | last_tile_id = entry.tile_id; 130 | } 131 | 132 | // Write Run Lengths 133 | for entry in &self.entries { 134 | writer.write_u32_varint(entry.run_length)?; 135 | } 136 | 137 | // Write Lengths 138 | for entry in &self.entries { 139 | writer.write_u32_varint(entry.length)?; 140 | } 141 | 142 | // Write Offsets 143 | let mut last_offset = 0; 144 | for entry in &self.entries { 145 | let offset_to_write = if entry.offset == last_offset + u64::from(entry.length) { 146 | 0 147 | } else { 148 | entry.offset + 1 149 | }; 150 | writer.write_u64_varint(offset_to_write)?; 151 | last_offset = entry.offset; 152 | } 153 | 154 | Ok(()) 155 | } 156 | } 157 | 158 | #[derive(Clone, Default, Debug)] 159 | pub struct DirEntry { 160 | pub(crate) tile_id: u64, 161 | pub(crate) offset: u64, 162 | pub(crate) length: u32, 163 | pub(crate) run_length: u32, 164 | } 165 | 166 | impl DirEntry { 167 | pub(crate) fn is_leaf(&self) -> bool { 168 | self.run_length == 0 169 | } 170 | 171 | #[cfg(feature = "iter-async")] 172 | #[must_use] 173 | pub fn iter_coords(&self) -> DirEntryCoordsIter<'_> { 174 | DirEntryCoordsIter { 175 | entry: self, 176 | current: 0, 177 | } 178 | } 179 | } 180 | 181 | #[cfg(feature = "iter-async")] 182 | pub struct DirEntryCoordsIter<'a> { 183 | entry: &'a DirEntry, 184 | current: u32, 185 | } 186 | 187 | #[cfg(feature = "iter-async")] 188 | impl Iterator for DirEntryCoordsIter<'_> { 189 | type Item = TileId; 190 | 191 | fn next(&mut self) -> Option { 192 | if self.current < self.entry.run_length { 193 | let current = u64::from(self.current); 194 | self.current += 1; 195 | Some(TileId::new(self.entry.tile_id + current).expect("invalid entry data")) 196 | } else { 197 | None 198 | } 199 | } 200 | } 201 | 202 | #[cfg(test)] 203 | mod tests { 204 | use std::io::{BufReader, Read, Write}; 205 | 206 | use bytes::BytesMut; 207 | 208 | use crate::header::HEADER_SIZE; 209 | use crate::tests::RASTER_FILE; 210 | #[cfg(feature = "iter-async")] 211 | use crate::tile::test::coord; 212 | use crate::{Directory, Header}; 213 | 214 | fn read_root_directory(file: &str) -> Directory { 215 | let test_file = std::fs::File::open(file).unwrap(); 216 | let mut reader = BufReader::new(test_file); 217 | 218 | let mut header_bytes = BytesMut::zeroed(HEADER_SIZE); 219 | reader.read_exact(header_bytes.as_mut()).unwrap(); 220 | 221 | let header = Header::try_from_bytes(header_bytes.freeze()).unwrap(); 222 | let mut directory_bytes = BytesMut::zeroed(usize::try_from(header.root_length).unwrap()); 223 | reader.read_exact(directory_bytes.as_mut()).unwrap(); 224 | 225 | let mut decompressed = BytesMut::zeroed(directory_bytes.len() * 2); 226 | { 227 | let mut gunzip = flate2::write::GzDecoder::new(decompressed.as_mut()); 228 | gunzip.write_all(&directory_bytes).unwrap(); 229 | } 230 | 231 | Directory::try_from(decompressed.freeze()).unwrap() 232 | } 233 | 234 | #[test] 235 | fn root_directory() { 236 | let directory = read_root_directory(RASTER_FILE); 237 | assert_eq!(directory.entries.len(), 84); 238 | // Note: this is not true for all tiles, just the first few... 239 | for nth in 0..10 { 240 | assert_eq!(directory.entries[nth].tile_id, nth as u64); 241 | } 242 | 243 | #[cfg(feature = "iter-async")] 244 | assert_eq!( 245 | directory.entries[57].iter_coords().collect::>(), 246 | vec![coord(3, 4, 6).into()] 247 | ); 248 | 249 | // ...it breaks the pattern on the 59th tile, because it has a run length of 2 250 | assert_eq!(directory.entries[58].tile_id, 58); 251 | assert_eq!(directory.entries[58].run_length, 2); 252 | assert_eq!(directory.entries[58].offset, 422_070); 253 | assert_eq!(directory.entries[58].length, 850); 254 | 255 | // that also means that it has two entries in xyz 256 | #[cfg(feature = "iter-async")] 257 | assert_eq!( 258 | directory.entries[58].iter_coords().collect::>(), 259 | vec![coord(3, 4, 7).into(), coord(3, 5, 7).into()] 260 | ); 261 | } 262 | 263 | #[test] 264 | #[cfg(feature = "write")] 265 | fn write_directory() { 266 | use crate::writer::WriteTo as _; 267 | 268 | let root_dir = read_root_directory(RASTER_FILE); 269 | let mut buf = vec![]; 270 | root_dir.write_to(&mut buf).unwrap(); 271 | let dir = Directory::try_from(bytes::Bytes::from(buf)).unwrap(); 272 | assert!( 273 | root_dir 274 | .entries 275 | .iter() 276 | .enumerate() 277 | .all(|(idx, entry)| dir.entries[idx].tile_id == entry.tile_id 278 | && dir.entries[idx].run_length == entry.run_length 279 | && dir.entries[idx].offset == entry.offset 280 | && dir.entries[idx].length == entry.length) 281 | ); 282 | } 283 | } 284 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | use std::string::FromUtf8Error; 2 | 3 | use thiserror::Error; 4 | 5 | use crate::Compression; 6 | 7 | /// A specialized [`Result`] type for `PMTiles` operations. 8 | pub type PmtResult = Result; 9 | 10 | /// Errors that can occur while reading `PMTiles` files. 11 | #[derive(Debug, Error)] 12 | pub enum PmtError { 13 | #[error("Invalid magic number")] 14 | InvalidMagicNumber, 15 | #[error("Invalid PMTiles version")] 16 | UnsupportedPmTilesVersion, 17 | #[error("Invalid compression")] 18 | InvalidCompression, 19 | #[error("Unsupported compression {0:?}")] 20 | UnsupportedCompression(Compression), 21 | #[error("Invalid PMTiles entry")] 22 | InvalidEntry, 23 | #[error("Invalid header")] 24 | InvalidHeader, 25 | #[error("Invalid metadata")] 26 | InvalidMetadata, 27 | #[cfg(feature = "write")] 28 | #[error("Directory index element overflow")] 29 | IndexEntryOverflow, 30 | #[error("Invalid metadata UTF-8 encoding: {0}")] 31 | InvalidMetadataUtf8Encoding(#[from] FromUtf8Error), 32 | #[error("Invalid tile type")] 33 | InvalidTileType, 34 | #[error("IO Error {0}")] 35 | Reading(#[from] std::io::Error), 36 | #[cfg(feature = "mmap-async-tokio")] 37 | #[error("Unable to open mmap file")] 38 | UnableToOpenMmapFile, 39 | #[error("Unexpected number of bytes returned [expected: {0}, received: {1}].")] 40 | UnexpectedNumberOfBytesReturned(usize, usize), 41 | #[cfg(feature = "http-async")] 42 | #[error("Range requests unsupported")] 43 | RangeRequestsUnsupported, 44 | #[cfg(any( 45 | feature = "http-async", 46 | feature = "__async-s3", 47 | feature = "__async-aws-s3" 48 | ))] 49 | #[error("HTTP response body is too long, Response {0}B > requested {1}B")] 50 | ResponseBodyTooLong(usize, usize), 51 | #[cfg(feature = "http-async")] 52 | #[error(transparent)] 53 | Http(#[from] reqwest::Error), 54 | #[cfg(feature = "http-async")] 55 | #[error(transparent)] 56 | InvalidHeaderValue(#[from] reqwest::header::InvalidHeaderValue), 57 | #[cfg(feature = "__async-s3")] 58 | #[error(transparent)] 59 | S3(#[from] s3::error::S3Error), 60 | #[cfg(feature = "__async-aws-s3")] 61 | #[error(transparent)] 62 | AwsS3Request( 63 | #[from] Box>, 64 | ), 65 | } 66 | -------------------------------------------------------------------------------- /src/header.rs: -------------------------------------------------------------------------------- 1 | use std::num::NonZeroU64; 2 | use std::panic::catch_unwind; 3 | 4 | use bytes::{Buf, Bytes}; 5 | 6 | use crate::{PmtError, PmtResult}; 7 | 8 | #[cfg(any(feature = "__async", feature = "write"))] 9 | pub(crate) const MAX_INITIAL_BYTES: usize = 16_384; 10 | #[cfg(any(test, feature = "__async", feature = "write"))] 11 | pub(crate) const HEADER_SIZE: usize = 127; 12 | 13 | #[derive(Debug)] 14 | #[allow(dead_code)] 15 | pub struct Header { 16 | pub(crate) version: u8, 17 | pub(crate) root_offset: u64, 18 | pub(crate) root_length: u64, 19 | pub(crate) metadata_offset: u64, 20 | pub(crate) metadata_length: u64, 21 | pub(crate) leaf_offset: u64, 22 | pub(crate) leaf_length: u64, 23 | pub(crate) data_offset: u64, 24 | pub(crate) data_length: u64, 25 | pub(crate) n_addressed_tiles: Option, 26 | pub(crate) n_tile_entries: Option, 27 | pub(crate) n_tile_contents: Option, 28 | pub(crate) clustered: bool, 29 | pub(crate) internal_compression: Compression, 30 | pub tile_compression: Compression, 31 | pub tile_type: TileType, 32 | pub min_zoom: u8, 33 | pub max_zoom: u8, 34 | pub min_longitude: f32, 35 | pub min_latitude: f32, 36 | pub max_longitude: f32, 37 | pub max_latitude: f32, 38 | pub center_zoom: u8, 39 | pub center_longitude: f32, 40 | pub center_latitude: f32, 41 | } 42 | 43 | impl Header { 44 | #[cfg(feature = "write")] 45 | pub(crate) fn new(tile_compression: Compression, tile_type: TileType) -> Self { 46 | #[expect(clippy::excessive_precision)] 47 | Self { 48 | version: 3, 49 | root_offset: HEADER_SIZE as u64, 50 | root_length: 0, 51 | metadata_offset: MAX_INITIAL_BYTES as u64, 52 | metadata_length: 0, 53 | leaf_offset: 0, 54 | leaf_length: 0, 55 | data_offset: 0, 56 | data_length: 0, 57 | n_addressed_tiles: None, 58 | n_tile_entries: None, 59 | n_tile_contents: None, 60 | clustered: true, 61 | internal_compression: Compression::Gzip, 62 | tile_compression, 63 | tile_type, 64 | min_zoom: 0, 65 | max_zoom: 22, 66 | min_longitude: -180.0, 67 | min_latitude: -85.051_129, 68 | max_longitude: 180.0, 69 | max_latitude: 85.051_129, 70 | center_zoom: 0, 71 | center_longitude: 0.0, 72 | center_latitude: 0.0, 73 | } 74 | } 75 | } 76 | 77 | #[derive(Debug, Eq, PartialEq, Copy, Clone)] 78 | pub enum Compression { 79 | Unknown, 80 | None, 81 | Gzip, 82 | Brotli, 83 | Zstd, 84 | } 85 | 86 | impl Compression { 87 | #[must_use] 88 | pub fn content_encoding(self) -> Option<&'static str> { 89 | Some(match self { 90 | Compression::Gzip => "gzip", 91 | Compression::Brotli => "br", 92 | _ => None?, 93 | }) 94 | } 95 | } 96 | 97 | impl TryInto for u8 { 98 | type Error = PmtError; 99 | 100 | fn try_into(self) -> Result { 101 | match self { 102 | 0 => Ok(Compression::Unknown), 103 | 1 => Ok(Compression::None), 104 | 2 => Ok(Compression::Gzip), 105 | 3 => Ok(Compression::Brotli), 106 | 4 => Ok(Compression::Zstd), 107 | _ => Err(PmtError::InvalidCompression), 108 | } 109 | } 110 | } 111 | 112 | #[cfg(feature = "tilejson")] 113 | impl Header { 114 | #[must_use] 115 | pub fn get_tilejson(&self, sources: Vec) -> tilejson::TileJSON { 116 | tilejson::tilejson! { 117 | tiles: sources, 118 | minzoom: self.min_zoom, 119 | maxzoom: self.max_zoom, 120 | bounds: self.get_bounds(), 121 | center: self.get_center(), 122 | } 123 | } 124 | 125 | #[must_use] 126 | pub fn get_bounds(&self) -> tilejson::Bounds { 127 | tilejson::Bounds::new( 128 | f64::from(self.min_longitude), 129 | f64::from(self.min_latitude), 130 | f64::from(self.max_longitude), 131 | f64::from(self.max_latitude), 132 | ) 133 | } 134 | 135 | #[must_use] 136 | pub fn get_center(&self) -> tilejson::Center { 137 | tilejson::Center::new( 138 | f64::from(self.center_longitude), 139 | f64::from(self.center_latitude), 140 | self.center_zoom, 141 | ) 142 | } 143 | } 144 | 145 | #[derive(Debug, Eq, PartialEq, Copy, Clone)] 146 | pub enum TileType { 147 | Unknown, 148 | Mvt, 149 | Png, 150 | Jpeg, 151 | Webp, 152 | } 153 | 154 | impl TileType { 155 | #[must_use] 156 | pub fn content_type(self) -> &'static str { 157 | match self { 158 | TileType::Mvt => "application/vnd.mapbox-vector-tile", 159 | TileType::Png => "image/png", 160 | TileType::Webp => "image/webp", 161 | TileType::Jpeg => "image/jpeg", 162 | TileType::Unknown => "application/octet-stream", 163 | } 164 | } 165 | } 166 | 167 | impl TryInto for u8 { 168 | type Error = PmtError; 169 | 170 | fn try_into(self) -> Result { 171 | match self { 172 | 0 => Ok(TileType::Unknown), 173 | 1 => Ok(TileType::Mvt), 174 | 2 => Ok(TileType::Png), 175 | 3 => Ok(TileType::Jpeg), 176 | 4 => Ok(TileType::Webp), 177 | _ => Err(PmtError::InvalidTileType), 178 | } 179 | } 180 | } 181 | 182 | static V3_MAGIC: &str = "PMTiles"; 183 | static V2_MAGIC: &str = "PM"; 184 | 185 | impl Header { 186 | #[expect(clippy::cast_precision_loss)] 187 | fn read_coordinate_part(mut buf: B) -> f32 { 188 | // TODO: would it be more precise to do `((value as f64) / 10_000_000.) as f32` ? 189 | buf.get_i32_le() as f32 / 10_000_000. 190 | } 191 | 192 | pub fn try_from_bytes(mut bytes: Bytes) -> PmtResult { 193 | let magic_bytes = bytes.split_to(V3_MAGIC.len()); 194 | 195 | // Assert magic 196 | if magic_bytes != V3_MAGIC { 197 | return Err(if magic_bytes.starts_with(V2_MAGIC.as_bytes()) { 198 | PmtError::UnsupportedPmTilesVersion 199 | } else { 200 | PmtError::InvalidMagicNumber 201 | }); 202 | } 203 | 204 | // Wrap the panics that are possible in `get_u*_le` calls. (Panic occurs if the buffer is exhausted.) 205 | catch_unwind(move || { 206 | Ok(Self { 207 | version: bytes.get_u8(), 208 | root_offset: bytes.get_u64_le(), 209 | root_length: bytes.get_u64_le(), 210 | metadata_offset: bytes.get_u64_le(), 211 | metadata_length: bytes.get_u64_le(), 212 | leaf_offset: bytes.get_u64_le(), 213 | leaf_length: bytes.get_u64_le(), 214 | data_offset: bytes.get_u64_le(), 215 | data_length: bytes.get_u64_le(), 216 | n_addressed_tiles: NonZeroU64::new(bytes.get_u64_le()), 217 | n_tile_entries: NonZeroU64::new(bytes.get_u64_le()), 218 | n_tile_contents: NonZeroU64::new(bytes.get_u64_le()), 219 | clustered: bytes.get_u8() == 1, 220 | internal_compression: bytes.get_u8().try_into()?, 221 | tile_compression: bytes.get_u8().try_into()?, 222 | tile_type: bytes.get_u8().try_into()?, 223 | min_zoom: bytes.get_u8(), 224 | max_zoom: bytes.get_u8(), 225 | min_longitude: Self::read_coordinate_part(&mut bytes), 226 | min_latitude: Self::read_coordinate_part(&mut bytes), 227 | max_longitude: Self::read_coordinate_part(&mut bytes), 228 | max_latitude: Self::read_coordinate_part(&mut bytes), 229 | center_zoom: bytes.get_u8(), 230 | center_longitude: Self::read_coordinate_part(&mut bytes), 231 | center_latitude: Self::read_coordinate_part(&mut bytes), 232 | }) 233 | }) 234 | .map_err(|_| PmtError::InvalidHeader)? 235 | } 236 | } 237 | 238 | #[cfg(feature = "write")] 239 | impl crate::writer::WriteTo for Header { 240 | fn write_to(&self, writer: &mut W) -> std::io::Result<()> { 241 | use std::num::NonZero; 242 | 243 | // Write a magic number 244 | writer.write_all(V3_MAGIC.as_bytes())?; 245 | 246 | // Write header fields 247 | writer.write_all(&[self.version])?; 248 | writer.write_all(&self.root_offset.to_le_bytes())?; 249 | writer.write_all(&self.root_length.to_le_bytes())?; 250 | writer.write_all(&self.metadata_offset.to_le_bytes())?; 251 | writer.write_all(&self.metadata_length.to_le_bytes())?; 252 | writer.write_all(&self.leaf_offset.to_le_bytes())?; 253 | writer.write_all(&self.leaf_length.to_le_bytes())?; 254 | writer.write_all(&self.data_offset.to_le_bytes())?; 255 | writer.write_all(&self.data_length.to_le_bytes())?; 256 | writer.write_all(&self.n_addressed_tiles.map_or(0, NonZero::get).to_le_bytes())?; 257 | writer.write_all(&self.n_tile_entries.map_or(0, NonZero::get).to_le_bytes())?; 258 | writer.write_all(&self.n_tile_contents.map_or(0, NonZero::get).to_le_bytes())?; 259 | writer.write_all(&[u8::from(self.clustered)])?; 260 | writer.write_all(&[self.internal_compression as u8])?; 261 | writer.write_all(&[self.tile_compression as u8])?; 262 | writer.write_all(&[self.tile_type as u8])?; 263 | writer.write_all(&[self.min_zoom])?; 264 | writer.write_all(&[self.max_zoom])?; 265 | Self::write_coordinate_part(writer, self.min_longitude)?; 266 | Self::write_coordinate_part(writer, self.min_latitude)?; 267 | Self::write_coordinate_part(writer, self.max_longitude)?; 268 | Self::write_coordinate_part(writer, self.max_latitude)?; 269 | writer.write_all(&[self.center_zoom])?; 270 | Self::write_coordinate_part(writer, self.center_longitude)?; 271 | Self::write_coordinate_part(writer, self.center_latitude)?; 272 | 273 | Ok(()) 274 | } 275 | } 276 | 277 | impl Header { 278 | #[cfg(feature = "write")] 279 | #[expect(clippy::cast_possible_truncation)] 280 | fn write_coordinate_part(writer: &mut W, value: f32) -> std::io::Result<()> { 281 | writer.write_all(&((value * 10_000_000.0) as i32).to_le_bytes()) 282 | } 283 | } 284 | 285 | #[cfg(test)] 286 | mod tests { 287 | #![expect(clippy::unreadable_literal, clippy::float_cmp)] 288 | 289 | use std::fs::File; 290 | use std::io::Read; 291 | use std::num::NonZeroU64; 292 | 293 | use bytes::{Bytes, BytesMut}; 294 | 295 | use crate::header::HEADER_SIZE; 296 | use crate::tests::{RASTER_FILE, VECTOR_FILE}; 297 | use crate::{Header, TileType}; 298 | 299 | #[test] 300 | fn read_header() { 301 | let mut test = File::open(RASTER_FILE).unwrap(); 302 | let mut header_bytes = [0; HEADER_SIZE]; 303 | test.read_exact(header_bytes.as_mut_slice()).unwrap(); 304 | 305 | let header = Header::try_from_bytes(Bytes::copy_from_slice(&header_bytes)).unwrap(); 306 | 307 | // TODO: should be 3, but currently the ascii char 3, assert_eq!(header.version, 3); 308 | assert_eq!(header.tile_type, TileType::Png); 309 | assert_eq!(header.n_addressed_tiles, NonZeroU64::new(85)); 310 | assert_eq!(header.n_tile_entries, NonZeroU64::new(84)); 311 | assert_eq!(header.n_tile_contents, NonZeroU64::new(80)); 312 | assert_eq!(header.min_zoom, 0); 313 | assert_eq!(header.max_zoom, 3); 314 | assert_eq!(header.center_zoom, 0); 315 | assert_eq!(header.center_latitude, 0.0); 316 | assert_eq!(header.center_longitude, 0.0); 317 | assert_eq!(header.min_latitude, -85.0); 318 | assert_eq!(header.max_latitude, 85.0); 319 | assert_eq!(header.min_longitude, -180.0); 320 | assert_eq!(header.max_longitude, 180.0); 321 | assert!(header.clustered); 322 | } 323 | 324 | #[test] 325 | fn read_valid_mvt_header() { 326 | let mut test = File::open(VECTOR_FILE).unwrap(); 327 | let mut header_bytes = BytesMut::zeroed(HEADER_SIZE); 328 | test.read_exact(header_bytes.as_mut()).unwrap(); 329 | 330 | let header = Header::try_from_bytes(header_bytes.freeze()).unwrap(); 331 | 332 | assert_eq!(header.version, 3); 333 | assert_eq!(header.tile_type, TileType::Mvt); 334 | assert_eq!(header.n_addressed_tiles, NonZeroU64::new(108)); 335 | assert_eq!(header.n_tile_entries, NonZeroU64::new(108)); 336 | assert_eq!(header.n_tile_contents, NonZeroU64::new(106)); 337 | assert_eq!(header.min_zoom, 0); 338 | assert_eq!(header.max_zoom, 14); 339 | assert_eq!(header.center_zoom, 0); 340 | assert_eq!(header.center_latitude, 43.779778); 341 | assert_eq!(header.center_longitude, 11.241483); 342 | assert_eq!(header.min_latitude, 43.727013); 343 | assert_eq!(header.max_latitude, 43.832542); 344 | assert_eq!(header.min_longitude, 11.154026); 345 | assert_eq!(header.max_longitude, 11.328939); 346 | assert!(header.clustered); 347 | } 348 | 349 | #[test] 350 | #[cfg(feature = "tilejson")] 351 | fn get_tilejson_raster() { 352 | use tilejson::{Bounds, Center}; 353 | 354 | let mut test = File::open(RASTER_FILE).unwrap(); 355 | let mut header_bytes = BytesMut::zeroed(HEADER_SIZE); 356 | test.read_exact(header_bytes.as_mut()).unwrap(); 357 | let header = Header::try_from_bytes(header_bytes.freeze()).unwrap(); 358 | let tj = header.get_tilejson(Vec::new()); 359 | 360 | assert_eq!(tj.center, Some(Center::default())); 361 | assert_eq!(tj.bounds, Some(Bounds::new(-180.0, -85.0, 180.0, 85.0))); 362 | } 363 | 364 | #[test] 365 | #[cfg(feature = "tilejson")] 366 | fn get_tilejson_vector() { 367 | use tilejson::{Bounds, Center}; 368 | 369 | let mut test = File::open(VECTOR_FILE).unwrap(); 370 | let mut header_bytes = BytesMut::zeroed(HEADER_SIZE); 371 | test.read_exact(header_bytes.as_mut()).unwrap(); 372 | let header = Header::try_from_bytes(header_bytes.freeze()).unwrap(); 373 | let tj = header.get_tilejson(Vec::new()); 374 | 375 | assert_eq!( 376 | tj.center, 377 | Some(Center::new(11.241482734680176, 43.77977752685547, 0)) 378 | ); 379 | 380 | assert_eq!( 381 | tj.bounds, 382 | Some(Bounds::new( 383 | 11.15402603149414, 384 | 43.727012634277344, 385 | 11.328939437866211, 386 | 43.832542419433594 387 | )) 388 | ); 389 | } 390 | 391 | #[test] 392 | #[cfg(feature = "write")] 393 | fn write_header() { 394 | use crate::writer::WriteTo as _; 395 | 396 | let mut test = File::open(RASTER_FILE).unwrap(); 397 | let mut header_bytes = [0; HEADER_SIZE]; 398 | test.read_exact(header_bytes.as_mut_slice()).unwrap(); 399 | let header = Header::try_from_bytes(Bytes::copy_from_slice(&header_bytes)).unwrap(); 400 | 401 | let mut buf = vec![]; 402 | header.write_to(&mut buf).unwrap(); 403 | let out = Header::try_from_bytes(Bytes::from(buf)).unwrap(); 404 | assert_eq!(header.version, out.version); 405 | assert_eq!(header.tile_type, out.tile_type); 406 | assert_eq!(header.n_addressed_tiles, out.n_addressed_tiles); 407 | assert_eq!(header.n_tile_entries, out.n_tile_entries); 408 | assert_eq!(header.n_tile_contents, out.n_tile_contents); 409 | assert_eq!(header.min_zoom, out.min_zoom); 410 | assert_eq!(header.max_zoom, out.max_zoom); 411 | assert_eq!(header.center_zoom, out.center_zoom); 412 | assert_eq!(header.center_latitude, out.center_latitude); 413 | assert_eq!(header.center_longitude, out.center_longitude); 414 | assert_eq!(header.min_latitude, out.min_latitude); 415 | assert_eq!(header.max_latitude, out.max_latitude); 416 | assert_eq!(header.min_longitude, out.min_longitude); 417 | assert_eq!(header.max_longitude, out.max_longitude); 418 | assert_eq!(header.clustered, out.clustered); 419 | } 420 | } 421 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(all(feature = "__all_non_conflicting"), doc = include_str!("../README.md"))] 2 | 3 | #[cfg(feature = "__async")] 4 | mod async_reader; 5 | #[cfg(feature = "__async")] 6 | pub use async_reader::{AsyncBackend, AsyncPmTilesReader}; 7 | 8 | #[cfg(feature = "__async-aws-s3")] 9 | mod backend_aws_s3; 10 | #[cfg(feature = "http-async")] 11 | mod backend_http; 12 | #[cfg(feature = "mmap-async-tokio")] 13 | mod backend_mmap; 14 | #[cfg(feature = "__async-s3")] 15 | mod backend_s3; 16 | 17 | #[cfg(feature = "__async")] 18 | mod cache; 19 | #[cfg(feature = "__async")] 20 | pub use cache::{DirCacheResult, DirectoryCache, HashMapCache, NoCache}; 21 | 22 | mod directory; 23 | mod error; 24 | mod header; 25 | mod tile; 26 | #[cfg(feature = "write")] 27 | mod writer; 28 | 29 | /// Re-export of crate exposed in our API to simplify dependency management 30 | #[cfg(feature = "__async-aws-s3")] 31 | pub use aws_sdk_s3; 32 | #[cfg(feature = "aws-s3-async")] 33 | pub use backend_aws_s3::AwsS3Backend; 34 | #[cfg(feature = "http-async")] 35 | pub use backend_http::HttpBackend; 36 | #[cfg(feature = "mmap-async-tokio")] 37 | pub use backend_mmap::MmapBackend; 38 | #[cfg(feature = "__async-s3")] 39 | pub use backend_s3::S3Backend; 40 | #[cfg(feature = "iter-async")] 41 | pub use directory::DirEntryCoordsIter; 42 | pub use directory::{DirEntry, Directory}; 43 | pub use error::{PmtError, PmtResult}; 44 | pub use header::{Compression, Header, TileType}; 45 | /// Re-export of crate exposed in our API to simplify dependency management 46 | #[cfg(feature = "http-async")] 47 | pub use reqwest; 48 | /// Re-export of crate exposed in our API to simplify dependency management 49 | #[cfg(feature = "__async-s3")] 50 | pub use s3; 51 | pub use tile::{MAX_TILE_ID, MAX_ZOOM, PYRAMID_SIZE_BY_ZOOM, TileCoord, TileId}; 52 | /// Re-export of crate exposed in our API to simplify dependency management 53 | #[cfg(feature = "tilejson")] 54 | pub use tilejson; 55 | #[cfg(feature = "write")] 56 | pub use writer::{PmTilesStreamWriter, PmTilesWriter}; 57 | 58 | #[cfg(test)] 59 | mod tests { 60 | pub const RASTER_FILE: &str = "fixtures/stamen_toner(raster)CC-BY+ODbL_z3.pmtiles"; 61 | pub const VECTOR_FILE: &str = "fixtures/protomaps(vector)ODbL_firenze.pmtiles"; 62 | } 63 | -------------------------------------------------------------------------------- /src/tile.rs: -------------------------------------------------------------------------------- 1 | use fast_hilbert::{h2xy, xy2h}; 2 | 3 | /// The pre-computed sizes of the tile pyramid for each zoom level. 4 | /// The size at zoom level `z` (array index) is equal to the number of tiles before that zoom level. 5 | /// 6 | /// ``` 7 | /// # use pmtiles::PYRAMID_SIZE_BY_ZOOM; 8 | /// let mut size_at_level = 0_u64; 9 | /// for z in 0..PYRAMID_SIZE_BY_ZOOM.len() { 10 | /// assert_eq!(PYRAMID_SIZE_BY_ZOOM[z], size_at_level, "Invalid value at zoom {z}"); 11 | /// // add number of tiles at this zoom level 12 | /// size_at_level += 4_u64.pow(z as u32); 13 | /// } 14 | /// ``` 15 | #[expect(clippy::unreadable_literal)] 16 | pub const PYRAMID_SIZE_BY_ZOOM: [u64; 32] = [ 17 | /* 0 */ 0, 18 | /* 1 */ 1, 19 | /* 2 */ 5, 20 | /* 3 */ 21, 21 | /* 4 */ 85, 22 | /* 5 */ 341, 23 | /* 6 */ 1365, 24 | /* 7 */ 5461, 25 | /* 8 */ 21845, 26 | /* 9 */ 87381, 27 | /* 10 */ 349525, 28 | /* 11 */ 1398101, 29 | /* 12 */ 5592405, 30 | /* 13 */ 22369621, 31 | /* 14 */ 89478485, 32 | /* 15 */ 357913941, 33 | /* 16 */ 1431655765, 34 | /* 17 */ 5726623061, 35 | /* 18 */ 22906492245, 36 | /* 19 */ 91625968981, 37 | /* 20 */ 366503875925, 38 | /* 21 */ 1466015503701, 39 | /* 22 */ 5864062014805, 40 | /* 23 */ 23456248059221, 41 | /* 24 */ 93824992236885, 42 | /* 25 */ 375299968947541, 43 | /* 26 */ 1501199875790165, 44 | /* 27 */ 6004799503160661, 45 | /* 28 */ 24019198012642645, 46 | /* 29 */ 96076792050570581, 47 | /* 30 */ 384307168202282325, 48 | /* 31 */ 1537228672809129301, 49 | ]; 50 | 51 | /// Maximum valid Tile Zoom level in the `PMTiles` format. 52 | /// 53 | /// ``` 54 | /// # use pmtiles::MAX_ZOOM; 55 | /// assert_eq!(MAX_ZOOM, 31); 56 | /// ``` 57 | #[expect(clippy::cast_possible_truncation)] 58 | pub const MAX_ZOOM: u8 = PYRAMID_SIZE_BY_ZOOM.len() as u8 - 1; 59 | 60 | /// Maximum valid Tile ID in the `PMTiles` format. 61 | /// 62 | /// ``` 63 | /// # use pmtiles::MAX_TILE_ID; 64 | /// assert_eq!(MAX_TILE_ID, 6148914691236517204); 65 | /// ``` 66 | pub const MAX_TILE_ID: u64 = 67 | PYRAMID_SIZE_BY_ZOOM[PYRAMID_SIZE_BY_ZOOM.len() - 1] + 4_u64.pow(MAX_ZOOM as u32) - 1; 68 | 69 | /// Represents a tile coordinate in the `PMTiles` format. 70 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] 71 | pub struct TileCoord { 72 | z: u8, 73 | x: u32, 74 | y: u32, 75 | } 76 | 77 | impl TileCoord { 78 | /// Create a new coordinate with the given zoom level and tile coordinates, or return `None` if the values are invalid. 79 | /// 80 | /// ``` 81 | /// # use pmtiles::TileCoord; 82 | /// let coord = TileCoord::new(18, 235085, 122323).unwrap(); 83 | /// assert_eq!(coord.z(), 18); 84 | /// assert_eq!(coord.x(), 235085); 85 | /// assert_eq!(coord.y(), 122323); 86 | /// assert!(TileCoord::new(32, 1, 3).is_none()); // Invalid zoom level 87 | /// assert!(TileCoord::new(2, 4, 0).is_none()); // Invalid x coordinate 88 | /// assert!(TileCoord::new(2, 0, 4).is_none()); // Invalid y coordinate 89 | /// ``` 90 | #[must_use] 91 | pub fn new(z: u8, x: u32, y: u32) -> Option { 92 | if z > MAX_ZOOM || x >= (1 << z) || y >= (1 << z) { 93 | return None; 94 | } 95 | Some(Self { z, x, y }) 96 | } 97 | 98 | /// Get the zoom level of this coordinate. 99 | #[must_use] 100 | pub fn z(&self) -> u8 { 101 | self.z 102 | } 103 | 104 | /// Get the x coordinate of this tile. 105 | #[must_use] 106 | pub fn x(&self) -> u32 { 107 | self.x 108 | } 109 | 110 | /// Get the y coordinate of this tile. 111 | #[must_use] 112 | pub fn y(&self) -> u32 { 113 | self.y 114 | } 115 | } 116 | 117 | /// Represents a unique identifier for a tile in the `PMTiles` format. 118 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] 119 | pub struct TileId(u64); 120 | 121 | impl TileId { 122 | /// Create a new `TileId` from the u64 value, or return `None` if the value is invalid. 123 | /// 124 | /// ``` 125 | /// # use pmtiles::TileId; 126 | /// assert_eq!(TileId::new(0).unwrap().value(), 0); 127 | /// assert!(TileId::new(6148914691236517204).is_some()); 128 | /// assert!(TileId::new(6148914691236517205).is_none()); 129 | /// ``` 130 | #[must_use] 131 | pub fn new(id: u64) -> Option { 132 | if id <= MAX_TILE_ID { 133 | Some(Self(id)) 134 | } else { 135 | None 136 | } 137 | } 138 | 139 | /// Get the underlying u64 value of this `TileId`. 140 | #[must_use] 141 | pub fn value(self) -> u64 { 142 | self.0 143 | } 144 | } 145 | 146 | impl From for u64 { 147 | fn from(tile_id: TileId) -> Self { 148 | tile_id.0 149 | } 150 | } 151 | 152 | impl From for TileCoord { 153 | #[expect(clippy::cast_possible_truncation)] 154 | fn from(id: TileId) -> Self { 155 | let id = id.value(); 156 | let mut z = 0; 157 | let mut size = 0; 158 | for (idx, &val) in PYRAMID_SIZE_BY_ZOOM.iter().enumerate() { 159 | if id < val { 160 | // If we never break, it means the id is for the last zoom level. 161 | // The ID has been verified to be <= MAX_TILE_ID, so this is safe. 162 | break; 163 | } 164 | z = idx as u8; 165 | size = val; 166 | } 167 | 168 | if z > 0 { 169 | // Extract the Hilbert curve index and convert it to tile coordinates 170 | let (x, y) = h2xy::(id - size, z); 171 | TileCoord { z, x, y } 172 | } else { 173 | TileCoord { z: 0, x: 0, y: 0 } 174 | } 175 | } 176 | } 177 | 178 | impl From for TileId { 179 | fn from(coord: TileCoord) -> Self { 180 | let TileCoord { z, x, y } = coord; 181 | if z == 0 { 182 | // The 0/0/0 case would fail xy2h_discrete() 183 | TileId(0) 184 | } else { 185 | let base = PYRAMID_SIZE_BY_ZOOM 186 | .get(usize::from(z)) 187 | .expect("TileCoord should be valid"); // see TileCoord::new 188 | let tile_id = xy2h(x, y, z); 189 | 190 | TileId(base + tile_id) 191 | } 192 | } 193 | } 194 | 195 | #[cfg(test)] 196 | pub(crate) mod test { 197 | use crate::{MAX_TILE_ID, PYRAMID_SIZE_BY_ZOOM, TileCoord, TileId}; 198 | 199 | pub fn coord(z: u8, x: u32, y: u32) -> TileCoord { 200 | TileCoord::new(z, x, y).unwrap() 201 | } 202 | 203 | pub fn coord_to_id(z: u8, x: u32, y: u32) -> u64 { 204 | TileId::from(coord(z, x, y)).value() 205 | } 206 | 207 | pub fn id_to_coord(id: u64) -> (u8, u32, u32) { 208 | let coord = TileCoord::from(TileId::new(id).unwrap()); 209 | (coord.z(), coord.x(), coord.y()) 210 | } 211 | 212 | #[test] 213 | #[expect(clippy::unreadable_literal)] 214 | fn test_tile_id() { 215 | assert_eq!(TileId::new(0).unwrap().value(), 0); 216 | assert_eq!(TileId::new(MAX_TILE_ID + 1), None); 217 | assert_eq!(TileId::new(MAX_TILE_ID).unwrap().value(), MAX_TILE_ID); 218 | 219 | assert_eq!(coord_to_id(0, 0, 0), 0); 220 | assert_eq!(coord_to_id(1, 1, 0), 4); 221 | assert_eq!(coord_to_id(2, 1, 3), 11); 222 | assert_eq!(coord_to_id(3, 3, 0), 26); 223 | assert_eq!(coord_to_id(20, 0, 0), 366503875925); 224 | assert_eq!(coord_to_id(21, 0, 0), 1466015503701); 225 | assert_eq!(coord_to_id(22, 0, 0), 5864062014805); 226 | assert_eq!(coord_to_id(23, 0, 0), 23456248059221); 227 | assert_eq!(coord_to_id(24, 0, 0), 93824992236885); 228 | assert_eq!(coord_to_id(25, 0, 0), 375299968947541); 229 | assert_eq!(coord_to_id(26, 0, 0), 1501199875790165); 230 | assert_eq!(coord_to_id(27, 0, 0), 6004799503160661); 231 | assert_eq!(coord_to_id(28, 0, 0), 24019198012642645); 232 | assert_eq!(coord_to_id(31, 0, 0), 1537228672809129301); 233 | let max_v = (1 << 31) - 1; 234 | assert_eq!(coord_to_id(31, max_v, max_v), 4611686018427387903); 235 | assert_eq!(coord_to_id(31, 0, max_v), 3074457345618258602); 236 | assert_eq!(coord_to_id(31, max_v, 0), 6148914691236517204); 237 | } 238 | 239 | #[test] 240 | fn round_trip_ids() { 241 | const LAST_PYRAMID_IDX: usize = PYRAMID_SIZE_BY_ZOOM.len() - 1; 242 | for id in [ 243 | 0, 244 | 1, 245 | 2, 246 | 3, 247 | 4, 248 | 5, 249 | 6, 250 | PYRAMID_SIZE_BY_ZOOM[LAST_PYRAMID_IDX], 251 | PYRAMID_SIZE_BY_ZOOM[LAST_PYRAMID_IDX] - 1, 252 | PYRAMID_SIZE_BY_ZOOM[LAST_PYRAMID_IDX] + 1, 253 | MAX_TILE_ID - 1, 254 | MAX_TILE_ID, 255 | ] { 256 | test_id(id); 257 | } 258 | for id in 0..1000 { 259 | test_id(id); 260 | } 261 | } 262 | 263 | fn test_id(id: u64) { 264 | let id1 = TileId::new(id).unwrap(); 265 | let coord1 = TileCoord::from(id1); 266 | let coord2 = TileCoord::new(coord1.z, coord1.x, coord1.y).unwrap(); 267 | let id2 = TileId::from(coord2); 268 | assert_eq!(id, id2.value(), "Failed round-trip for id={id}"); 269 | } 270 | 271 | #[test] 272 | fn test_calc_tile_coords() { 273 | // Test round-trip conversion 274 | let test_cases = [ 275 | (0, 0, 0), 276 | (1, 1, 0), 277 | (2, 1, 3), 278 | (3, 3, 0), 279 | (20, 0, 0), 280 | (21, 0, 0), 281 | (22, 0, 0), 282 | (23, 0, 0), 283 | (24, 0, 0), 284 | (25, 0, 0), 285 | (26, 0, 0), 286 | (27, 0, 0), 287 | (28, 0, 0), 288 | ]; 289 | 290 | for (z, x, y) in test_cases { 291 | let (z2, x2, y2) = id_to_coord(coord_to_id(z, x, y)); 292 | assert_eq!( 293 | (z, x, y), 294 | (z2, x2, y2), 295 | "Failed round-trip for z={z}, x={x}, y={y}", 296 | ); 297 | } 298 | } 299 | } 300 | -------------------------------------------------------------------------------- /src/writer.rs: -------------------------------------------------------------------------------- 1 | use std::io::{BufWriter, Seek, Write}; 2 | 3 | use countio::Counter; 4 | use flate2::write::GzEncoder; 5 | 6 | use crate::PmtError::UnsupportedCompression; 7 | use crate::header::{HEADER_SIZE, MAX_INITIAL_BYTES}; 8 | use crate::{ 9 | Compression, DirEntry, Directory, Header, PmtError, PmtResult, TileCoord, TileId, TileType, 10 | }; 11 | 12 | /// Builder for creating a new writer. 13 | pub struct PmTilesWriter { 14 | header: Header, 15 | metadata: String, 16 | } 17 | 18 | /// `PMTiles` streaming writer. 19 | pub struct PmTilesStreamWriter { 20 | out: Counter>, 21 | header: Header, 22 | entries: Vec, 23 | n_addressed_tiles: u64, 24 | // TODO: Replace with digest HashMap for deduplicating non-subsequent tiles 25 | n_tile_contents: u64, 26 | prev_tile_data: Vec, 27 | } 28 | 29 | pub(crate) trait WriteTo { 30 | fn write_to(&self, writer: &mut W) -> std::io::Result<()>; 31 | 32 | fn write_compressed_to( 33 | &self, 34 | writer: &mut W, 35 | compression: Compression, 36 | ) -> PmtResult<()> { 37 | match compression { 38 | Compression::None => self.write_to(writer)?, 39 | Compression::Gzip => { 40 | let mut encoder = GzEncoder::new(writer, flate2::Compression::default()); 41 | self.write_to(&mut encoder)?; 42 | } 43 | v => Err(UnsupportedCompression(v))?, 44 | } 45 | Ok(()) 46 | } 47 | 48 | fn write_compressed_to_counted( 49 | &self, 50 | writer: &mut Counter, 51 | compression: Compression, 52 | ) -> PmtResult { 53 | let pos = writer.writer_bytes(); 54 | self.write_compressed_to(writer, compression)?; 55 | Ok(writer.writer_bytes() - pos) 56 | } 57 | 58 | fn compressed_size(&self, compression: Compression) -> PmtResult { 59 | let mut devnull = Counter::new(std::io::sink()); 60 | self.write_compressed_to(&mut devnull, compression)?; 61 | Ok(devnull.writer_bytes()) 62 | } 63 | } 64 | 65 | impl WriteTo for [u8] { 66 | fn write_to(&self, writer: &mut W) -> std::io::Result<()> { 67 | writer.write_all(self) 68 | } 69 | } 70 | 71 | impl PmTilesWriter { 72 | /// Create a new `PMTiles` writer with default values. 73 | #[must_use] 74 | pub fn new(tile_type: TileType) -> Self { 75 | let tile_compression = match tile_type { 76 | TileType::Mvt => Compression::Gzip, 77 | _ => Compression::None, 78 | }; 79 | let header = Header::new(tile_compression, tile_type); 80 | Self { 81 | header, 82 | metadata: "{}".to_string(), 83 | } 84 | } 85 | 86 | /// Set the compression for metadata and directories. 87 | #[must_use] 88 | pub fn internal_compression(mut self, compression: Compression) -> Self { 89 | self.header.internal_compression = compression; 90 | self 91 | } 92 | 93 | /// Set the compression for tile data. 94 | #[must_use] 95 | pub fn tile_compression(mut self, compression: Compression) -> Self { 96 | self.header.tile_compression = compression; 97 | self 98 | } 99 | 100 | /// Set the minimum zoom level of the tiles 101 | #[must_use] 102 | pub fn min_zoom(mut self, level: u8) -> Self { 103 | self.header.min_zoom = level; 104 | self 105 | } 106 | 107 | /// Set the maximum zoom level of the tiles 108 | #[must_use] 109 | pub fn max_zoom(mut self, level: u8) -> Self { 110 | self.header.max_zoom = level; 111 | self 112 | } 113 | 114 | /// Set the bounds of the tiles 115 | #[must_use] 116 | pub fn bounds(mut self, min_lon: f32, min_lat: f32, max_lon: f32, max_lat: f32) -> Self { 117 | self.header.min_latitude = min_lat; 118 | self.header.min_longitude = min_lon; 119 | self.header.max_latitude = max_lat; 120 | self.header.max_longitude = max_lon; 121 | self 122 | } 123 | 124 | /// Set the center zoom level. 125 | #[must_use] 126 | pub fn center_zoom(mut self, level: u8) -> Self { 127 | self.header.center_zoom = level; 128 | self 129 | } 130 | 131 | /// Set the center position. 132 | #[must_use] 133 | pub fn center(mut self, lon: f32, lat: f32) -> Self { 134 | self.header.center_latitude = lat; 135 | self.header.center_longitude = lon; 136 | self 137 | } 138 | 139 | /// Set the metadata, which must contain a valid JSON object. 140 | /// 141 | /// If the tile type has a value of MVT Vector Tile, the object must contain a key of `vector_layers` as described in the `TileJSON` 3.0 specification. 142 | #[must_use] 143 | pub fn metadata(mut self, metadata: &str) -> Self { 144 | self.metadata = metadata.to_string(); 145 | self 146 | } 147 | 148 | /// Create a new `PMTiles` writer. 149 | pub fn create(self, writer: W) -> PmtResult> { 150 | let mut out = Counter::new(BufWriter::new(writer)); 151 | 152 | // We use the following layout: 153 | // +--------+----------------+----------+-----------+------------------+ 154 | // | | | | | | 155 | // | Header | Root Directory | Metadata | Tile Data | Leaf Directories | 156 | // | | | | | | 157 | // +--------+----------------+----------+-----------+------------------+ 158 | // This allows writing without temporary files. But it requires Seek support. 159 | 160 | // Reserve space for the header and root directory 161 | out.write_all(&[0u8; MAX_INITIAL_BYTES])?; 162 | 163 | let metadata_length = self 164 | .metadata 165 | .as_bytes() 166 | .write_compressed_to_counted(&mut out, self.header.internal_compression)? 167 | as u64; 168 | 169 | let mut writer = PmTilesStreamWriter { 170 | out, 171 | header: self.header, 172 | entries: Vec::new(), 173 | n_addressed_tiles: 0, 174 | n_tile_contents: 0, 175 | prev_tile_data: vec![], 176 | }; 177 | writer.header.metadata_length = metadata_length; 178 | writer.header.data_offset = MAX_INITIAL_BYTES as u64 + metadata_length; 179 | 180 | Ok(writer) 181 | } 182 | } 183 | 184 | impl PmTilesStreamWriter { 185 | /// Add a tile to the writer. 186 | /// 187 | /// Tiles are deduplicated and written to output. 188 | /// The `tile_id` generated from `z/x/y` should be increasing for best read performance. 189 | pub fn add_tile(&mut self, coord: TileCoord, data: &[u8]) -> PmtResult<()> { 190 | self.add_tile_by_id(coord.into(), data) 191 | } 192 | 193 | /// Add a tile to the writer. 194 | /// 195 | /// Tiles are deduplicated and written to output. 196 | /// The `tile_id` should be increasing for best read performance. 197 | fn add_tile_by_id(&mut self, tile_id: TileId, data: &[u8]) -> PmtResult<()> { 198 | if data.is_empty() { 199 | // Ignore empty tiles, since the spec does not allow storing them 200 | return Ok(()); 201 | } 202 | 203 | let tile_id = tile_id.value(); 204 | let is_first = self.entries.is_empty(); 205 | if is_first && tile_id > 0 { 206 | self.header.clustered = false; 207 | } 208 | let mut first_entry = DirEntry { 209 | tile_id: 0, 210 | offset: 0, 211 | length: 0, 212 | run_length: 0, 213 | }; 214 | let last_entry = self.entries.last_mut().unwrap_or(&mut first_entry); 215 | 216 | self.n_addressed_tiles += 1; 217 | if !is_first 218 | && self.prev_tile_data == data 219 | && tile_id == last_entry.tile_id + u64::from(last_entry.run_length) 220 | { 221 | last_entry.run_length += 1; 222 | } else { 223 | let offset = last_entry.offset + u64::from(last_entry.length); 224 | // Write tile 225 | let len = 226 | data.write_compressed_to_counted(&mut self.out, self.header.tile_compression)?; 227 | let length = into_u32(len)?; 228 | self.n_tile_contents += 1; 229 | if tile_id != last_entry.tile_id + u64::from(last_entry.run_length) { 230 | self.header.clustered = false; 231 | } 232 | 233 | self.entries.push(DirEntry { 234 | tile_id, 235 | run_length: 1, // Will be increased by following identical tiles 236 | offset, 237 | length, 238 | }); 239 | 240 | self.prev_tile_data = data.to_vec(); 241 | } 242 | 243 | Ok(()) 244 | } 245 | 246 | /// Build root and leaf directories from entries. 247 | /// Leaf directories are written to the output. 248 | /// The root directory is returned. 249 | fn build_directories(&mut self) -> PmtResult { 250 | if !self.header.clustered { 251 | // Spec does only say that leaf directories *should* be in ascending order, 252 | // but sorted directories are better for readers anyway. 253 | self.entries.sort_by_key(|entry| entry.tile_id); 254 | } 255 | let (root_dir, num_leaves) = self.optimize_directories(MAX_INITIAL_BYTES - HEADER_SIZE)?; 256 | if num_leaves > 0 { 257 | // Write leaf directories 258 | for leaf in root_dir.entries() { 259 | let len = leaf.length as usize; 260 | let mut dir = Directory::with_capacity(len); 261 | for entry in self.entries.drain(0..len) { 262 | dir.push(entry); 263 | } 264 | dir.write_compressed_to(&mut self.out, self.header.internal_compression)?; 265 | } 266 | } 267 | Ok(root_dir) 268 | } 269 | 270 | fn optimize_directories(&self, target_root_len: usize) -> PmtResult<(Directory, usize)> { 271 | // Same logic as go-pmtiles (https://github.com/protomaps/go-pmtiles/blob/f1c24e6/pmtiles/directory.go#L368-L396) 272 | // and planetiler (https://github.com/onthegomap/planetiler/blob/6b3e152/planetiler-core/src/main/java/com/onthegomap/planetiler/pmtiles/WriteablePmtiles.java#L96-L118) 273 | if self.entries.len() < 16384 { 274 | let root_dir = Directory::from_entries(self.entries.clone()); 275 | let root_bytes = root_dir.compressed_size(self.header.internal_compression)?; 276 | // Case1: the entire directory fits into the target len 277 | if root_bytes <= target_root_len { 278 | return Ok((root_dir, 0)); 279 | } 280 | } 281 | 282 | // TODO: case 2: mixed tile entries/directory entries in root 283 | 284 | // case 3: root directory is leaf pointers only 285 | // use an iterative method, increasing the size of the leaf directory until the root fits 286 | 287 | let mut leaf_size = (self.entries.len() / 3500).max(4096); 288 | loop { 289 | let (root_dir, num_leaves) = self.build_roots_leaves(leaf_size)?; 290 | let root_bytes = root_dir.compressed_size(self.header.internal_compression)?; 291 | if root_bytes <= target_root_len { 292 | return Ok((root_dir, num_leaves)); 293 | } 294 | leaf_size += leaf_size / 5; // go-pmtiles: leaf_size *= 1.2 295 | } 296 | } 297 | 298 | fn build_roots_leaves(&self, leaf_size: usize) -> PmtResult<(Directory, usize)> { 299 | let mut root_dir = Directory::with_capacity(self.entries.len() / leaf_size); 300 | let mut offset = 0; 301 | for chunk in self.entries.chunks(leaf_size) { 302 | let leaf_size = self.dir_size(chunk)?; 303 | root_dir.push(DirEntry { 304 | tile_id: chunk[0].tile_id, 305 | offset, 306 | length: into_u32(leaf_size)?, 307 | run_length: 0, 308 | }); 309 | offset += leaf_size as u64; 310 | } 311 | 312 | let num_leaves = root_dir.entries().len(); 313 | Ok((root_dir, num_leaves)) 314 | } 315 | 316 | fn dir_size(&self, entries: &[DirEntry]) -> PmtResult { 317 | let dir = Directory::from_entries(entries.to_vec()); 318 | dir.compressed_size(self.header.internal_compression) 319 | } 320 | 321 | /// Finish writing the `PMTiles` file. 322 | pub fn finalize(mut self) -> PmtResult<()> { 323 | if let Some(last) = self.entries.last() { 324 | self.header.data_length = last.offset + u64::from(last.length); 325 | self.header.leaf_offset = self.header.data_offset + self.header.data_length; 326 | self.header.n_addressed_tiles = self.n_addressed_tiles.try_into().ok(); 327 | self.header.n_tile_entries = (self.entries.len() as u64).try_into().ok(); 328 | self.header.n_tile_contents = self.n_tile_contents.try_into().ok(); 329 | } 330 | // Write leaf directories and get a root directory 331 | let root_dir = self.build_directories()?; 332 | // Determine compressed root directory length 333 | let mut root_dir_buf = vec![]; 334 | root_dir.write_compressed_to(&mut root_dir_buf, self.header.internal_compression)?; 335 | self.header.root_length = root_dir_buf.len() as u64; 336 | 337 | // Write header and root directory 338 | self.out.rewind()?; 339 | self.header.write_to(&mut self.out)?; 340 | self.out.write_all(&root_dir_buf)?; 341 | self.out.flush()?; 342 | 343 | Ok(()) 344 | } 345 | } 346 | 347 | fn into_u32(v: usize) -> PmtResult { 348 | v.try_into().map_err(|_| PmtError::IndexEntryOverflow) 349 | } 350 | 351 | #[cfg(test)] 352 | #[cfg(feature = "mmap-async-tokio")] 353 | #[expect(clippy::float_cmp)] 354 | mod tests { 355 | use std::fs::File; 356 | 357 | use tempfile::NamedTempFile; 358 | 359 | use crate::header::{HEADER_SIZE, MAX_INITIAL_BYTES}; 360 | use crate::tests::RASTER_FILE; 361 | use crate::{ 362 | AsyncPmTilesReader, Compression, DirEntry, Directory, MmapBackend, PmTilesWriter, 363 | TileCoord, TileId, TileType, 364 | }; 365 | 366 | fn get_temp_file_path(suffix: &str) -> std::io::Result { 367 | let temp_file = NamedTempFile::with_suffix(suffix)?; 368 | Ok(temp_file.path().to_string_lossy().into_owned()) 369 | } 370 | 371 | #[tokio::test] 372 | async fn roundtrip_raster() { 373 | let backend = MmapBackend::try_from(RASTER_FILE).await.unwrap(); 374 | let tiles_in = AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 375 | let header_in = tiles_in.get_header(); 376 | let metadata_in = tiles_in.get_metadata().await.unwrap(); 377 | let num_tiles = header_in.n_addressed_tiles.unwrap(); 378 | 379 | let path = get_temp_file_path("pmtiles").unwrap(); 380 | // let path = "test.pmtiles".to_string(); 381 | let file = File::create(path.clone()).unwrap(); 382 | let mut writer = PmTilesWriter::new(header_in.tile_type) 383 | .max_zoom(header_in.max_zoom) 384 | .metadata(&metadata_in) 385 | .create(file) 386 | .unwrap(); 387 | for id in 0..num_tiles.into() { 388 | let id = TileId::new(id).unwrap(); 389 | let tile = tiles_in.get_tile(id).await.unwrap().unwrap(); 390 | writer.add_tile_by_id(id, &tile).unwrap(); 391 | } 392 | writer.finalize().unwrap(); 393 | 394 | let backend = MmapBackend::try_from(&path).await.unwrap(); 395 | let tiles_out = AsyncPmTilesReader::try_from_source(backend).await.unwrap(); 396 | 397 | // Compare headers 398 | let header_out = tiles_out.get_header(); 399 | // TODO: should be 3, but currently the ascii char 3, assert_eq!(header_in.version, header_out.version); 400 | assert_eq!(header_in.tile_type, header_out.tile_type); 401 | assert_eq!(header_in.n_addressed_tiles, header_out.n_addressed_tiles); 402 | assert_eq!(header_in.n_tile_entries, header_out.n_tile_entries); 403 | // assert_eq!(header_in.n_tile_contents, header_out.n_tile_contents); 404 | assert_eq!(Some(84), header_out.n_tile_contents.map(Into::into)); 405 | assert_eq!(header_in.min_zoom, header_out.min_zoom); 406 | assert_eq!(header_in.max_zoom, header_out.max_zoom); 407 | assert_eq!(header_in.center_zoom, header_out.center_zoom); 408 | assert_eq!(header_in.center_latitude, header_out.center_latitude); 409 | assert_eq!(header_in.center_longitude, header_out.center_longitude); 410 | assert_eq!( 411 | header_in.min_latitude.round(), 412 | header_out.min_latitude.round() 413 | ); 414 | assert_eq!( 415 | header_in.max_latitude.round(), 416 | header_out.max_latitude.round() 417 | ); 418 | assert_eq!(header_in.min_longitude, header_out.min_longitude); 419 | assert_eq!(header_in.max_longitude, header_out.max_longitude); 420 | assert_eq!(header_in.clustered, header_out.clustered); 421 | 422 | // Compare metadata 423 | let metadata_out = tiles_out.get_metadata().await.unwrap(); 424 | assert_eq!(metadata_in, metadata_out); 425 | 426 | // Compare tiles 427 | for (z, x, y) in [(0, 0, 0), (2, 2, 2), (3, 4, 5)] { 428 | let coord = TileCoord::new(z, x, y).unwrap(); 429 | let tile_in = tiles_in.get_tile(coord).await.unwrap().unwrap(); 430 | let tile_out = tiles_out.get_tile(coord).await.unwrap().unwrap(); 431 | assert_eq!(tile_in.len(), tile_out.len()); 432 | } 433 | } 434 | 435 | fn gen_entries(num_tiles: u64) -> (Directory, usize) { 436 | let path = get_temp_file_path("pmtiles").unwrap(); 437 | let file = File::create(path).unwrap(); 438 | let mut writer = PmTilesWriter::new(TileType::Png) 439 | // flate2 compression is extremely slow in debug mode 440 | .internal_compression(Compression::None) 441 | .create(file) 442 | .unwrap(); 443 | for tile_id in 0..num_tiles { 444 | writer.entries.push(DirEntry { 445 | tile_id, 446 | run_length: 1, 447 | offset: tile_id, 448 | length: 1, 449 | }); 450 | } 451 | writer 452 | .optimize_directories(MAX_INITIAL_BYTES - HEADER_SIZE) 453 | .unwrap() 454 | } 455 | 456 | #[test] 457 | fn no_leaves() { 458 | let (root_dir, num_leaves) = gen_entries(100); 459 | assert_eq!(num_leaves, 0); 460 | assert_eq!(root_dir.entries().len(), 100); 461 | } 462 | 463 | #[test] 464 | fn with_leaves() { 465 | let (root_dir, num_leaves) = gen_entries(20000); 466 | assert_eq!(num_leaves, 5); 467 | assert_eq!(root_dir.entries().len(), num_leaves); 468 | } 469 | 470 | #[test] 471 | fn unclustered() { 472 | let file = get_temp_file_path("pmtiles").unwrap(); 473 | let file = File::create(file).unwrap(); 474 | let mut writer = PmTilesWriter::new(TileType::Png).create(file).unwrap(); 475 | 476 | let id = TileId::new(0).unwrap(); 477 | writer.add_tile_by_id(id, &[0, 1, 2, 3]).unwrap(); 478 | assert!(writer.header.clustered); 479 | 480 | let id = TileId::new(2).unwrap(); 481 | writer.add_tile_by_id(id, &[0, 1, 2, 3]).unwrap(); 482 | assert!(!writer.header.clustered); 483 | 484 | writer.finalize().unwrap(); 485 | } 486 | } 487 | --------------------------------------------------------------------------------