├── .circleci
└── config.yml
├── .gitignore
├── .hgignore
├── .hgtags
├── CHANGELOG.rst
├── LICENSE
├── MANIFEST.in
├── NOTICE
├── README.rst
├── gdal2mbtiles
├── __init__.py
├── constants.py
├── default_rgba.png
├── exceptions.py
├── gd_types.py
├── gdal.py
├── helpers.py
├── main.py
├── mbtiles.py
├── renderers.py
├── storages.py
├── utils.py
└── vips.py
├── pytest.ini
├── setup.cfg
├── setup.py
├── tests
├── __init__.py
├── bluemarble-aligned-ll.tif
├── bluemarble-foreign.tif
├── bluemarble-slightly-too-big.tif
├── bluemarble-spanning-foreign.tif
├── bluemarble-spanning-ll.tif
├── bluemarble-wgs84.tif
├── bluemarble.tif
├── bluemarble.xcf
├── paletted.nodata.tif
├── paletted.tif
├── srtm.nodata.tif
├── srtm.tif
├── test_gdal.py
├── test_helpers.py
├── test_mbtiles.py
├── test_renderers.py
├── test_scripts.py
├── test_spatial_reference.py
├── test_storages.py
├── test_types.py
├── test_vips.py
└── upsampling.tif
└── tox.ini
/.circleci/config.yml:
--------------------------------------------------------------------------------
1 | version: 2.1
2 |
3 | commands:
4 | add-ubuntugis-ppa:
5 | parameters:
6 | archive: &ubuntugis-parameter
7 | type: enum
8 | enum:
9 | - "" # means don't use the ppa
10 | - stable
11 | - unstable
12 | default: ""
13 |
14 | steps:
15 | - run:
16 | name: Add ubuntugis PPA
17 | command: |
18 | apt-get update && apt-get install -y software-properties-common
19 | if [[ "<< parameters.archive >>" = "stable" ]]
20 | then
21 | archive='ppa'
22 | else
23 | archive='ubuntugis-unstable'
24 | fi
25 | add-apt-repository -y ppa:ubuntugis/${archive}
26 |
27 | install-system-packages:
28 | steps:
29 | - run:
30 | name: Install most system packages
31 | command: |
32 | apt-get update && \
33 | DEBIAN_FRONTEND=noninteractive apt-get install -y \
34 | libgdal-dev \
35 | gdal-bin \
36 | libtiff5 \
37 | optipng \
38 | pngquant \
39 | python3-pip \
40 |
41 | - run:
42 | name: Install libvips
43 | command: |
44 | DEBIAN_FRONTEND=noninteractive \
45 | apt-get install -y --no-install-recommends \
46 | libvips \
47 | libvips-dev \
48 |
49 | run-tox-tests:
50 | parameters:
51 | tox: &tox-parameter
52 | type: enum
53 | enum:
54 | - py2
55 | - py3
56 | - pinned-vips
57 | default: py3
58 |
59 | steps:
60 | - checkout
61 | - run:
62 | name: Run tox tests
63 | command: |
64 | python3 -m pip install tox
65 | GDAL_VERSION=$(gdal-config --version) tox -e << parameters.tox >> -- "-m not skip_on_ci"
66 |
67 | executors:
68 | xenial:
69 | docker:
70 | - image: ubuntu:xenial
71 | resource_class: small
72 |
73 | bionic:
74 | docker:
75 | - image: ubuntu:bionic
76 | resource_class: small
77 |
78 | focal:
79 | docker:
80 | - image: ubuntu:focal
81 | resource_class: small
82 |
83 | jobs:
84 | test:
85 | parameters:
86 | executor:
87 | type: executor
88 | ubuntugis:
89 | <<: *ubuntugis-parameter
90 | tox:
91 | <<: *tox-parameter
92 |
93 | executor: << parameters.executor >>
94 | steps:
95 | - when:
96 | condition: << parameters.ubuntugis >>
97 | steps:
98 | - add-ubuntugis-ppa:
99 | archive: << parameters.ubuntugis >>
100 | - install-system-packages
101 | - run-tox-tests:
102 | tox: << parameters.tox >>
103 |
104 | workflows:
105 | version: 2
106 |
107 | xenial:
108 | jobs:
109 |
110 | # pypi has no matching GDAL version for Xenial's default 1.11
111 | # - test:
112 | # name: test-xenial
113 | # executor: xenial
114 |
115 | - test:
116 | name: test-xenial-stablegis
117 | executor: xenial
118 | ubuntugis: stable
119 |
120 | - test:
121 | name: test-xenial-stablegis-py2
122 | executor: xenial
123 | ubuntugis: stable
124 | tox: py2
125 |
126 | - test:
127 | name: test-xenial-stablegis-vips
128 | executor: xenial
129 | ubuntugis: stable
130 | tox: pinned-vips
131 |
132 | - test:
133 | name: test-xenial-unstablegis
134 | executor: xenial
135 | ubuntugis: unstable
136 |
137 | - test:
138 | name: test-xenial-unstablegis-vips
139 | executor: xenial
140 | ubuntugis: unstable
141 | tox: pinned-vips
142 |
143 | bionic:
144 | jobs:
145 | - test:
146 | name: test-bionic
147 | executor: bionic
148 |
149 | - test:
150 | name: test-bionic-vips
151 | executor: bionic
152 | tox: pinned-vips
153 |
154 | - test:
155 | name: test-bionic-stablegis
156 | executor: bionic
157 | ubuntugis: stable
158 |
159 | - test:
160 | name: test-bionic-stablegis-vips
161 | executor: bionic
162 | ubuntugis: stable
163 | tox: pinned-vips
164 |
165 | - test:
166 | name: test-bionic-unstablegis
167 | executor: bionic
168 | ubuntugis: unstable
169 |
170 | - test:
171 | name: test-bionic-unstablegis-vips
172 | executor: bionic
173 | ubuntugis: unstable
174 | tox: pinned-vips
175 |
176 | focal:
177 | jobs:
178 | - test:
179 | name: test-focal
180 | executor: focal
181 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | *.egg-info/
3 | MANIFEST
4 |
5 | .cache/
6 | .tox/
--------------------------------------------------------------------------------
/.hgignore:
--------------------------------------------------------------------------------
1 | syntax: glob
2 |
3 | *.py[co]
4 |
5 | build/
6 | dist/
7 |
8 | # virtualenv stuff
9 | bin/
10 | include/
11 | lib/
12 | local/
13 |
--------------------------------------------------------------------------------
/.hgtags:
--------------------------------------------------------------------------------
1 | 0b5b8c4d17b8478e7d9aff9c5fb209b9ff422418 version-1.0.0
2 | fcf49aeb100bc39494892aa37f4d983826f8fedc version-1.1.0
3 | 69d8b7d7c0c9a42abd2cb1889e425852cc91b140 version-1.2.0
4 | 69d8b7d7c0c9a42abd2cb1889e425852cc91b140 version-1.2.0
5 | 2f00ad4ab2ecafc5d595b9b8ab4dde9eb83cbebc version-1.2.0
6 |
--------------------------------------------------------------------------------
/CHANGELOG.rst:
--------------------------------------------------------------------------------
1 | =============
2 | Release Notes
3 | =============
4 |
5 | The format is based on `Keep a Changelog `_
6 | and this project attempts to adhere to `Semantic Versioning `_.
7 |
8 | Unreleased
9 | ------------
10 |
11 | 2.1.5
12 | ------
13 | * Fix SpatialReference.GetEPSGCode failling to recognise QGIS style PROJCS name.
14 |
15 | 2.1.4
16 | ------
17 | * Fixing GDAL 3 backwards incompatible change which switches axis in coordinate transformation - see: https://github.com/OSGeo/gdal/issues/1546
18 |
19 | 2.1.3
20 | ------
21 |
22 | * Fix float overflow bug
23 | * Unpin pyvips and fix related issue - install pyvips==2.1.8 if any issues
24 | * Fix renderer tests
25 | * Fix deprecation warnings
26 | * Fix python3.7+ pytest errors
27 | * Update author email
28 | * Update CI
29 | * Update README
30 |
31 | 2.1.2
32 | -----
33 |
34 | * Update docs
35 | * Pin pyvips
36 |
37 | 2.1.1
38 | -----
39 |
40 | * Revert commit f7fde54, which reintroduced tiling issues fixed by 9231133.
41 |
42 |
43 | 2.1.0
44 | -----
45 |
46 | * Add --png8 argument to quantize 32-bit RGBA to 8-bit RGBA paletted PNGs.
47 | * Specify `NUM_THREADS` option for gdal_translate to use all CPUs
48 | * Update MANIFEST.in to include required files
49 |
50 |
51 | 2.0.0
52 | -----
53 |
54 | * Add support for Python 3.5 and 3.6
55 | * Replace vipsCC with pyvips
56 | * Use pytest and tox for testing
57 | * types.py has been renamed to gd_types.py to avoid import conflicts
58 | * Remove multiprocessing as it creates noise in mbtiles output files with
59 | pyvips and doesn't appear to have a significant impact on processing speeds
60 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright [yyyy] [name of copyright owner]
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
203 |
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include LICENSE
2 | include NOTICE
3 |
4 | include gdal2mbtiles/default_rgba.png
5 |
--------------------------------------------------------------------------------
/NOTICE:
--------------------------------------------------------------------------------
1 | gdal2mbtiles
2 | Copyright 2012 Ecometrica
3 |
4 | This product includes software developed at
5 | Ecometrica (http://ecometrica.com/).
6 |
--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
1 | ============
2 | gdal2mbtiles
3 | ============
4 |
5 | Convert GDAL-readable datasets into an MBTiles file
6 | ===================================================
7 |
8 | **gdal2mbtiles** helps you generate web mapping tiles that can be shown
9 | through a browser-based mapping library on your website.
10 |
11 | `GDAL-readable files`_ are images that are georeference, that means that
12 | they are positioned and projected on to the world. In order to display a
13 | dynamic map on the web, you don't want to serve the whole image at once,
14 | so it must be sliced into tiles that are hosted by a tile server.
15 |
16 | The MBTiles_ file format was developed by MapBox_ to make tile storage
17 | easier. You can upload the final file to their service, or run your own
18 | tile server. MapBox provides one called TileStream_.
19 |
20 | Later versions of GDAL (>= 2) allow generation of mbtiles files via the
21 | ``gdal_translate`` and ``gdaladdo`` commands. However, gdal2mbtiles offers some
22 | advantages:
23 |
24 | * allows you to specify an upper resolution/zoom level. GDAL always uses the
25 | native resolution of the input raster to determine the highest zoom level of
26 | the mbtiles output, whereas gdal2mbtiles can also upsample to create zoom levels
27 | at a higher resolution than your original file.
28 | * the ``gdal_translate`` command only converts the geotiff at the native resolution,
29 | so the lower resolutions are added to the file via overviews (``gdaladdo``)
30 | * ``gdaladdo`` can only add overviews down to the zoom level corresponding to
31 | the size of the tile/block size (256x256). gdal2mbtiles can always create images
32 | down to zoom level 1.
33 | * performance: gdal2mbtiles uses pyvips for image processing, which is parallel
34 | and quick. Compared to the equivalent processing with GDAL, gdal2mbtiles is
35 | typically 2-4 times quicker. For example:
36 |
37 | * a resolution 14 file, 13000x11000 pixels, min resolution 0, max resolution
38 | 14: ~5 minutes with gdal2mbtiles and ~8 minutes with GDAL commands.
39 | * a resoluton 11 file, 200,000x200,000, zoom level 11 only: ~30min with
40 | gdal2mbtiles and ~133min with GDAL (with ``GDAL_CACHE_MAX`` and
41 | ``GDAL_NUM_THREADS`` options)
42 |
43 |
44 | Installation
45 | ============
46 |
47 | PyPI package page: https://pypi.python.org/pypi/gdal2mbtiles/
48 |
49 | .. warning:: gdal2mbtiles requires Python 2.7 or higher and relies on
50 | installing the items from the `External Dependencies`_ section below *before*
51 | the python package.
52 |
53 | Using pip::
54 |
55 | $ pip install gdal2mbtiles
56 |
57 | From source::
58 |
59 | $ git clone https://github.com/ecometrica/gdal2mbtiles.git
60 | $ cd gdal2mbtiles
61 | $ python setup.py install
62 |
63 | External Dependencies
64 | ---------------------
65 |
66 | We rely on GDAL_ to read georeferenced datasets.
67 | Under Debian or Ubuntu, you can install the GDAL library & binary via apt.
68 |
69 | Default GDAL versions in Ubuntu LTS:
70 |
71 | * Xenial: 1.11
72 | * Bionic: 2.2
73 | * Focal: 3.0
74 |
75 | .. warning::
76 | GDAL 2 is the current supported version.
77 | GDAL 3 support is in progress - `contributions <#contributing>`_ welcome!
78 |
79 | We recommend using the `UbuntuGIS`_ PPA to get more recent versions of GDAL, if
80 | needed, as is the case for Xenial.
81 |
82 | .. code-block:: sh
83 |
84 | sudo add-apt-repository ppa:ubuntugis/ppa && sudo apt-get update
85 | sudo apt-get install gdal-bin libgdal-dev
86 |
87 | The ubuntugis PPA also usually includes ``python-gdal`` or ``python3-gdal``
88 | that will install the python bindings at the system level. Installing
89 | that may be enough for you if you aren't planning to use a non-default python
90 | or a `virtual environment`_.
91 |
92 | Otherwise, you will also need to install the GDAL python bindings package from
93 | `PyPI `_. Make sure to install the version that matches the installed
94 | GDAL library. You can double-check that version with ``gdal-config --version``.
95 |
96 | .. code-block:: sh
97 |
98 | pip install \
99 | --global-option=build_ext \
100 | --global-option=--gdal-config=/usr/bin/gdal-config \
101 | --global-option=--include-dirs=/usr/include/gdal/ \
102 | GDAL=="$(gdal-config --version)"
103 |
104 | We also rely on VIPS_ (version 8.2+) to do fast image processing.
105 |
106 | Under Debian or Ubuntu, run the following to install it without the GUI nip2::
107 |
108 | $ sudo apt-get install --no-install-recommends libvips libvips-dev
109 |
110 | You'll also need a few other libraries to deal with large TIFF files and
111 | to optimize the resulting PNG tiles.
112 |
113 | Under Debian or Ubuntu, run the following to install them::
114 |
115 | $ sudo apt-get install libtiff5 optipng pngquant
116 |
117 |
118 | Command Line Interface
119 | ======================
120 |
121 | .. code-block:: console
122 |
123 | $ gdal2mbtiles --help
124 | usage: gdal2mbtiles [-h] [-v] [--name NAME] [--description DESCRIPTION]
125 | [--layer-type {baselayer,overlay}] [--version VERSION]
126 | [--format {jpg,png}]
127 | [--spatial-reference SPATIAL_REFERENCE]
128 | [--resampling {near,bilinear,cubic,cubicspline,lanczos}]
129 | [--min-resolution MIN_RESOLUTION]
130 | [--max-resolution MAX_RESOLUTION] [--fill-borders]
131 | [--no-fill-borders] [--zoom-offset N]
132 | [--coloring {gradient,palette,exact}]
133 | [--color BAND-VALUE:HTML-COLOR]
134 | [--colorize-band COLORIZE-BAND]
135 | [--png8 PNG8]
136 | [INPUT] [OUTPUT]
137 |
138 | Converts a GDAL-readable into an MBTiles file
139 |
140 | optional arguments:
141 | -h, --help show this help message and exit
142 | -v, --verbose explain what is being done
143 |
144 | Positional arguments:
145 | INPUT GDAL-readable file.
146 | OUTPUT Output filename. Defaults to INPUT.mbtiles
147 |
148 | MBTiles metadata arguments:
149 | --name NAME Human-readable name of the tileset. Defaults to INPUT
150 | --description DESCRIPTION
151 | Description of the layer. Defaults to ""
152 | --layer-type {baselayer,overlay}
153 | Type of layer. Defaults to "overlay"
154 | --version VERSION Version of the tileset. Defaults to "1.0.0"
155 | --format {jpg,png} Tile image format. Defaults to "png"
156 |
157 | GDAL warp arguments:
158 | --spatial-reference SPATIAL_REFERENCE
159 | Destination EPSG spatial reference. Defaults to 3857
160 | --resampling {near,bilinear,cubic,cubicspline,lanczos}
161 | Resampling algorithm for warping. Defaults to "near"
162 | (nearest-neighbour)
163 |
164 | Rendering arguments:
165 | --min-resolution MIN_RESOLUTION
166 | Minimum resolution/zoom level to render and slice.
167 | Defaults to None (do not downsample)
168 | --max-resolution MAX_RESOLUTION
169 | Maximum resolution/zoom level to render and slice.
170 | Defaults to None (do not upsample)
171 | --fill-borders Fill image to whole world with empty tiles. Default.
172 | --no-fill-borders Do not add borders to fill image.
173 | --zoom-offset N Offset zoom level by N to fit unprojected images to
174 | square maps. Defaults to 0.
175 | --png8 Quantizes 32-bit RGBA to 8-bit RGBA paletted PNGs.
176 | value range from 2 to 256. Default to False.
177 |
178 | Coloring arguments:
179 | --coloring {gradient,palette,exact}
180 | Coloring algorithm.
181 | --color BAND-VALUE:HTML-COLOR
182 | Examples: --color="0:#ff00ff" --color=255:red
183 | --colorize-band COLORIZE-BAND
184 | Raster band to colorize. Defaults to 1
185 |
186 |
187 | Contributing
188 | ============
189 |
190 | Reporting bugs and submitting patches
191 | -------------------------------------
192 |
193 | Please check our `issue tracker`_ for known bugs and feature requests.
194 |
195 | We accept pull requests for fixes and new features.
196 |
197 | Development and Testing
198 | -----------------------
199 |
200 | We use `Tox`_ and `Pytest`_ to test locally and `CircleCI`_ for remote testing.
201 |
202 | 1. Clone the repo
203 | 2. Install whichever `External Dependencies`_ are suitable for your OS/VM.
204 | 3. Create and activate a `virtual environment`_
205 | 4. Install tox: ``pip install tox``
206 | 5. Set the GDAL_CONFIG env var for tox via the venv activations script.
207 |
208 | If using virtualenv:
209 | ``echo 'export GDAL_VERSION=$(gdal-config --version)' >> $VIRTUAL_ENV/bin/postactivate``
210 |
211 | If using venv:
212 | ``echo 'export GDAL_VERSION=$(gdal-config --version)' >> $VIRTUAL_ENV/bin/activate``
213 |
214 | 6. Run tests to confirm all is working: ``tox``
215 | 7. Do some development:
216 |
217 | - Make some changes
218 | - Run the tests
219 | - Fix any errors
220 | - Run the tests again
221 | - Update CHANGELOG.rst with a line about the change in the UNRELEASED section
222 | - Add yourself to AUTHORS.rst if not already there
223 | - Write a nice commit message
224 | - Repeat
225 |
226 | 8. Make a PR
227 |
228 | You don't need to worry initially about testing in every combination of GDAL
229 | and Ubuntu, leave that to the remote CI build matrix when you make a PR and let
230 | the reviewers figure out if it needs more work from that.
231 |
232 | Credits
233 | =======
234 |
235 | Maxime Dupuis and Simon Law wrote this program, with the generous
236 | support of Ecometrica_.
237 |
238 | See AUTHORS.rst for the full list of contributors.
239 |
240 | .. _GDAL-readable files: http://www.gdal.org/formats_list.html
241 | .. _MBTiles: http://mapbox.com/developers/mbtiles/
242 | .. _MapBox: http://mapbox.com/
243 | .. _TileStream: https://github.com/mapbox/tilestream
244 |
245 | .. _GDAL: http://www.gdal.org/
246 | .. _UbuntuGIS: https://launchpad.net/~ubuntugis/
247 | .. _VIPS: http://www.vips.ecs.soton.ac.uk/
248 |
249 | .. _GDAL_PyPI: https://https://pypi.org/project/GDAL/
250 | .. _Tox: https://tox.readthedocs.io/
251 | .. _Pytest: https://docs.pytest.org/
252 | .. _virtual environment: https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment
253 |
254 | .. _issue tracker: https://github.com/ecometrica/gdal2mbtiles/issues
255 | .. _Ecometrica: http://ecometrica.com/
256 |
257 | .. _CircleCI: https://circleci.com/
258 |
--------------------------------------------------------------------------------
/gdal2mbtiles/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Licensed to Ecometrica under one or more contributor license
4 | # agreements. See the NOTICE file distributed with this work
5 | # for additional information regarding copyright ownership.
6 | # Ecometrica licenses this file to you under the Apache
7 | # License, Version 2.0 (the "License"); you may not use this
8 | # file except in compliance with the License. You may obtain a
9 | # copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 |
20 | from __future__ import (absolute_import, division, print_function,
21 | unicode_literals)
22 |
23 | __version__ = '2.1.5'
24 |
--------------------------------------------------------------------------------
/gdal2mbtiles/constants.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Licensed to Ecometrica under one or more contributor license
4 | # agreements. See the NOTICE file distributed with this work
5 | # for additional information regarding copyright ownership.
6 | # Ecometrica licenses this file to you under the Apache
7 | # License, Version 2.0 (the "License"); you may not use this
8 | # file except in compliance with the License. You may obtain a
9 | # copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 |
20 | from __future__ import (absolute_import, division, print_function,
21 | unicode_literals)
22 |
23 | from math import pi
24 | from numpy import array
25 |
26 |
27 | # EPSG constants
28 | EPSG_WEB_MERCATOR = 3857
29 |
30 | # ESRI / QGIS constants with their EPSG code equivalent
31 | ESRI_102113_PROJ = 'WGS_1984_Web_Mercator' # EPSG:3785
32 | ESRI_102100_PROJ = 'WGS_1984_Web_Mercator_Auxiliary_Sphere' # EPSG:3857
33 | QGIS_3857_PROJ = 'WGS 84 / Pseudo-Mercator' # EPSG:3857
34 |
35 | # Output constants
36 | TILE_SIDE = 256 # in pixels
37 |
38 | # Command-line programs
39 | GDALINFO = 'gdalinfo'
40 | GDALTRANSLATE = 'gdal_translate'
41 | GDALWARP = 'gdalwarp'
42 |
43 | # SEMI_MAJOR is a constant referring to the WGS84 Semi Major Axis.
44 | WGS84_SEMI_MAJOR = 6378137.0
45 |
46 | # Note: web-Mercator = pseudo-Mercator = EPSG 3857
47 | # The extents of the web-Mercator are constants.
48 | # Since the projection is formed from a sphere the extents of the projection
49 | # form a square.
50 | # For the values of the extents refer to:
51 | # OpenLayer lib: http://docs.openlayers.org/library/spherical_mercator.html
52 | EPSG3857_EXTENT = pi * WGS84_SEMI_MAJOR
53 |
54 | EPSG3857_EXTENTS = array([[-EPSG3857_EXTENT]*2, [EPSG3857_EXTENT]*2])
55 |
--------------------------------------------------------------------------------
/gdal2mbtiles/default_rgba.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/gdal2mbtiles/default_rgba.png
--------------------------------------------------------------------------------
/gdal2mbtiles/exceptions.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Licensed to Ecometrica under one or more contributor license
4 | # agreements. See the NOTICE file distributed with this work
5 | # for additional information regarding copyright ownership.
6 | # Ecometrica licenses this file to you under the Apache
7 | # License, Version 2.0 (the "License"); you may not use this
8 | # file except in compliance with the License. You may obtain a
9 | # copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 |
20 | from __future__ import (absolute_import, division, print_function,
21 | unicode_literals)
22 |
23 | from subprocess import CalledProcessError
24 |
25 |
26 | class GdalError(RuntimeError):
27 | # HACK: GDAL uses RuntimeError for everything!!!!!!! :-(
28 | pass
29 |
30 |
31 | class CalledGdalError(CalledProcessError, GdalError):
32 | """Error when calling a GDAL command-line utility."""
33 | def __init__(self, returncode, cmd, output=None, error=None):
34 | super(CalledGdalError, self).__init__(returncode=returncode, cmd=cmd,
35 | output=output)
36 | self.error = error
37 |
38 | def __str__(self):
39 | return super(CalledGdalError, self).__str__() + ': %s' % self.error
40 |
41 |
42 | class UnalignedInputError(ValueError):
43 | pass
44 |
45 |
46 | class UnknownResamplingMethodError(ValueError):
47 | pass
48 |
49 |
50 | class VrtError(ValueError):
51 | pass
52 |
--------------------------------------------------------------------------------
/gdal2mbtiles/gd_types.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Licensed to Ecometrica under one or more contributor license
4 | # agreements. See the NOTICE file distributed with this work
5 | # for additional information regarding copyright ownership.
6 | # Ecometrica licenses this file to you under the Apache
7 | # License, Version 2.0 (the "License"); you may not use this
8 | # file except in compliance with the License. You may obtain a
9 | # copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 |
20 | from __future__ import (absolute_import, division, print_function,
21 | unicode_literals)
22 |
23 | from collections import namedtuple
24 |
25 | import webcolors
26 |
27 |
28 | GdalFormat = namedtuple(typename='GdalFormat',
29 | field_names=['name', 'attributes', 'description',
30 | 'can_read', 'can_write', 'can_update',
31 | 'has_virtual_io'])
32 |
33 |
34 | def enum(**enums):
35 | E = namedtuple(typename='enum',
36 | field_names=list(enums.keys()))
37 | return E(**enums)
38 |
39 |
40 | _rgba = namedtuple(typename='_rgba',
41 | field_names=['r', 'g', 'b', 'a'])
42 |
43 |
44 | class rgba(_rgba):
45 | """Represents an RGBA color."""
46 | def __new__(cls, r, g, b, a=255):
47 | return super(rgba, cls).__new__(cls, r, g, b, a)
48 |
49 | @classmethod
50 | def webcolor(cls, color):
51 | """Returns an RGBA color from its HTML/CSS representation."""
52 | if color.startswith('#'):
53 | return cls(*webcolors.hex_to_rgb(color))
54 | return cls(*webcolors.name_to_rgb(color))
55 |
56 |
57 | _Extents = namedtuple('Extents', ['lower_left', 'upper_right'])
58 |
59 |
60 | class Extents(_Extents):
61 | def __contains__(self, other):
62 | if isinstance(other, type(self)):
63 | # TODO: Support testing against own type
64 | raise NotImplementedError()
65 | elif isinstance(other, (tuple, list, XY)):
66 | return (self.lower_left.x <= other[0] < self.upper_right.x and
67 | self.lower_left.y <= other[1] < self.upper_right.y)
68 | raise TypeError("Can't handle {0!r}".format(other))
69 |
70 | def almost_equal(self, other, places=None, delta=None):
71 | return (self.lower_left.almost_equal(other.lower_left,
72 | places=places, delta=delta) and
73 | self.upper_right.almost_equal(other.upper_right,
74 | places=places, delta=delta))
75 |
76 | @property
77 | def dimensions(self):
78 | return self.upper_right - self.lower_left
79 |
80 |
81 | _XY = namedtuple('XY', ['x', 'y'])
82 |
83 |
84 | class XY(_XY):
85 | def __add__(self, other):
86 | return type(self)(x=self.x + other.x,
87 | y=self.y + other.y)
88 |
89 | def __sub__(self, other):
90 | return type(self)(x=self.x - other.x,
91 | y=self.y - other.y)
92 |
93 | def __mul__(self, other):
94 | return type(self)(x=self.x * other,
95 | y=self.y * other)
96 |
97 | def __truediv__(self, other):
98 | return type(self)(x=self.x / other,
99 | y=self.y / other)
100 |
101 | def floor(self):
102 | return type(self)(int(self.x), int(self.y))
103 |
104 | def almost_equal(self, other, places=None, delta=None):
105 | if self.x == other[0] and self.y == other[1]:
106 | return True # Shortcut
107 |
108 | if delta is not None and places is not None:
109 | raise TypeError("specify delta or places not both")
110 |
111 | if delta is not None:
112 | return (abs(self.x - other[0]) <= delta and
113 | abs(self.y - other[1]) <= delta)
114 |
115 | if places is None:
116 | places = 7
117 |
118 | return (round(abs(other[0] - self.x), places) == 0 and
119 | round(abs(other[1] - self.y), places) == 0)
120 |
--------------------------------------------------------------------------------
/gdal2mbtiles/helpers.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Licensed to Ecometrica under one or more contributor license
4 | # agreements. See the NOTICE file distributed with this work
5 | # for additional information regarding copyright ownership.
6 | # Ecometrica licenses this file to you under the Apache
7 | # License, Version 2.0 (the "License"); you may not use this
8 | # file except in compliance with the License. You may obtain a
9 | # copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 |
20 | from __future__ import (absolute_import, division, print_function,
21 | unicode_literals)
22 |
23 | from functools import partial
24 | from tempfile import NamedTemporaryFile
25 |
26 | from .gdal import Dataset, preprocess
27 | from .renderers import PngRenderer
28 | from .storages import MbtilesStorage, NestedFileStorage, SimpleFileStorage
29 | from .vips import TmsPyramid, validate_resolutions
30 |
31 |
32 | def image_mbtiles(inputfile, outputfile, metadata,
33 | min_resolution=None, max_resolution=None, fill_borders=None,
34 | zoom_offset=None, colors=None, renderer=None,
35 | preprocessor=None, pngdata=None):
36 | """
37 | Slices a GDAL-readable inputfile into a pyramid of PNG tiles.
38 |
39 | inputfile: Filename
40 | outputfile: The output .mbtiles file.
41 | min_resolution: Minimum resolution to downsample tiles.
42 | max_resolution: Maximum resolution to upsample tiles.
43 | fill_borders: Fill borders of image with empty tiles.
44 | zoom_offset: Offset zoom level to fit unprojected images to square maps.
45 |
46 | colors: Color palette applied to single band files.
47 | colors=ColorGradient({0: rgba(0, 0, 0, 255),
48 | 10: rgba(255, 255, 255, 255)})
49 | Defaults to no colorization.
50 | preprocessor: Function to run on the TmsPyramid before slicing.
51 |
52 | If `min_resolution` is None, don't downsample.
53 | If `max_resolution` is None, don't upsample.
54 | """
55 |
56 | if pngdata is None:
57 | pngdata = dict()
58 |
59 | if renderer is None:
60 | renderer = PngRenderer(**pngdata)
61 |
62 | with MbtilesStorage.create(filename=outputfile,
63 | metadata=metadata,
64 | zoom_offset=zoom_offset,
65 | renderer=renderer) as storage:
66 | pyramid = TmsPyramid(inputfile=inputfile,
67 | storage=storage,
68 | min_resolution=min_resolution,
69 | max_resolution=max_resolution)
70 | if preprocessor is None:
71 | preprocessor = colorize
72 |
73 | pyramid = preprocessor(**locals())
74 |
75 | pyramid.slice(fill_borders=fill_borders)
76 |
77 | # Add metadata extensions
78 | if zoom_offset is None:
79 | zoom_offset = 0
80 | if min_resolution is None:
81 | min_resolution = pyramid.resolution
82 | if max_resolution is None:
83 | max_resolution = pyramid.resolution
84 |
85 | metadata = storage.mbtiles.metadata
86 | metadata['x-minzoom'] = min_resolution + zoom_offset
87 | metadata['x-maxzoom'] = max_resolution + zoom_offset
88 |
89 |
90 | def image_pyramid(inputfile, outputdir,
91 | min_resolution=None, max_resolution=None, fill_borders=None,
92 | colors=None, renderer=None, preprocessor=None):
93 | """
94 | Slices a GDAL-readable inputfile into a pyramid of PNG tiles.
95 |
96 | inputfile: Filename
97 | outputdir: The output directory for the PNG tiles.
98 | min_resolution: Minimum resolution to downsample tiles.
99 | max_resolution: Maximum resolution to upsample tiles.
100 | fill_borders: Fill borders of image with empty tiles.
101 | preprocessor: Function to run on the TmsPyramid before slicing.
102 |
103 | Filenames are in the format ``{tms_z}/{tms_x}/{tms_y}.png``.
104 |
105 | If a tile duplicates another tile already known to this process, a symlink
106 | may be created instead of rendering the same tile to PNG again.
107 |
108 | If `min_resolution` is None, don't downsample.
109 | If `max_resolution` is None, don't upsample.
110 | """
111 | if renderer is None:
112 | renderer = PngRenderer()
113 | storage = NestedFileStorage(outputdir=outputdir,
114 | renderer=renderer)
115 | pyramid = TmsPyramid(inputfile=inputfile,
116 | storage=storage,
117 | min_resolution=min_resolution,
118 | max_resolution=max_resolution)
119 | if preprocessor is None:
120 | preprocessor = colorize
121 | pyramid = preprocessor(**locals())
122 | pyramid.slice(fill_borders=fill_borders)
123 |
124 |
125 | def image_slice(inputfile, outputdir, fill_borders=None,
126 | colors=None, renderer=None, preprocessor=None):
127 | """
128 | Slices a GDAL-readable inputfile into PNG tiles.
129 |
130 | inputfile: Filename
131 | outputdir: The output directory for the PNG tiles.
132 | fill_borders: Fill borders of image with empty tiles.
133 | colors: Color palette applied to single band files.
134 | colors=ColorGradient({0: rgba(0, 0, 0, 255),
135 | 10: rgba(255, 255, 255, 255)})
136 | Defaults to no colorization.
137 | preprocessor: Function to run on the TmsPyramid before slicing.
138 |
139 | Filenames are in the format ``{tms_z}-{tms_x}-{tms_y}-{image_hash}.png``.
140 |
141 | If a tile duplicates another tile already known to this process, a symlink
142 | is created instead of rendering the same tile to PNG again.
143 | """
144 | if renderer is None:
145 | renderer = PngRenderer()
146 | storage = SimpleFileStorage(outputdir=outputdir,
147 | renderer=renderer)
148 | pyramid = TmsPyramid(inputfile=inputfile,
149 | storage=storage,
150 | min_resolution=None,
151 | max_resolution=None)
152 | if preprocessor is None:
153 | preprocessor = colorize
154 | pyramid = preprocessor(**locals())
155 | pyramid.slice(fill_borders=fill_borders)
156 |
157 |
158 | def warp_mbtiles(inputfile, outputfile, metadata, colors=None, band=None,
159 | spatial_ref=None, resampling=None,
160 | min_resolution=None, max_resolution=None, fill_borders=None,
161 | zoom_offset=None, renderer=None, pngdata=None):
162 | """
163 | Warps a GDAL-readable inputfile into a pyramid of PNG tiles.
164 |
165 | inputfile: Filename
166 | outputfile: The output .mbtiles file.
167 |
168 | colors: Color palette applied to single band files.
169 | colors=ColorGradient({0: rgba(0, 0, 0, 255),
170 | 10: rgba(255, 255, 255, 255)})
171 | Defaults to no colorization.
172 | band: Select band to palettize and expand to RGBA. Defaults to 1.
173 | spatial_ref: Destination gdal.SpatialReference. Defaults to EPSG:3857,
174 | Web Mercator
175 | resampling: Resampling algorithm. Defaults to GDAL's default,
176 | nearest neighbour as of GDAL 1.9.1.
177 |
178 | min_resolution: Minimum resolution to downsample tiles.
179 | max_resolution: Maximum resolution to upsample tiles.
180 | fill_borders: Fill borders of image with empty tiles.
181 | zoom_offset: Offset zoom level to fit unprojected images to square maps.
182 |
183 | If `min_resolution` is None, don't downsample.
184 | If `max_resolution` is None, don't upsample.
185 | """
186 | if colors and band is None:
187 | band = 1
188 |
189 | if pngdata is None:
190 | pngdata = dict()
191 |
192 | with NamedTemporaryFile(suffix='.tif') as tempfile:
193 | dataset = Dataset(inputfile)
194 | validate_resolutions(resolution=dataset.GetNativeResolution(),
195 | min_resolution=min_resolution,
196 | max_resolution=max_resolution,
197 | strict=False)
198 | warped = preprocess(inputfile=inputfile, outputfile=tempfile.name,
199 | band=band, spatial_ref=spatial_ref,
200 | resampling=resampling, compress='LZW')
201 | preprocessor = partial(resample_after_warp,
202 | whole_world=dataset.IsWholeWorld())
203 | return image_mbtiles(inputfile=warped, outputfile=outputfile,
204 | metadata=metadata,
205 | min_resolution=min_resolution,
206 | max_resolution=max_resolution,
207 | colors=colors, renderer=renderer,
208 | preprocessor=preprocessor,
209 | fill_borders=fill_borders,
210 | zoom_offset=zoom_offset,
211 | pngdata=pngdata)
212 |
213 |
214 | def warp_pyramid(inputfile, outputdir, colors=None, band=None,
215 | spatial_ref=None, resampling=None,
216 | min_resolution=None, max_resolution=None, fill_borders=None,
217 | renderer=None):
218 | """
219 | Warps a GDAL-readable inputfile into a pyramid of PNG tiles.
220 |
221 | inputfile: Filename
222 | outputdir: The output directory for the PNG tiles.
223 |
224 | colors: Color palette applied to single band files.
225 | colors=ColorGradient({0: rgba(0, 0, 0, 255),
226 | 10: rgba(255, 255, 255, 255)})
227 | Defaults to no colorization.
228 | band: Select band to palettize and expand to RGBA. Defaults to 1.
229 | spatial_ref: Destination gdal.SpatialReference. Defaults to EPSG:3857,
230 | Web Mercator
231 | resampling: Resampling algorithm. Defaults to GDAL's default,
232 | nearest neighbour as of GDAL 1.9.1.
233 |
234 | min_resolution: Minimum resolution to downsample tiles.
235 | max_resolution: Maximum resolution to upsample tiles.
236 | fill_borders: Fill borders of image with empty tiles.
237 |
238 | Filenames are in the format ``{tms_z}/{tms_x}/{tms_y}.png``.
239 |
240 | If a tile duplicates another tile already known to this process, a symlink
241 | may be created instead of rendering the same tile to PNG again.
242 |
243 | If `min_resolution` is None, don't downsample.
244 | If `max_resolution` is None, don't upsample.
245 | """
246 | if colors and band is None:
247 | band = 1
248 |
249 | with NamedTemporaryFile(suffix='.tif') as tempfile:
250 | dataset = Dataset(inputfile)
251 | validate_resolutions(resolution=dataset.GetNativeResolution(),
252 | min_resolution=min_resolution,
253 | max_resolution=max_resolution,
254 | strict=False)
255 | warped = preprocess(inputfile=inputfile, outputfile=tempfile.name,
256 | band=band, spatial_ref=spatial_ref,
257 | resampling=resampling, compress='LZW')
258 | preprocessor = partial(resample_after_warp,
259 | whole_world=dataset.IsWholeWorld())
260 | return image_pyramid(inputfile=warped, outputdir=outputdir,
261 | min_resolution=min_resolution,
262 | max_resolution=max_resolution,
263 | colors=colors, renderer=renderer,
264 | preprocessor=preprocessor,
265 | fill_borders=fill_borders)
266 |
267 |
268 | def warp_slice(inputfile, outputdir, fill_borders=None, colors=None, band=None,
269 | spatial_ref=None, resampling=None,
270 | renderer=None):
271 | """
272 | Warps a GDAL-readable inputfile into a directory of PNG tiles.
273 |
274 | inputfile: Filename
275 | outputdir: The output directory for the PNG tiles.
276 |
277 | fill_borders: Fill borders of image with empty tiles.
278 | colors: Color palette applied to single band files.
279 | colors=ColorGradient({0: rgba(0, 0, 0, 255),
280 | 10: rgba(255, 255, 255, 255)})
281 | Defaults to no colorization.
282 | band: Select band to palettize and expand to RGBA. Defaults to 1.
283 | spatial_ref: Destination gdal.SpatialReference. Defaults to EPSG:3857,
284 | Web Mercator
285 | resampling: Resampling algorithm. Defaults to GDAL's default,
286 | nearest neighbour as of GDAL 1.9.1.
287 |
288 | min_resolution: Minimum resolution to downsample tiles.
289 | max_resolution: Maximum resolution to upsample tiles.
290 |
291 | Filenames are in the format ``{tms_z}-{tms_x}-{tms_y}-{image_hash}.png``.
292 |
293 | If a tile duplicates another tile already known to this process, a symlink
294 | may be created instead of rendering the same tile to PNG again.
295 | """
296 | if colors and band is None:
297 | band = 1
298 |
299 | with NamedTemporaryFile(suffix='.tif') as tempfile:
300 | dataset = Dataset(inputfile)
301 | warped = preprocess(inputfile=inputfile, outputfile=tempfile.name,
302 | band=band, spatial_ref=spatial_ref,
303 | resampling=resampling, compress='LZW')
304 | preprocessor = partial(resample_after_warp,
305 | whole_world=dataset.IsWholeWorld())
306 | return image_slice(inputfile=warped, outputdir=outputdir,
307 | colors=colors, renderer=renderer,
308 | preprocessor=preprocessor,
309 | fill_borders=fill_borders)
310 |
311 |
312 | # Preprocessors
313 |
314 | def resample_after_warp(pyramid, colors, whole_world, **kwargs):
315 | resolution = pyramid.dataset.GetNativeResolution()
316 | if whole_world:
317 | # We must resample the image to fit whole tiles, even if this makes the
318 | # extents of the image go PAST the full world.
319 | #
320 | # This is because GDAL sometimes reprojects from a whole world image
321 | # into a partial world image, due to rounding errors.
322 | pyramid.dataset.resample_to_world()
323 | else:
324 | pyramid.dataset.resample(resolution=resolution)
325 | colorize(pyramid=pyramid, colors=colors)
326 | pyramid.dataset.align_to_grid(resolution=resolution)
327 | return pyramid
328 |
329 |
330 | def colorize(pyramid, colors, **kwargs):
331 | if colors is not None:
332 | pyramid.colorize(colors)
333 | return pyramid
334 |
--------------------------------------------------------------------------------
/gdal2mbtiles/main.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Quickstart
4 | # ----------
5 | #
6 | # To turn any GDAL-readable file into an MBTiles file, run:
7 | # $ gdal2mbtiles filename.tiff
8 | # This creates a filename.mbtiles that can be served from a TMS service like
9 | # Mapbox.
10 | #
11 | # You can explicitly specify an output filename:
12 | # $ gdal2mbtiles input.tiff output.mbtiles
13 | #
14 | # You can also pipe in any GDAL-readable file:
15 | # $ cat input.tiff | gdal2mbtiles > output.mbtiles
16 | #
17 | # Licensed to Ecometrica under one or more contributor license
18 | # agreements. See the NOTICE file distributed with this work
19 | # for additional information regarding copyright ownership.
20 | # Ecometrica licenses this file to you under the Apache
21 | # License, Version 2.0 (the "License"); you may not use this
22 | # file except in compliance with the License. You may obtain a
23 | # copy of the License at
24 | #
25 | # http://www.apache.org/licenses/LICENSE-2.0
26 | #
27 | # Unless required by applicable law or agreed to in writing,
28 | # software distributed under the License is distributed on an
29 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
30 | # KIND, either express or implied. See the License for the
31 | # specific language governing permissions and limitations
32 | # under the License.
33 |
34 | from __future__ import (absolute_import, division, print_function,
35 | unicode_literals)
36 |
37 |
38 | import argparse
39 | from contextlib import contextmanager
40 | import logging
41 | import os
42 | from shutil import copyfileobj
43 | import sys
44 | from tempfile import NamedTemporaryFile
45 |
46 | if __name__ == '__main__' and __package__ is None:
47 | # HACK: Force this to work when called directly
48 | import gdal2mbtiles
49 | __package__ = gdal2mbtiles.__name__
50 |
51 | from .gdal import RESAMPLING_METHODS, SpatialReference
52 | from .gd_types import rgba
53 | from .mbtiles import Metadata
54 |
55 |
56 | COLORING_METHODS = {
57 | 'exact': 'ColorExact',
58 | 'gradient': 'ColorGradient',
59 | 'palette': 'ColorPalette',
60 | }
61 |
62 |
63 | def coloring_arg(s):
64 | """Validates --coloring"""
65 | from gdal2mbtiles import vips
66 | if s is None:
67 | return None
68 | return getattr(vips, COLORING_METHODS[s])
69 |
70 |
71 | def color_arg(s):
72 | """Validates --color"""
73 | try:
74 | band_value, html_color = s.split(':', 1)
75 | except ValueError:
76 | raise argparse.ArgumentTypeError(
77 | "'{0}' must be in format: BAND-VALUE:HTML-COLOR".format(s)
78 | )
79 |
80 | try:
81 | band_value = float(band_value)
82 | except ValueError:
83 | raise argparse.ArgumentTypeError(
84 | "'{0}' is not a valid number".format(band_value)
85 | )
86 |
87 | try:
88 | color = rgba.webcolor(html_color)
89 | except ValueError:
90 | raise argparse.ArgumentTypeError(
91 | "'{0}' is not a valid HTML color".format(html_color)
92 | )
93 |
94 | return band_value, color
95 |
96 |
97 | def colorize_band_arg(s):
98 | """Validates --colorize-band"""
99 | try:
100 | result = int(s)
101 | except ValueError:
102 | raise argparse.ArgumentTypeError("invalid int value: '{0}'".format(s))
103 | if result <= 0:
104 | raise argparse.ArgumentTypeError(
105 | "'{0}' must be 1 or greater".format(s)
106 | )
107 | return result
108 |
109 |
110 | def png8_arg(s):
111 | """Validates --png8"""
112 | if s is None:
113 | result = s
114 | else:
115 | try:
116 | result = int(s)
117 | except ValueError:
118 | raise argparse.ArgumentTypeError("invalid int value: '{0}'".format(s))
119 | if not 2 <= result <= 256:
120 | raise ValueError(
121 | 'png8 must be between 2 and 256: {0!r}'.format(result)
122 | )
123 | return result
124 |
125 |
126 | def parse_args(args):
127 | """Parses command-line `args`"""
128 |
129 | LatestMetadata = Metadata.latest()
130 |
131 | parser = argparse.ArgumentParser(
132 | description='Converts a GDAL-readable into an MBTiles file'
133 | )
134 | parser.add_argument('-v', '--verbose', action='count',
135 | help='explain what is being done')
136 |
137 | group = parser.add_argument_group(title='Positional arguments')
138 | group.add_argument('INPUT', type=argparse.FileType('rb'), nargs='?',
139 | default=sys.stdin,
140 | help='GDAL-readable file.')
141 | group.add_argument('OUTPUT', type=argparse.FileType('wb'), nargs='?',
142 | help='Output filename. Defaults to INPUT.mbtiles')
143 |
144 | group = parser.add_argument_group(title='MBTiles metadata arguments')
145 | group.add_argument('--name', default=None,
146 | help=('Human-readable name of the tileset. '
147 | 'Defaults to INPUT'))
148 | group.add_argument('--description', default="",
149 | help='Description of the layer. Defaults to ""')
150 | group.add_argument('--layer-type',
151 | default=LatestMetadata.TYPES.OVERLAY,
152 | choices=LatestMetadata.TYPES,
153 | help='Type of layer. Defaults to "overlay"')
154 | group.add_argument('--version', default='1.0.0',
155 | help='Version of the tileset. Defaults to "1.0.0"')
156 | group.add_argument('--format',
157 | default=LatestMetadata.FORMATS.PNG,
158 | choices=LatestMetadata.FORMATS,
159 | help='Tile image format. Defaults to "png"')
160 |
161 | group = parser.add_argument_group(title='GDAL warp arguments')
162 | group.add_argument('--spatial-reference', type=int, default=3857,
163 | help=('Destination EPSG spatial reference. '
164 | 'Defaults to 3857'))
165 | group.add_argument('--resampling',
166 | default='near',
167 | choices=list(RESAMPLING_METHODS.values()),
168 | help=('Resampling algorithm for warping. '
169 | 'Defaults to "near" (nearest-neighbour)'))
170 |
171 | group = parser.add_argument_group(title='Rendering arguments')
172 | group.add_argument('--min-resolution', type=int, default=None,
173 | help=('Minimum resolution/zoom level to render and slice. '
174 | 'Defaults to None (do not downsample)'))
175 | group.add_argument('--max-resolution', type=int, default=None,
176 | help=('Maximum resolution/zoom level to render and slice. '
177 | 'Defaults to None (do not upsample)'))
178 | group.add_argument('--fill-borders',
179 | action='store_const', const=True, default=True,
180 | help=('Fill image to whole world with empty tiles. '
181 | 'Default.'))
182 | group.add_argument('--no-fill-borders', dest='fill_borders',
183 | action='store_const', const=False,
184 | help='Do not add borders to fill image.')
185 | group.add_argument('--zoom-offset', type=int, default=0,
186 | metavar='N',
187 | help=('Offset zoom level by N to fit unprojected '
188 | 'images to square maps. Defaults to 0.'))
189 |
190 | group = parser.add_argument_group(title='Coloring arguments')
191 | group.add_argument('--coloring', default=None,
192 | choices=COLORING_METHODS,
193 | help='Coloring algorithm.')
194 | group.add_argument('--color', dest='colors', action='append',
195 | type=color_arg, metavar='BAND-VALUE:HTML-COLOR',
196 | help=('Examples: --color="0:#ff00ff" --color=255:red'))
197 | group.add_argument('--colorize-band', metavar='COLORIZE-BAND',
198 | type=colorize_band_arg, default=None,
199 | help='Raster band to colorize. Defaults to 1')
200 | group.add_argument('--png8', default=None,
201 | type=png8_arg,
202 | help=('Quantizes 32-bit RGBA to 8-bit RGBA paletted '
203 | 'PNGs. If an integer, specifies number of '
204 | 'colors in palette between 2 and 256. '
205 | 'Default to False.'))
206 |
207 | args = parser.parse_args(args=args)
208 |
209 | # Guess at the OUTPUT based on the INPUT
210 | if args.OUTPUT is None:
211 | if args.INPUT == sys.stdin:
212 | args.OUTPUT = sys.stdout
213 | else:
214 | # Set default output name based on input name
215 | args.OUTPUT = open(
216 | os.path.splitext(args.INPUT.name)[0] + '.mbtiles',
217 | mode='wb'
218 | )
219 |
220 | if args.name is None:
221 | args.name = os.path.basename(args.INPUT.name)
222 |
223 | # Make sure that --color and --coloring match up
224 | if args.coloring is None and (args.colors or
225 | args.colorize_band is not None):
226 | parser.error('must provide --coloring')
227 | elif args.coloring is not None and not args.colors:
228 | parser.error('must provide at least one --color')
229 |
230 | # Transform choices into ColorBase classes
231 | args.coloring = coloring_arg(args.coloring)
232 |
233 | return args
234 |
235 |
236 | @contextmanager
237 | def input_output(inputfile, outputfile):
238 | tempfiles = []
239 |
240 | infile = inputfile
241 | if inputfile == sys.stdin:
242 | infile = NamedTemporaryFile()
243 | copyfileobj(inputfile, infile)
244 | infile.seek(0)
245 | tempfiles.append(infile)
246 |
247 | outfile = outputfile
248 | if outputfile == sys.stdout:
249 | outfile = NamedTemporaryFile()
250 | tempfiles.append(outfile)
251 |
252 | try:
253 | yield infile, outfile
254 | if outputfile == sys.stdout:
255 | copyfileobj(open(outfile.name, 'rb'), outputfile)
256 | finally:
257 | for f in tempfiles:
258 | f.close()
259 |
260 |
261 | def main(args=None, use_logging=True):
262 | if args is None:
263 | args = sys.argv[1:]
264 | args = parse_args(args=args)
265 |
266 | if use_logging:
267 | configure_logging(args)
268 |
269 | # HACK: Import here, so that VIPS doesn't parse sys.argv!!!
270 | # In vimagemodule.cxx, SWIG_init actually does argument parsing
271 | from gdal2mbtiles.helpers import warp_mbtiles
272 |
273 | with input_output(inputfile=args.INPUT,
274 | outputfile=args.OUTPUT) as (inputfile, outputfile):
275 | # MBTiles
276 | metadata = dict(
277 | description=args.description,
278 | format=args.format,
279 | name=args.name,
280 | type=args.layer_type,
281 | version=args.version,
282 | )
283 |
284 | # GDAL
285 | spatial_ref = SpatialReference.FromEPSG(args.spatial_reference)
286 |
287 | # Coloring
288 | if not args.coloring:
289 | colors = band = None
290 | else:
291 | colors = args.coloring(args.colors)
292 | band = args.colorize_band
293 |
294 | # PNG rendering
295 | pngdata = {'png8': args.png8}
296 |
297 | warp_mbtiles(inputfile=inputfile.name, outputfile=outputfile.name,
298 | # MBTiles
299 | metadata=metadata,
300 | # GDAL
301 | spatial_ref=spatial_ref, resampling=args.resampling,
302 | # Rendering
303 | min_resolution=args.min_resolution,
304 | max_resolution=args.max_resolution,
305 | fill_borders=args.fill_borders,
306 | zoom_offset=args.zoom_offset,
307 | pngdata=pngdata,
308 | # Coloring
309 | colors=colors, band=band)
310 | return 0
311 |
312 |
313 | def configure_logging(args):
314 | if not args.verbose:
315 | return
316 |
317 | if args.verbose == 1:
318 | level = logging.INFO
319 | fmt = '%(message)s'
320 | else:
321 | level = logging.DEBUG
322 | fmt = '%(asctime)s %(module)s: %(message)s'
323 |
324 | logging.basicConfig(level=level, format=fmt,
325 | datefmt='%Y-%m-%d %H:%M:%S')
326 |
327 |
328 | if __name__ == '__main__':
329 | retcode = main(args=sys.argv[1:])
330 | sys.exit(retcode)
331 |
--------------------------------------------------------------------------------
/gdal2mbtiles/mbtiles.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Licensed to Ecometrica under one or more contributor license
4 | # agreements. See the NOTICE file distributed with this work
5 | # for additional information regarding copyright ownership.
6 | # Ecometrica licenses this file to you under the Apache
7 | # License, Version 2.0 (the "License"); you may not use this
8 | # file except in compliance with the License. You may obtain a
9 | # copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 |
20 | from __future__ import (absolute_import, division, print_function,
21 | unicode_literals)
22 |
23 | from distutils.version import LooseVersion
24 | import errno
25 | import os
26 | import sqlite3
27 | from struct import pack, unpack
28 |
29 | try:
30 | # py3.3+
31 | from collections.abc import MutableMapping
32 | except ImportError:
33 | # py2.7
34 | from collections import MutableMapping
35 |
36 | try:
37 | basestring
38 | except NameError:
39 | basestring = str
40 |
41 | from .gd_types import enum
42 | from .utils import rmfile
43 |
44 |
45 | class MBTilesError(RuntimeError):
46 | pass
47 |
48 |
49 | class InvalidFileError(MBTilesError):
50 | pass
51 |
52 |
53 | class UnknownVersionError(MBTilesError):
54 | pass
55 |
56 |
57 | class MetadataError(MBTilesError):
58 | pass
59 |
60 |
61 | class MetadataKeyError(MetadataError, KeyError):
62 | pass
63 |
64 |
65 | class MetadataValueError(MetadataError, ValueError):
66 | pass
67 |
68 |
69 | class Metadata(MutableMapping):
70 | """
71 | Key-value metadata table expressed as a dictionary
72 | """
73 | VERSION = None
74 |
75 | MANDATORY = None
76 | OPTIONAL = None
77 |
78 | _all = None
79 |
80 | def __init__(self, mbtiles):
81 | """Links this Metadata wrapper to the MBTiles wrapper."""
82 | self.mbtiles = mbtiles
83 |
84 | def __delitem__(self, y):
85 | """Removes key `y` from the database."""
86 | if y in self.MANDATORY:
87 | raise MetadataKeyError(
88 | "Cannot delete mandatory key: {0!r}".format(y)
89 | )
90 | return self._delitem(y)
91 |
92 | def _delitem(self, y):
93 | """Removes key `y` from the database."""
94 | with self.mbtiles._conn:
95 | cursor = self.mbtiles._conn.execute(
96 | """
97 | DELETE FROM metadata
98 | WHERE name = :name
99 | """,
100 | {'name': y}
101 | )
102 | if not cursor.rowcount:
103 | raise MetadataKeyError(repr(y))
104 |
105 | def __getitem__(self, y):
106 | """Gets value for key `y` from the database."""
107 | cursor = self.mbtiles._conn.execute(
108 | """
109 | SELECT value FROM metadata
110 | WHERE name = :name
111 | """,
112 | {'name': y}
113 | )
114 | value = cursor.fetchone()
115 | if value is None:
116 | raise MetadataKeyError(repr(y))
117 | return value[0]
118 |
119 | def __setitem__(self, i, y):
120 | cleaner = getattr(self, '_clean_' + i, None)
121 | if cleaner is not None:
122 | y = cleaner(y)
123 |
124 | return self._setitem(i, y)
125 |
126 | def _setitem(self, i, y):
127 | """Sets value `y` for key `i` in the database."""
128 | with self.mbtiles._conn:
129 | self.mbtiles._conn.execute(
130 | """
131 | INSERT OR REPLACE INTO metadata (name, value)
132 | VALUES (:name, :value)
133 | """,
134 | {'name': i, 'value': y}
135 | )
136 |
137 | def __iter__(self):
138 | for k in self.keys():
139 | yield k
140 |
141 | def __len__(self):
142 | return len(self.keys())
143 |
144 | def keys(self):
145 | """Returns a list of keys from the database."""
146 | try:
147 | cursor = self.mbtiles._conn.execute(
148 | """
149 | SELECT name FROM metadata
150 | """,
151 | )
152 | except sqlite3.OperationalError:
153 | raise InvalidFileError("Invalid MBTiles file.")
154 | result = cursor.fetchall()
155 | if not result:
156 | return result
157 | return list(zip(*result))[0]
158 |
159 | def _setup(self, metadata):
160 | missing = set(self.MANDATORY) - set(metadata.keys())
161 | if missing:
162 | raise MetadataKeyError(
163 | "Required keys missing from metadata: {0}".format(
164 | ', '.join(missing)
165 | )
166 | )
167 | self.update(metadata)
168 |
169 | @classmethod
170 | def _detect(cls, keys):
171 | version = None
172 | for ver, M in sorted(cls.all().items()):
173 | if set(keys).issuperset(set(M.MANDATORY)):
174 | version = ver
175 | if version is None:
176 | raise InvalidFileError("Invalid MBTiles file.")
177 | return version
178 |
179 | @classmethod
180 | def detect(cls, mbtiles):
181 | """Returns the Metadata version detected from `mbtiles`."""
182 | return cls._detect(keys=list(cls(mbtiles=mbtiles).keys()))
183 |
184 | @classmethod
185 | def all(cls):
186 | """Returns all Metadata classes."""
187 | if cls._all is None:
188 | def subclasses(base):
189 | for m in base.__subclasses__():
190 | yield m
191 | for n in subclasses(base=m):
192 | yield n
193 |
194 | cls._all = dict((m.VERSION, m)
195 | for m in subclasses(base=Metadata))
196 | return cls._all
197 |
198 | @classmethod
199 | def latest(cls):
200 | """Returns the latest Metadata class."""
201 | return sorted(list(cls.all().items()),
202 | key=(lambda k: LooseVersion(k[0])),
203 | reverse=True)[0][1]
204 |
205 |
206 | class Metadata_1_0(Metadata):
207 | """
208 | Mandatory metadata:
209 | name: The plain-english name of the tileset.
210 | type: mbtiles.TYPES.OVERLAY or mbtiles.TYPES.BASELAYER
211 | version: The version of the tileset, as a plain number.
212 | description: A description of the layer as plain text.
213 | """
214 |
215 | VERSION = '1.0'
216 |
217 | MANDATORY = ('name', 'type', 'version', 'description')
218 | OPTIONAL = ()
219 |
220 | TYPES = enum(OVERLAY='overlay',
221 | BASELAYER='baselayer')
222 |
223 | def _clean_type(self, value):
224 | if value not in self.TYPES:
225 | raise MetadataValueError(
226 | "type {value!r} must be one of: {types}".format(
227 | value=value,
228 | types=', '.join(sorted(self.TYPES))
229 | )
230 | )
231 | return value
232 |
233 |
234 | class Metadata_1_1(Metadata_1_0):
235 | """
236 | Mandatory metadata:
237 | name: The plain-english name of the tileset.
238 | type: mbtiles.TYPES.OVERLAY or mbtiles.TYPES.BASELAYER
239 | version: The version of the tileset, as a plain number.
240 | description: A description of the layer as plain text.
241 | format: The image file format of the tile data:
242 | mbtiles.FORMATS.PNG or mbtiles.FORMATS.JPG
243 |
244 | Optional metadata:
245 | bounds: The maximum extent of the rendered map area. Bounds must define
246 | an area covered by all zoom levels. The bounds are represented
247 | in WGS:84 latitude and longitude values, in the OpenLayers
248 | Bounds format (left, bottom, right, top).
249 | Example of the full earth: '-180.0,-85,180,85'.
250 | """
251 | VERSION = '1.1'
252 |
253 | MANDATORY = Metadata_1_0.MANDATORY + ('format',)
254 | OPTIONAL = Metadata_1_0.OPTIONAL + ('bounds',)
255 |
256 | FORMATS = enum(PNG='png',
257 | JPG='jpg')
258 |
259 | def _clean_format(self, value):
260 | if value not in self.FORMATS:
261 | raise MetadataValueError(
262 | "format {value!r} must be one of: {formats}".format(
263 | value=value,
264 | formats=', '.join(sorted(self.FORMATS))
265 | )
266 | )
267 | return value
268 |
269 | def _clean_bounds(self, value, places=5):
270 | if isinstance(value, basestring):
271 | left, bottom, right, top = [float(b) for b in value.split(',')]
272 | else:
273 | left, bottom, right, top = value
274 |
275 | # Preventing ridiculous values due to floating point
276 | left = round(left, places)
277 | bottom = round(bottom, places)
278 | right = round(right, places)
279 | top = round(top, places)
280 |
281 | try:
282 | if left >= right or bottom >= top or \
283 | left < -180.0 or right > 180.0 or \
284 | bottom < -90.0 or top > 90.0:
285 | raise ValueError()
286 | except ValueError:
287 | raise MetadataValueError("Invalid bounds: {0!r}".format(value))
288 |
289 | return '{left!r},{bottom!r},{right!r},{top!r}'.format(
290 | left=left, bottom=bottom, right=right, top=top
291 | )
292 |
293 |
294 | class Metadata_1_2(Metadata_1_1):
295 | """
296 | Mandatory metadata:
297 | name: The plain-english name of the tileset.
298 | type: mbtiles.TYPES.OVERLAY or mbtiles.TYPES.BASELAYER
299 | version: The version of the tileset, as a plain number.
300 | description: A description of the layer as plain text.
301 | format: The image file format of the tile data:
302 | mbtiles.FORMATS.PNG or mbtiles.FORMATS.JPG
303 |
304 | Optional metadata:
305 | bounds: The maximum extent of the rendered map area. Bounds must define
306 | an area covered by all zoom levels. The bounds are represented
307 | in WGS:84 latitude and longitude values, in the OpenLayers
308 | Bounds format (left, bottom, right, top).
309 | Example of the full earth: '-180.0,-85,180,85'.
310 | attribution: An attribution string, which explains in English (and
311 | HTML) the sources of data and/or style for the map.
312 | """
313 |
314 | VERSION = '1.2'
315 | OPTIONAL = Metadata_1_1.OPTIONAL + ('attribution',)
316 |
317 |
318 | class MBTiles(object):
319 | """Represents an MBTiles file."""
320 |
321 | Metadata = Metadata
322 |
323 | # Pragmas for the SQLite connection
324 | _connection_options = {
325 | 'auto_vacuum': 'NONE',
326 | 'encoding': '"UTF-8"',
327 | 'foreign_keys': '0',
328 | 'journal_mode': 'MEMORY',
329 | 'locking_mode': 'EXCLUSIVE',
330 | 'synchronous': 'OFF',
331 | }
332 |
333 | def __init__(self, filename, version=None, options=None,
334 | create=False):
335 | """Opens an MBTiles file named `filename`"""
336 | self.filename = filename
337 | self._conn = None
338 | self._metadata = None
339 | self._version = version
340 |
341 | self.open(options=options, create=create)
342 |
343 | def __del__(self):
344 | self.close()
345 |
346 | def __enter__(self):
347 | return self
348 |
349 | def __exit__(self, type, value, traceback):
350 | self.close()
351 |
352 | def close(self, remove_journal=True):
353 | """Closes the file."""
354 | if self._conn is not None:
355 | if remove_journal:
356 | self._conn.execute('PRAGMA journal_mode = DELETE')
357 | self._conn.close()
358 | self._conn = None
359 |
360 | @property
361 | def closed(self):
362 | """Returns True if the file is closed."""
363 | return not bool(self._conn)
364 |
365 | def open(self, options=None, create=False):
366 | """Re-opens the file."""
367 | result = self._open(options=options, create=create)
368 | self.metadata
369 | return result
370 |
371 | def _open(self, options=None, create=False):
372 | self.close()
373 |
374 | if self.filename != ':memory:':
375 | mode = 'wb' if create else 'rb'
376 | with open(self.filename, mode):
377 | # Raises exceptions if the file can't be opened
378 | pass
379 |
380 | try:
381 | self._conn = sqlite3.connect(self.filename)
382 | except sqlite3.OperationalError:
383 | raise InvalidFileError("Invalid MBTiles file.")
384 | self._conn.text_factory = lambda x: x.decode('utf-8', 'ignore')
385 |
386 | # Pragmas derived from options
387 | if options is None:
388 | options = self._connection_options
389 | try:
390 | self._conn.executescript(
391 | '\n'.join('PRAGMA {0} = {1};'.format(k, v)
392 | for k, v in options.items())
393 | )
394 | except sqlite3.DatabaseError:
395 | self.close(remove_journal=False)
396 | raise InvalidFileError("Invalid MBTiles file")
397 | except Exception:
398 | self.close(remove_journal=False)
399 | raise
400 | return self._conn
401 |
402 | @classmethod
403 | def create(cls, filename, metadata, version=None):
404 | """Create a new MBTiles file. See `Metadata`"""
405 | if version is None:
406 | version = cls.Metadata._detect(keys=list(metadata.keys()))
407 | mbtiles = cls._create(filename=filename, version=version)
408 | mbtiles.metadata._setup(metadata)
409 | return mbtiles
410 |
411 | @classmethod
412 | def _create(cls, filename, version):
413 | """
414 | Creates a new MBTiles file named `filename`.
415 |
416 | If `filename` already exists, it gets deleted and recreated.
417 | """
418 | # The MBTiles spec defines a tiles table as:
419 | # CREATE TABLE tiles (
420 | # zoom_level INTEGER,
421 | # tile_column INTEGER,
422 | # tile_row INTEGER,
423 | # tile_data BLOB
424 | # );
425 | #
426 | # However, we wish to normalize the tile_data, so we store each
427 | # in the images table.
428 | rmfile(filename, ignore_missing=True)
429 | try:
430 | os.remove(filename)
431 | except OSError as e:
432 | if e.errno != errno.ENOENT: # Removing a non-existent file is OK.
433 | raise
434 |
435 | mbtiles = cls(filename=filename, version=version, create=True)
436 |
437 | conn = mbtiles._conn
438 | with conn:
439 | conn.execute(
440 | """
441 | CREATE TABLE images (
442 | tile_id INTEGER PRIMARY KEY,
443 | tile_data BLOB NOT NULL
444 | )
445 | """
446 | )
447 |
448 | # Then we reference the Z/X/Y coordinates in the map table.
449 | conn.execute(
450 | """
451 | CREATE TABLE map (
452 | zoom_level INTEGER NOT NULL,
453 | tile_column INTEGER NOT NULL,
454 | tile_row INTEGER NOT NULL,
455 | tile_id INTEGER NOT NULL
456 | REFERENCES images (tile_id)
457 | ON DELETE CASCADE ON UPDATE CASCADE,
458 | PRIMARY KEY (zoom_level, tile_column, tile_row)
459 | )
460 | """
461 | )
462 |
463 | # Finally, we emulate the tiles table using a view.
464 | conn.execute(
465 | """
466 | CREATE VIEW tiles AS
467 | SELECT zoom_level, tile_column, tile_row, tile_data
468 | FROM map, images
469 | WHERE map.tile_id = images.tile_id
470 | """
471 | )
472 |
473 | # We also need a table to store metadata.
474 | conn.execute(
475 | """
476 | CREATE TABLE metadata (
477 | name TEXT PRIMARY KEY,
478 | value TEXT NOT NULL
479 | )
480 | """
481 | )
482 |
483 | return mbtiles
484 |
485 | @property
486 | def version(self):
487 | if self._version is None:
488 | self._version = self.Metadata.detect(mbtiles=self)
489 | return self._version
490 |
491 | @property
492 | def metadata(self):
493 | """Returns a dictionary-like Metadata object."""
494 | if self._metadata is None:
495 | try:
496 | M = self.Metadata.all()[self.version]
497 | except KeyError:
498 | raise UnknownVersionError(
499 | 'Unknown version {0}'.format(self._version)
500 | )
501 | self._metadata = M(mbtiles=self)
502 | return self._metadata
503 |
504 | def insert(self, x, y, z, hashed, data=None):
505 | """
506 | Inserts a tile in the database at coordinates `x`, `y`, `z`.
507 |
508 | x, y, z: TMS coordinates for the tile.
509 | hashed: Integer hash of the raw image data, not compressed or encoded.
510 | data: Compressed and encoded image buffer.
511 | """
512 | # tile_id must be a 64-bit signed integer, but hashing functions
513 | # produce unsigned integers.
514 | hashed = unpack(b'q', pack(b'Q', hashed & 0xffffffffffffffff))[0]
515 | with self._conn:
516 | if data is not None:
517 | # Insert tile data into images
518 | self._conn.execute(
519 | """
520 | INSERT OR REPLACE INTO images (tile_id, tile_data)
521 | VALUES (:hashed, :data)
522 | """,
523 | {'hashed': hashed, 'data': data}
524 | )
525 |
526 | # Always associate map with image
527 | self._conn.execute(
528 | """
529 | INSERT OR REPLACE
530 | INTO map (zoom_level, tile_column, tile_row, tile_id)
531 | VALUES (:z, :x, :y, :hashed)
532 | """,
533 | {'x': x, 'y': y, 'z': z, 'hashed': hashed}
534 | )
535 |
536 | def get(self, x, y, z):
537 | """
538 | Returns the compressed image data at coordinates `x`, `y`, `z`.
539 |
540 | x, y, z: TMS coordinates for the tile.
541 | """
542 | cursor = self._conn.execute(
543 | """
544 | SELECT tile_data FROM tiles
545 | WHERE zoom_level = :z AND
546 | tile_column = :x AND
547 | tile_row = :y
548 | """,
549 | {'x': x, 'y': y, 'z': z}
550 | )
551 | result = cursor.fetchone()
552 | if result is None:
553 | return None
554 | return result[0]
555 |
556 | def all(self):
557 | """
558 | Returns all of the compressed image data
559 | """
560 | cursor = self._conn.execute(
561 | """
562 | SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles
563 | ORDER BY zoom_level, tile_column, tile_row
564 | """
565 | )
566 | while True:
567 | rows = cursor.fetchmany()
568 | if not rows:
569 | return
570 | for z, x, y, data in rows:
571 | yield z, x, y, data
572 |
--------------------------------------------------------------------------------
/gdal2mbtiles/renderers.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Licensed to Ecometrica under one or more contributor license
4 | # agreements. See the NOTICE file distributed with this work
5 | # for additional information regarding copyright ownership.
6 | # Ecometrica licenses this file to you under the Apache
7 | # License, Version 2.0 (the "License"); you may not use this
8 | # file except in compliance with the License. You may obtain a
9 | # copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 |
20 | from __future__ import (absolute_import, division, print_function,
21 | unicode_literals)
22 |
23 | import os
24 | from subprocess import check_call
25 | from tempfile import gettempdir, NamedTemporaryFile
26 |
27 | from .utils import rmfile
28 |
29 |
30 | class Renderer(object):
31 | _suffix = ''
32 |
33 | def __init__(self, suffix=None, tempdir=None):
34 | if suffix is None:
35 | suffix = self.__class__._suffix
36 | self.suffix = suffix
37 |
38 | if tempdir is None:
39 | tempdir = gettempdir()
40 | self.tempdir = tempdir
41 |
42 | def __str__(self):
43 | return 'Renderer(suffix={suffix!r})'.format(**self.__dict__)
44 |
45 | def render(self, image):
46 | raise NotImplementedError()
47 |
48 |
49 | class JpegRenderer(Renderer):
50 | """
51 | Render a VIPS image as a JPEG to filename.
52 |
53 | Since JPEGs cannot contain transparent areas, the alpha channel is
54 | discarded.
55 |
56 | compression: JPEG compression level. Default 75.
57 | interlace: Filename of ICC profile. Default None.
58 | suffix: Suffix for filename. Default '.jpeg'.
59 | """
60 | _suffix = '.jpeg'
61 |
62 | def __init__(self, compression=None, profile=None, **kwargs):
63 | if compression is None:
64 | compression = 75
65 | _compression = int(compression)
66 | if not 0 <= _compression <= 100:
67 | raise ValueError(
68 | 'compression must be between 0 and 100: {0!r}'.format(
69 | compression
70 | )
71 | )
72 | self.compression = _compression
73 |
74 | if profile is None:
75 | profile = 'none'
76 | self.profile = profile
77 |
78 | super(JpegRenderer, self).__init__(**kwargs)
79 |
80 | @property
81 | def _vips_options(self):
82 | return {
83 | 'Q': self.compression,
84 | 'profile': self.profile
85 | }
86 |
87 | def render(self, image):
88 | """Returns the rendered VIPS `image`."""
89 | if image.bands > 3:
90 | # Strip out alpha channel, otherwise transparent pixels turn white.
91 | image = image.extract_band(0, n=3)
92 | with NamedTemporaryFile(suffix=self.suffix,
93 | dir=self.tempdir) as rendered:
94 | image.write_to_file(rendered.name, **self._vips_options)
95 | return rendered.read()
96 |
97 |
98 | class PngRenderer(Renderer):
99 | """
100 | Render a VIPS image as a PNG.
101 |
102 | compression: PNG compression level. Default 6.
103 | interlace: Use ADAM7 interlacing. Default False.
104 | png8: Quantizes 32-bit RGBA to 8-bit RGBA paletted PNGs. Default False.
105 | If an integer, specifies number of colors in palette.
106 | If True, defaults to 256 colors.
107 | optimize: Optimizes PNG using optipng. Default False. See `optipng -h`.
108 | suffix: Suffix for filename. Default '.png'.
109 |
110 | If optimize is not False, then compression is ignored and set to 0, to
111 | prevent double-compression. In general, VIPS compression is faster than
112 | optimizing with OptiPNG.
113 | """
114 | _suffix = '.png'
115 |
116 | PNGQUANT = 'pngquant'
117 | OPTIPNG = 'optipng'
118 |
119 | def __init__(self, compression=None, interlace=None, png8=None,
120 | optimize=None, **kwargs):
121 | if compression is None:
122 | compression = 6
123 | _compression = int(compression)
124 | if not 0 <= _compression <= 9:
125 | raise ValueError(
126 | 'compression must be between 0 and 9: {0!r}'.format(
127 | compression
128 | )
129 | )
130 | self.compression = _compression
131 |
132 | self.interlace = bool(interlace)
133 |
134 | _png8 = png8
135 | if _png8 is None:
136 | _png8 = False
137 | elif _png8 is True:
138 | _png8 = 256
139 | if _png8 is not False:
140 | _png8 = int(_png8)
141 | if not 2 <= _png8 <= 256:
142 | raise ValueError(
143 | 'png8 must be between 2 and 256: {0!r}'.format(png8)
144 | )
145 | self.png8 = _png8
146 |
147 | _optimize = optimize
148 | if _optimize is None:
149 | _optimize = False
150 | if _optimize is not False:
151 | _optimize = int(_optimize)
152 | if not 0 <= _optimize <= 7:
153 | raise ValueError(
154 | 'optimize must be between 0 and 7: {0!r}'.format(optimize)
155 | )
156 | if _optimize:
157 | self.compression = 1 # Reduce cost of double-compression
158 | self.optimize = _optimize
159 |
160 | super(PngRenderer, self).__init__(**kwargs)
161 |
162 | @property
163 | def _vips_options(self):
164 | return {
165 | 'compression': self.compression,
166 | 'interlace': self.interlace
167 | }
168 |
169 | def render(self, image):
170 | """Returns the rendered VIPS `image`."""
171 | with NamedTemporaryFile(suffix=self.suffix,
172 | dir=self.tempdir) as rendered:
173 | image.write_to_file(rendered.name, **self._vips_options)
174 | filename = rendered.name
175 |
176 | if self.png8 is not False:
177 | check_call([self.PNGQUANT, '--force', str(self.png8),
178 | filename])
179 | filename = os.path.splitext(filename)[0] + '-fs8.png'
180 |
181 | if self.optimize is not False:
182 | check_call([self.OPTIPNG, '-o{0:d}'.format(self.optimize),
183 | '-quiet', filename])
184 |
185 | with open(filename, 'rb') as result:
186 | if rendered.name != filename:
187 | rmfile(filename, ignore_missing=True)
188 | return result.read()
189 |
190 |
191 | class TouchRenderer(Renderer):
192 | """For testing only. Only creates files, doesn't actually render."""
193 | _suffix = ''
194 |
195 | def render(self, image):
196 | """Touches `filename` and returns its value."""
197 | return b''
198 |
--------------------------------------------------------------------------------
/gdal2mbtiles/storages.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Licensed to Ecometrica under one or more contributor license
4 | # agreements. See the NOTICE file distributed with this work
5 | # for additional information regarding copyright ownership.
6 | # Ecometrica licenses this file to you under the Apache
7 | # License, Version 2.0 (the "License"); you may not use this
8 | # file except in compliance with the License. You may obtain a
9 | # copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 |
20 | from __future__ import (absolute_import, division, print_function,
21 | unicode_literals)
22 |
23 | import sys
24 |
25 | from collections import defaultdict
26 | from functools import partial
27 | import os
28 |
29 | from .constants import TILE_SIDE
30 | from .gdal import SpatialReference
31 | from .mbtiles import MBTiles
32 | from .gd_types import rgba
33 | from .utils import intmd5, makedirs
34 | from .vips import VImageAdapter
35 |
36 |
37 | try:
38 | basestring
39 | except NameError:
40 | basestring = str
41 |
42 |
43 | class Storage(object):
44 | """Base class for storages."""
45 |
46 | def __init__(self, renderer, pool=None):
47 | """
48 | Initialize a storage.
49 |
50 | renderer: Used to render images into tiles.
51 | """
52 | self.renderer = renderer
53 |
54 | self.hasher = intmd5
55 |
56 | def __enter__(self):
57 | return self
58 |
59 | def __exit__(self, type, value, traceback):
60 | return
61 |
62 | def get_hash(self, image):
63 | """Returns the image content hash."""
64 | return self.hasher(image.write_to_memory())
65 |
66 | def filepath(self, x, y, z, hashed):
67 | """Returns the filepath."""
68 | raise NotImplementedError()
69 |
70 | def post_import(self, pyramid):
71 | """Runs after `pyramid` has finished importing into this storage."""
72 | pass
73 |
74 | def save(self, x, y, z, image):
75 | """Saves `image` at coordinates `x`, `y`, and `z`."""
76 | raise NotImplementedError()
77 |
78 | def save_border(self, x, y, z):
79 | """Saves a border image at coordinates `x`, `y`, and `z`."""
80 | self.save(x=x, y=y, z=z, image=self._border_image())
81 |
82 | @classmethod
83 | def _border_image(cls, width=TILE_SIDE, height=TILE_SIDE):
84 | """Returns a border image suitable for borders."""
85 | image = VImageAdapter.new_rgba(
86 | width, height, ink=rgba(r=0, g=0, b=0, a=0)
87 | )
88 | return image
89 |
90 |
91 | class SimpleFileStorage(Storage):
92 | """
93 | Saves tiles in `outputdir` as 'z-x-y-hash.ext'.
94 | """
95 |
96 | def __init__(self, renderer, outputdir, seen=None, **kwargs):
97 | """
98 | Initializes storage.
99 |
100 | renderer: Used to render images into tiles.
101 | outputdir: Output directory for tiles
102 | pool: Process pool to coordinate subprocesses.
103 | """
104 | super(SimpleFileStorage, self).__init__(renderer=renderer,
105 | **kwargs)
106 | if seen is None:
107 | seen = {}
108 | self.seen = seen
109 | self._border_hashed = None
110 |
111 | self.outputdir = outputdir
112 | makedirs(self.outputdir, ignore_exists=True)
113 |
114 | def filepath(self, x, y, z, hashed):
115 | """Returns the filepath, relative to self.outputdir."""
116 | return ('{z}-{x}-{y}-{hashed:x}'.format(**locals()) +
117 | self.renderer.suffix)
118 |
119 | def save(self, x, y, z, image):
120 | """Saves `image` at coordinates `x`, `y`, and `z`."""
121 | hashed = self.get_hash(image)
122 | filepath = self.filepath(x=x, y=y, z=z, hashed=hashed)
123 | if hashed in self.seen:
124 | self.symlink(src=self.seen[hashed], dst=filepath)
125 | else:
126 | self.seen[hashed] = filepath
127 | contents = self.renderer.render(image)
128 | outputfile = os.path.join(self.outputdir, filepath)
129 | with open(outputfile, 'wb') as output:
130 | output.write(contents)
131 |
132 | def symlink(self, src, dst):
133 | """Creates a relative symlink from dst to src."""
134 | absdst = os.path.join(self.outputdir, dst)
135 | abssrc = os.path.join(self.outputdir, src)
136 | srcpath = os.path.relpath(abssrc,
137 | start=os.path.dirname(absdst))
138 | os.symlink(srcpath, absdst)
139 |
140 | def save_border(self, x, y, z):
141 | """Saves a border image at coordinates `x`, `y`, and `z`."""
142 | if self._border_hashed is None or self._border_hashed not in self.seen:
143 | image = self._border_image()
144 | self._border_hashed = self.get_hash(image)
145 | self.save(x=x, y=y, z=z, image=image)
146 | else:
147 | # self._border_hashed will already be in self.seen
148 | filepath = self.filepath(x=x, y=y, z=z, hashed=self._border_hashed)
149 | self.symlink(src=self.seen[self._border_hashed], dst=filepath)
150 |
151 |
152 | class NestedFileStorage(SimpleFileStorage):
153 | """
154 | Saves tiles in `outputdir` as 'z/x/y.ext' for serving via static site.
155 | """
156 |
157 | def __init__(self, renderer, **kwargs):
158 | """
159 | Initializes storage.
160 |
161 | renderer: Used to render images into tiles.
162 | outputdir: Output directory for tiles
163 | pool: Process pool to coordinate subprocesses.
164 | """
165 | super(NestedFileStorage, self).__init__(renderer=renderer,
166 | **kwargs)
167 | self.madedirs = defaultdict(partial(defaultdict, bool))
168 |
169 | def filepath(self, x, y, z, hashed):
170 | """Returns the filepath, relative to self.outputdir."""
171 | return (os.path.join(str(z), str(x), str(y)) +
172 | self.renderer.suffix)
173 |
174 | def makedirs(self, x, y, z):
175 | if not self.madedirs[z][x]:
176 | makedirs(os.path.join(self.outputdir, str(z), str(x)),
177 | ignore_exists=True)
178 | self.madedirs[z][x] = True
179 |
180 | def save(self, x, y, z, image):
181 | """Saves `image` at coordinates `x`, `y`, and `z`."""
182 | self.makedirs(x=x, y=y, z=z)
183 | return super(NestedFileStorage, self).save(x=x, y=y, z=z, image=image)
184 |
185 | def save_border(self, x, y, z):
186 | """Saves a border image at coordinates `x`, `y`, and `z`."""
187 | self.makedirs(x=x, y=y, z=z)
188 | return super(NestedFileStorage, self).save_border(x=x, y=y, z=z)
189 |
190 |
191 | class MbtilesStorage(Storage):
192 | """
193 | Saves tiles in `filename` in the MBTiles format.
194 |
195 | http://mapbox.com/developers/mbtiles/
196 | """
197 | def __init__(self, renderer, filename, zoom_offset=None, seen=None,
198 | **kwargs):
199 | """
200 | Initializes storage.
201 |
202 | renderer: Used to render images into tiles.
203 | filename: Name of the MBTiles file.
204 | pool: Process pool to coordinate subprocesses.
205 | """
206 | super(MbtilesStorage, self).__init__(renderer=renderer,
207 | **kwargs)
208 | if zoom_offset is None:
209 | zoom_offset = 0
210 | self.zoom_offset = zoom_offset
211 |
212 | if seen is None:
213 | seen = set()
214 | self.seen = seen
215 | self._border_hashed = None
216 |
217 | self.mbtiles = None
218 |
219 | if isinstance(filename, basestring):
220 | self.filename = filename
221 | self.mbtiles = MBTiles(filename=filename)
222 | else:
223 | self.mbtiles = filename
224 | self.filename = self.mbtiles.filename
225 |
226 | def __del__(self):
227 | if self.mbtiles is not None:
228 | self.mbtiles.close()
229 |
230 | def __exit__(self, type, value, traceback):
231 | if self.mbtiles is not None:
232 | self.mbtiles.close()
233 |
234 | @classmethod
235 | def create(cls, renderer, filename, metadata, zoom_offset=None,
236 | version=None, **kwargs):
237 | """
238 | Creates a new MBTiles file.
239 |
240 | renderer: Used to render images into tiles.
241 | filename: Name of the MBTiles file.
242 | metadata: Metadata dictionary.
243 | zoom_offset: Offset zoom level.
244 |
245 | version: Optional MBTiles version.
246 | pool: Process pool to coordinate subprocesses.
247 |
248 | Metadata is also taken as **kwargs. See `mbtiles.Metadata`.
249 | """
250 | bounds = metadata.get('bounds', None)
251 | if bounds is not None:
252 | metadata['bounds'] = bounds.lower_left + bounds.upper_right
253 | mbtiles = MBTiles.create(filename=filename, metadata=metadata,
254 | version=version)
255 | return cls(renderer=renderer,
256 | filename=mbtiles,
257 | zoom_offset=zoom_offset,
258 | **kwargs)
259 |
260 | def post_import(self, pyramid):
261 | """Insert the dataset extents into the metadata."""
262 | # The MBTiles spec says that the bounds must be in EPSG:4326
263 | transform = pyramid.dataset.GetCoordinateTransformation(
264 | dst_ref=SpatialReference.FromEPSG(4326)
265 | )
266 |
267 | lower_left, upper_right = pyramid.dataset.GetTiledExtents(
268 | transform=transform
269 | )
270 | self.mbtiles.metadata['bounds'] = (lower_left.x, lower_left.y,
271 | upper_right.x, upper_right.y)
272 |
273 | def save(self, x, y, z, image):
274 | """Saves `image` at coordinates `x`, `y`, and `z`."""
275 | hashed = self.get_hash(image)
276 | if hashed in self.seen:
277 | self.mbtiles.insert(x=x, y=y,
278 | z=z + self.zoom_offset,
279 | hashed=hashed)
280 | else:
281 | self.seen.add(hashed)
282 | contents = self.renderer.render(image)
283 | if sys.version_info < (3, 0):
284 | data = buffer(contents)
285 | else:
286 | data = memoryview(contents)
287 | self.mbtiles.insert(x=x, y=y,
288 | z=z + self.zoom_offset,
289 | hashed=hashed,
290 | data=data)
291 |
292 | def save_border(self, x, y, z):
293 | """Saves a border image at coordinates `x`, `y`, and `z`."""
294 | if self._border_hashed is None:
295 | image = self._border_image()
296 | self.save(x=x, y=y, z=z, image=image)
297 | self._border_hashed = self.get_hash(image)
298 | else:
299 | # self._border_hashed will already be inserted
300 | self.mbtiles.insert(x=x, y=y,
301 | z=z + self.zoom_offset,
302 | hashed=self._border_hashed)
303 |
--------------------------------------------------------------------------------
/gdal2mbtiles/utils.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Licensed to Ecometrica under one or more contributor license
4 | # agreements. See the NOTICE file distributed with this work
5 | # for additional information regarding copyright ownership.
6 | # Ecometrica licenses this file to you under the Apache
7 | # License, Version 2.0 (the "License"); you may not use this
8 | # file except in compliance with the License. You may obtain a
9 | # copy of the License at
10 | #
11 | # http://www.apache.org/licenses/LICENSE-2.0
12 | #
13 | # Unless required by applicable law or agreed to in writing,
14 | # software distributed under the License is distributed on an
15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 | # KIND, either express or implied. See the License for the
17 | # specific language governing permissions and limitations
18 | # under the License.
19 |
20 | from __future__ import (absolute_import, division, print_function,
21 | unicode_literals)
22 |
23 | from contextlib import contextmanager
24 | import errno
25 | from hashlib import md5
26 | import os
27 | from shutil import rmtree
28 | from tempfile import mkdtemp
29 |
30 |
31 | @contextmanager
32 | def tempenv(name, value):
33 | original = os.environ.get(name, None)
34 | os.environ[name] = value
35 | yield
36 | if original is None:
37 | del os.environ[name]
38 | else:
39 | os.environ[name] = original
40 |
41 |
42 | @contextmanager
43 | def NamedTemporaryDir(**kwargs):
44 | dirname = mkdtemp(**kwargs)
45 | yield dirname
46 | rmtree(dirname, ignore_errors=True)
47 |
48 |
49 | def makedirs(d, ignore_exists=False):
50 | """Like `os.makedirs`, but doesn't raise OSError if ignore_exists."""
51 | try:
52 | os.makedirs(d)
53 | except OSError as e:
54 | if ignore_exists and e.errno == errno.EEXIST:
55 | return
56 | raise
57 |
58 |
59 | def rmfile(path, ignore_missing=False):
60 | """Like `os.remove`, but doesn't raise OSError if ignore_missing."""
61 | try:
62 | os.remove(path)
63 | except OSError as e:
64 | if ignore_missing and e.errno == errno.ENOENT:
65 | return
66 | raise
67 |
68 |
69 | def recursive_listdir(directory):
70 | """Generator of all files in `directory`, recursively."""
71 | for root, dirs, files in os.walk(directory):
72 | root = os.path.relpath(root, directory)
73 | if root == '.':
74 | root = ''
75 | for d in dirs:
76 | yield os.path.join(root, d) + os.path.sep
77 | for f in files:
78 | yield os.path.join(root, f)
79 |
80 |
81 | def intmd5(x):
82 | """Returns the MD5 digest of `x` as an integer."""
83 | return int(md5(x).hexdigest(), base=16)
84 |
--------------------------------------------------------------------------------
/pytest.ini:
--------------------------------------------------------------------------------
1 | [pytest]
2 | markers =
3 | slow: mark a test as slow
4 | skip_on_ci: mark a test that should be skipped on ci
5 |
--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
1 | [aliases]
2 | test=pytest
3 |
4 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 |
4 | from setuptools import setup
5 |
6 | import gdal2mbtiles
7 |
8 |
9 | # Hack to prevent stupid TypeError: 'NoneType' object is not callable error on
10 | # exit of python setup.py test in multiprocessing/util.py _exit_function when
11 | # running python setup.py test (see
12 | # http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html)
13 | import multiprocessing
14 | multiprocessing
15 |
16 | setup(
17 | name='gdal2mbtiles',
18 | version=gdal2mbtiles.__version__,
19 | description=(
20 | 'Converts a GDAL-readable dataset into an MBTiles file.'
21 | 'This is used to generate web maps.'
22 | ),
23 | long_description=open('README.rst').read(),
24 | license='Apache Software License, version 2.0',
25 |
26 | author='Ecometrica',
27 | author_email='software@ecometrica.com',
28 | url='https://github.com/ecometrica/gdal2mbtiles',
29 |
30 | packages=['gdal2mbtiles'],
31 | include_package_data=True,
32 |
33 | # You also need certain dependencies that aren't in PyPi:
34 | # gdal-bin, libgdal-dev, libvips, libvips-dev, libtiff5, optipng, pngquant
35 | install_requires=[
36 | 'future',
37 | 'numexpr',
38 | 'numpy',
39 | 'pyvips',
40 | 'webcolors',
41 | ],
42 |
43 | extras_require={
44 | "tests": [
45 | "pytest",
46 | "pytest-pythonpath",
47 | "distro; platform_system=='Linux'"
48 | ],
49 | },
50 |
51 | entry_points={
52 | 'console_scripts': [
53 | 'gdal2mbtiles = gdal2mbtiles.main:main',
54 | ]
55 | },
56 |
57 | classifiers=[
58 | 'Development Status :: 5 - Production/Stable',
59 | 'Environment :: Console',
60 | 'Intended Audience :: Other Audience',
61 | 'License :: OSI Approved :: Apache Software License',
62 | 'Operating System :: POSIX',
63 | 'Programming Language :: Python :: 3.6',
64 | 'Programming Language :: Python :: 3.7',
65 | 'Programming Language :: Python :: 3.8',
66 | 'Topic :: Multimedia :: Graphics :: Graphics Conversion',
67 | 'Topic :: Scientific/Engineering :: GIS',
68 | ],
69 |
70 | zip_safe=True,
71 | )
72 |
--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from __future__ import (absolute_import, division, print_function,
4 | unicode_literals)
5 |
--------------------------------------------------------------------------------
/tests/bluemarble-aligned-ll.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/bluemarble-aligned-ll.tif
--------------------------------------------------------------------------------
/tests/bluemarble-foreign.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/bluemarble-foreign.tif
--------------------------------------------------------------------------------
/tests/bluemarble-slightly-too-big.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/bluemarble-slightly-too-big.tif
--------------------------------------------------------------------------------
/tests/bluemarble-spanning-foreign.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/bluemarble-spanning-foreign.tif
--------------------------------------------------------------------------------
/tests/bluemarble-spanning-ll.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/bluemarble-spanning-ll.tif
--------------------------------------------------------------------------------
/tests/bluemarble-wgs84.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/bluemarble-wgs84.tif
--------------------------------------------------------------------------------
/tests/bluemarble.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/bluemarble.tif
--------------------------------------------------------------------------------
/tests/bluemarble.xcf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/bluemarble.xcf
--------------------------------------------------------------------------------
/tests/paletted.nodata.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/paletted.nodata.tif
--------------------------------------------------------------------------------
/tests/paletted.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/paletted.tif
--------------------------------------------------------------------------------
/tests/srtm.nodata.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/srtm.nodata.tif
--------------------------------------------------------------------------------
/tests/srtm.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/srtm.tif
--------------------------------------------------------------------------------
/tests/test_helpers.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from __future__ import (absolute_import, division, print_function,
4 | unicode_literals)
5 |
6 | import os
7 | from tempfile import NamedTemporaryFile
8 | import unittest
9 |
10 | from gdal2mbtiles.exceptions import UnalignedInputError
11 | from gdal2mbtiles.gdal import Dataset
12 | from gdal2mbtiles.helpers import (image_mbtiles, image_pyramid, image_slice,
13 | warp_mbtiles, warp_pyramid, warp_slice)
14 | from gdal2mbtiles.renderers import TouchRenderer
15 | from gdal2mbtiles.storages import MbtilesStorage
16 | from gdal2mbtiles.utils import intmd5, NamedTemporaryDir, recursive_listdir
17 |
18 | TEST_ASSET_DIR = os.path.dirname(__file__)
19 |
20 |
21 | class TestImageMbtiles(unittest.TestCase):
22 | def setUp(self):
23 | self.inputfile = os.path.join(TEST_ASSET_DIR, 'bluemarble-aligned-ll.tif')
24 |
25 | def test_simple(self):
26 | with NamedTemporaryFile(suffix='.mbtiles') as outputfile:
27 | metadata = dict(
28 | name='bluemarble-aligned',
29 | type='baselayer',
30 | version='1.0.0',
31 | description='BlueMarble 2004-07 Aligned',
32 | format='png',
33 | )
34 | image_mbtiles(inputfile=self.inputfile, outputfile=outputfile.name,
35 | metadata=metadata,
36 | min_resolution=0, max_resolution=3,
37 | renderer=TouchRenderer(suffix='.png'))
38 | with MbtilesStorage(renderer=None,
39 | filename=outputfile.name) as storage:
40 | self.assertEqual(
41 | set((z, x, y) for z, x, y, data in storage.mbtiles.all()),
42 | set([(0, 0, 0)] +
43 | [(1, x, y) for x in range(0, 2) for y in range(0, 2)] +
44 | [(2, x, y) for x in range(0, 4) for y in range(0, 4)] +
45 | [(3, x, y) for x in range(0, 8) for y in range(0, 8)])
46 | )
47 | self.assertEqual(
48 | storage.mbtiles.metadata['bounds'],
49 | '-90.0,-90.0,0.0,0.0'
50 | )
51 | self.assertEqual(storage.mbtiles.metadata['x-minzoom'], '0')
52 | self.assertEqual(storage.mbtiles.metadata['x-maxzoom'], '3')
53 |
54 |
55 | class TestImagePyramid(unittest.TestCase):
56 | def setUp(self):
57 | self.inputfile = os.path.join(TEST_ASSET_DIR, 'bluemarble.tif')
58 | self.alignedfile = os.path.join(TEST_ASSET_DIR, 'bluemarble-aligned-ll.tif')
59 | self.spanningfile = os.path.join(TEST_ASSET_DIR, 'bluemarble-spanning-ll.tif')
60 | self.upsamplingfile = os.path.join(TEST_ASSET_DIR, 'upsampling.tif')
61 |
62 | def test_simple(self):
63 | with NamedTemporaryDir() as outputdir:
64 | # Native resolution only
65 | image_pyramid(inputfile=self.inputfile, outputdir=outputdir,
66 | renderer=TouchRenderer(suffix='.png'))
67 |
68 | self.assertEqual(
69 | set(recursive_listdir(outputdir)),
70 | set((
71 | '2/',
72 | '2/0/',
73 | '2/0/0.png',
74 | '2/0/1.png',
75 | '2/0/2.png',
76 | '2/0/3.png',
77 | '2/1/',
78 | '2/1/0.png',
79 | '2/1/1.png',
80 | '2/1/2.png',
81 | '2/1/3.png',
82 | '2/2/',
83 | '2/2/0.png',
84 | '2/2/1.png',
85 | '2/2/2.png',
86 | '2/2/3.png',
87 | '2/3/',
88 | '2/3/0.png',
89 | '2/3/1.png',
90 | '2/3/2.png',
91 | '2/3/3.png',
92 | ))
93 | )
94 |
95 | def test_downsample(self):
96 | with NamedTemporaryDir() as outputdir:
97 | image_pyramid(inputfile=self.inputfile, outputdir=outputdir,
98 | min_resolution=0,
99 | renderer=TouchRenderer(suffix='.png'))
100 |
101 | files = set(recursive_listdir(outputdir))
102 | self.assertEqual(
103 | files,
104 | set((
105 | '0/',
106 | '0/0/',
107 | '0/0/0.png',
108 | '1/',
109 | '1/0/',
110 | '1/0/0.png',
111 | '1/0/1.png',
112 | '1/1/',
113 | '1/1/0.png',
114 | '1/1/1.png',
115 | '2/',
116 | '2/0/',
117 | '2/0/0.png',
118 | '2/0/1.png',
119 | '2/0/2.png',
120 | '2/0/3.png',
121 | '2/1/',
122 | '2/1/0.png',
123 | '2/1/1.png',
124 | '2/1/2.png',
125 | '2/1/3.png',
126 | '2/2/',
127 | '2/2/0.png',
128 | '2/2/1.png',
129 | '2/2/2.png',
130 | '2/2/3.png',
131 | '2/3/',
132 | '2/3/0.png',
133 | '2/3/1.png',
134 | '2/3/2.png',
135 | '2/3/3.png',
136 | ))
137 | )
138 |
139 | def test_downsample_aligned(self):
140 | with NamedTemporaryDir() as outputdir:
141 | image_pyramid(inputfile=self.alignedfile, outputdir=outputdir,
142 | min_resolution=0,
143 | renderer=TouchRenderer(suffix='.png'))
144 |
145 | files = set(recursive_listdir(outputdir))
146 | self.assertEqual(
147 | files,
148 | set((
149 | '0/',
150 | '0/0/',
151 | '0/0/0.png',
152 | '1/',
153 | '1/0/',
154 | '1/1/',
155 | '1/0/0.png',
156 | '2/',
157 | '2/0/',
158 | '2/1/',
159 | '2/2/',
160 | '2/3/',
161 | '2/1/1.png',
162 | # The following are the borders
163 | '1/0/1.png',
164 | '1/1/0.png',
165 | '1/1/1.png',
166 | '2/0/0.png',
167 | '2/0/1.png',
168 | '2/0/2.png',
169 | '2/0/3.png',
170 | '2/1/0.png',
171 | '2/1/2.png',
172 | '2/1/3.png',
173 | '2/2/0.png',
174 | '2/2/1.png',
175 | '2/2/2.png',
176 | '2/2/3.png',
177 | '2/3/0.png',
178 | '2/3/1.png',
179 | '2/3/2.png',
180 | '2/3/3.png',
181 | ))
182 | )
183 |
184 | def test_downsample_spanning(self):
185 | with NamedTemporaryDir() as outputdir:
186 | self.assertRaises(UnalignedInputError,
187 | image_pyramid,
188 | inputfile=self.spanningfile, outputdir=outputdir,
189 | min_resolution=0,
190 | renderer=TouchRenderer(suffix='.png'))
191 |
192 | def test_upsample(self):
193 | with NamedTemporaryDir() as outputdir:
194 | dataset = Dataset(self.inputfile)
195 | image_pyramid(inputfile=self.inputfile, outputdir=outputdir,
196 | max_resolution=dataset.GetNativeResolution() + 1,
197 | renderer=TouchRenderer(suffix='.png'))
198 |
199 | files = set(recursive_listdir(outputdir))
200 | self.assertEqual(
201 | files,
202 | set((
203 | '2/',
204 | '2/0/',
205 | '2/0/0.png',
206 | '2/0/1.png',
207 | '2/0/2.png',
208 | '2/0/3.png',
209 | '2/1/',
210 | '2/1/0.png',
211 | '2/1/1.png',
212 | '2/1/2.png',
213 | '2/1/3.png',
214 | '2/2/',
215 | '2/2/0.png',
216 | '2/2/1.png',
217 | '2/2/2.png',
218 | '2/2/3.png',
219 | '2/3/',
220 | '2/3/0.png',
221 | '2/3/1.png',
222 | '2/3/2.png',
223 | '2/3/3.png',
224 | '3/',
225 | '3/0/',
226 | '3/0/0.png',
227 | '3/0/1.png',
228 | '3/0/2.png',
229 | '3/0/3.png',
230 | '3/0/4.png',
231 | '3/0/5.png',
232 | '3/0/6.png',
233 | '3/0/7.png',
234 | '3/1/',
235 | '3/1/0.png',
236 | '3/1/1.png',
237 | '3/1/2.png',
238 | '3/1/3.png',
239 | '3/1/4.png',
240 | '3/1/5.png',
241 | '3/1/6.png',
242 | '3/1/7.png',
243 | '3/2/',
244 | '3/2/0.png',
245 | '3/2/1.png',
246 | '3/2/2.png',
247 | '3/2/3.png',
248 | '3/2/4.png',
249 | '3/2/5.png',
250 | '3/2/6.png',
251 | '3/2/7.png',
252 | '3/3/',
253 | '3/3/0.png',
254 | '3/3/1.png',
255 | '3/3/2.png',
256 | '3/3/3.png',
257 | '3/3/4.png',
258 | '3/3/5.png',
259 | '3/3/6.png',
260 | '3/3/7.png',
261 | '3/4/',
262 | '3/4/0.png',
263 | '3/4/1.png',
264 | '3/4/2.png',
265 | '3/4/3.png',
266 | '3/4/4.png',
267 | '3/4/5.png',
268 | '3/4/6.png',
269 | '3/4/7.png',
270 | '3/5/',
271 | '3/5/0.png',
272 | '3/5/1.png',
273 | '3/5/2.png',
274 | '3/5/3.png',
275 | '3/5/4.png',
276 | '3/5/5.png',
277 | '3/5/6.png',
278 | '3/5/7.png',
279 | '3/6/',
280 | '3/6/0.png',
281 | '3/6/1.png',
282 | '3/6/2.png',
283 | '3/6/3.png',
284 | '3/6/4.png',
285 | '3/6/5.png',
286 | '3/6/6.png',
287 | '3/6/7.png',
288 | '3/7/',
289 | '3/7/0.png',
290 | '3/7/1.png',
291 | '3/7/2.png',
292 | '3/7/3.png',
293 | '3/7/4.png',
294 | '3/7/5.png',
295 | '3/7/6.png',
296 | '3/7/7.png',
297 | ))
298 | )
299 |
300 | def test_upsample_symlink(self):
301 | with NamedTemporaryDir() as outputdir:
302 | zoom = 3
303 |
304 | dataset = Dataset(self.upsamplingfile)
305 | image_pyramid(inputfile=self.upsamplingfile, outputdir=outputdir,
306 | max_resolution=dataset.GetNativeResolution() + zoom,
307 | renderer=TouchRenderer(suffix='.png'))
308 |
309 | files = set(recursive_listdir(outputdir))
310 | self.assertEqual(
311 | files,
312 | set([
313 | '0/',
314 | '0/0/',
315 | '0/0/0.png',
316 | '1/',
317 | '1/0/',
318 | '1/0/0.png',
319 | '1/0/1.png',
320 | '1/1/',
321 | '1/1/0.png',
322 | '1/1/1.png',
323 | '2/',
324 | '2/0/',
325 | '2/0/0.png',
326 | '2/0/1.png',
327 | '2/0/2.png',
328 | '2/0/3.png',
329 | '2/1/',
330 | '2/1/0.png',
331 | '2/1/1.png',
332 | '2/1/2.png',
333 | '2/1/3.png',
334 | '2/2/',
335 | '2/2/0.png',
336 | '2/2/1.png',
337 | '2/2/2.png',
338 | '2/2/3.png',
339 | '2/3/',
340 | '2/3/0.png',
341 | '2/3/1.png',
342 | '2/3/2.png',
343 | '2/3/3.png',
344 | '3/',
345 | '3/0/',
346 | '3/0/0.png',
347 | '3/0/1.png',
348 | '3/0/2.png',
349 | '3/0/3.png',
350 | '3/0/4.png',
351 | '3/0/5.png',
352 | '3/0/6.png',
353 | '3/0/7.png',
354 | '3/1/',
355 | '3/1/0.png',
356 | '3/1/1.png',
357 | '3/1/2.png',
358 | '3/1/3.png',
359 | '3/1/4.png',
360 | '3/1/5.png',
361 | '3/1/6.png',
362 | '3/1/7.png',
363 | '3/2/',
364 | '3/2/0.png',
365 | '3/2/1.png',
366 | '3/2/2.png',
367 | '3/2/3.png',
368 | '3/2/4.png',
369 | '3/2/5.png',
370 | '3/2/6.png',
371 | '3/2/7.png',
372 | '3/3/',
373 | '3/3/0.png',
374 | '3/3/1.png',
375 | '3/3/2.png',
376 | '3/3/3.png',
377 | '3/3/4.png',
378 | '3/3/5.png',
379 | '3/3/6.png',
380 | '3/3/7.png',
381 | '3/4/',
382 | '3/4/0.png',
383 | '3/4/1.png',
384 | '3/4/2.png',
385 | '3/4/3.png',
386 | '3/4/4.png',
387 | '3/4/5.png',
388 | '3/4/6.png',
389 | '3/4/7.png',
390 | '3/5/',
391 | '3/5/0.png',
392 | '3/5/1.png',
393 | '3/5/2.png',
394 | '3/5/3.png',
395 | '3/5/4.png',
396 | '3/5/5.png',
397 | '3/5/6.png',
398 | '3/5/7.png',
399 | '3/6/',
400 | '3/6/0.png',
401 | '3/6/1.png',
402 | '3/6/2.png',
403 | '3/6/3.png',
404 | '3/6/4.png',
405 | '3/6/5.png',
406 | '3/6/6.png',
407 | '3/6/7.png',
408 | '3/7/',
409 | '3/7/0.png',
410 | '3/7/1.png',
411 | '3/7/2.png',
412 | '3/7/3.png',
413 | '3/7/4.png',
414 | '3/7/5.png',
415 | '3/7/6.png',
416 | '3/7/7.png',
417 | ])
418 | )
419 |
420 |
421 | class TestImageSlice(unittest.TestCase):
422 | def setUp(self):
423 | self.inputfile = os.path.join(TEST_ASSET_DIR, 'bluemarble.tif')
424 | self.alignedfile = os.path.join(TEST_ASSET_DIR, 'bluemarble-aligned-ll.tif')
425 | self.spanningfile = os.path.join(TEST_ASSET_DIR, 'bluemarble-spanning-ll.tif')
426 |
427 | def test_simple(self):
428 | with NamedTemporaryDir() as outputdir:
429 | image_slice(inputfile=self.inputfile, outputdir=outputdir,
430 | renderer=TouchRenderer(suffix='.png'))
431 |
432 | files = set(os.listdir(outputdir))
433 | self.assertEqual(
434 | files,
435 | set((
436 | '2-0-0-79f8c5f88c49812a4171f0f6263b01b1.png',
437 | '2-0-1-4e1061ab62c06d63eed467cca58883d1.png',
438 | '2-0-2-2b2617db83b03d9cd96e8a68cb07ced5.png',
439 | '2-0-3-44b9bb8a7bbdd6b8e01df1dce701b38c.png',
440 | '2-1-0-f1d310a7a502fece03b96acb8c704330.png',
441 | '2-1-1-194af8a96a88d76d424382d6f7b6112a.png',
442 | '2-1-2-1269123b2c3fd725c39c0a134f4c0e95.png',
443 | '2-1-3-62aec6122aade3337b8ebe9f6b9540fe.png',
444 | '2-2-0-6326c9b0cae2a8959d6afda71127dc52.png',
445 | '2-2-1-556518834b1015c6cf9a7a90bc9ec73.png',
446 | '2-2-2-730e6a45a495d1289f96e09b7b7731ef.png',
447 | '2-2-3-385dac69cdbf4608469b8538a0e47e2b.png',
448 | '2-3-0-66644871022656b835ea6cea03c3dc0f.png',
449 | '2-3-1-c81a64912d77024b3170d7ab2fb82310.png',
450 | '2-3-2-7ced761dd1dbe412c6f5b9511f0b291.png',
451 | '2-3-3-3f42d6a0e36064ca452aed393a303dd1.png',
452 | ))
453 | )
454 |
455 | def test_aligned(self):
456 | with NamedTemporaryDir() as outputdir:
457 | image_slice(inputfile=self.alignedfile, outputdir=outputdir,
458 | renderer=TouchRenderer(suffix='.png'))
459 |
460 | files = set(os.listdir(outputdir))
461 | self.assertEqual(
462 | files,
463 | set((
464 | '2-1-1-99c4a766657c5b65a62ef7da9906508b.png',
465 | # The following are the borders
466 | '2-0-0-ec87a838931d4d5d2e94a04644788a55.png',
467 | '2-0-1-ec87a838931d4d5d2e94a04644788a55.png',
468 | '2-0-2-ec87a838931d4d5d2e94a04644788a55.png',
469 | '2-0-3-ec87a838931d4d5d2e94a04644788a55.png',
470 | '2-1-0-ec87a838931d4d5d2e94a04644788a55.png',
471 | '2-1-2-ec87a838931d4d5d2e94a04644788a55.png',
472 | '2-1-3-ec87a838931d4d5d2e94a04644788a55.png',
473 | '2-2-0-ec87a838931d4d5d2e94a04644788a55.png',
474 | '2-2-1-ec87a838931d4d5d2e94a04644788a55.png',
475 | '2-2-2-ec87a838931d4d5d2e94a04644788a55.png',
476 | '2-2-3-ec87a838931d4d5d2e94a04644788a55.png',
477 | '2-3-0-ec87a838931d4d5d2e94a04644788a55.png',
478 | '2-3-1-ec87a838931d4d5d2e94a04644788a55.png',
479 | '2-3-2-ec87a838931d4d5d2e94a04644788a55.png',
480 | '2-3-3-ec87a838931d4d5d2e94a04644788a55.png',
481 | ))
482 | )
483 |
484 | def test_spanning(self):
485 | with NamedTemporaryDir() as outputdir:
486 | self.assertRaises(UnalignedInputError,
487 | image_slice,
488 | inputfile=self.spanningfile, outputdir=outputdir)
489 |
490 |
491 | class TestWarpMbtiles(unittest.TestCase):
492 | def setUp(self):
493 | self.inputfile = os.path.join(TEST_ASSET_DIR, 'bluemarble-spanning-ll.tif')
494 |
495 | def test_simple(self):
496 | with NamedTemporaryFile(suffix='.mbtiles') as outputfile:
497 | metadata = dict(
498 | name='bluemarble-aligned',
499 | type='baselayer',
500 | version='1.0.0',
501 | description='BlueMarble 2004-07 Aligned',
502 | format='png',
503 | )
504 | warp_mbtiles(inputfile=self.inputfile, outputfile=outputfile.name,
505 | metadata=metadata,
506 | min_resolution=0, max_resolution=3,
507 | renderer=TouchRenderer(suffix='.png'))
508 | with MbtilesStorage(renderer=None,
509 | filename=outputfile.name) as storage:
510 | self.assertEqual(
511 | set((z, x, y) for z, x, y, data in storage.mbtiles.all()),
512 | set([(0, 0, 0)] +
513 | [(1, x, y) for x in range(0, 2) for y in range(0, 2)] +
514 | [(2, x, y) for x in range(0, 4) for y in range(0, 4)] +
515 | [(3, x, y) for x in range(0, 8) for y in range(0, 8)])
516 | )
517 | self.assertEqual(
518 | storage.mbtiles.metadata['bounds'],
519 | '-180.0,-90.0,0.0,0.0'
520 | )
521 | self.assertEqual(storage.mbtiles.metadata['x-minzoom'], '0')
522 | self.assertEqual(storage.mbtiles.metadata['x-maxzoom'], '3')
523 |
524 | def test_zoom_offset(self):
525 | with NamedTemporaryFile(suffix='.mbtiles') as outputfile:
526 | metadata = dict(
527 | name='bluemarble-aligned',
528 | type='baselayer',
529 | version='1.0.0',
530 | description='BlueMarble 2004-07 Aligned',
531 | format='png',
532 | )
533 | warp_mbtiles(inputfile=self.inputfile, outputfile=outputfile.name,
534 | metadata=metadata,
535 | min_resolution=0, max_resolution=3, zoom_offset=2,
536 | renderer=TouchRenderer(suffix='.png'))
537 |
538 | with MbtilesStorage(renderer=None, filename=outputfile.name) as storage:
539 | self.assertEqual(
540 | set((z, x, y) for z, x, y, data in storage.mbtiles.all()),
541 | set([(2, 0, 0)] +
542 | [(3, x, y) for x in range(0, 2) for y in range(0, 2)] +
543 | [(4, x, y) for x in range(0, 4) for y in range(0, 4)] +
544 | [(5, x, y) for x in range(0, 8) for y in range(0, 8)])
545 | )
546 | self.assertEqual(
547 | storage.mbtiles.metadata['bounds'],
548 | '-180.0,-90.0,0.0,0.0'
549 | )
550 | self.assertEqual(storage.mbtiles.metadata['x-minzoom'], '2')
551 | self.assertEqual(storage.mbtiles.metadata['x-maxzoom'], '5')
552 |
553 |
554 | class TestWarpPyramid(unittest.TestCase):
555 | def setUp(self):
556 | self.inputfile = os.path.join(TEST_ASSET_DIR, 'bluemarble-spanning-ll.tif')
557 |
558 | def test_simple(self):
559 | with NamedTemporaryDir() as outputdir:
560 | warp_pyramid(inputfile=self.inputfile, outputdir=outputdir,
561 | min_resolution=0, max_resolution=3,
562 | renderer=TouchRenderer(suffix='.png'))
563 | self.assertEqual(
564 | set(recursive_listdir(outputdir)),
565 | set((
566 | '0/',
567 | '0/0/',
568 | '0/0/0.png',
569 | '1/',
570 | '1/0/',
571 | '1/0/0.png',
572 | '1/1/',
573 | '2/',
574 | '2/0/',
575 | '2/0/0.png',
576 | '2/0/1.png',
577 | '2/1/',
578 | '2/1/0.png',
579 | '2/1/1.png',
580 | '2/2/',
581 | '2/3/',
582 | '3/',
583 | '3/0/',
584 | '3/0/0.png',
585 | '3/0/1.png',
586 | '3/0/2.png',
587 | '3/0/3.png',
588 | '3/1/',
589 | '3/1/0.png',
590 | '3/1/1.png',
591 | '3/1/2.png',
592 | '3/1/3.png',
593 | '3/2/',
594 | '3/2/0.png',
595 | '3/2/1.png',
596 | '3/2/2.png',
597 | '3/2/3.png',
598 | '3/3/',
599 | '3/3/0.png',
600 | '3/3/1.png',
601 | '3/3/2.png',
602 | '3/3/3.png',
603 | '3/4/',
604 | '3/5/',
605 | '3/6/',
606 | '3/7/',
607 | # The following are the borders
608 | '1/0/1.png',
609 | '1/1/0.png',
610 | '1/1/1.png',
611 | '2/0/2.png',
612 | '2/0/3.png',
613 | '2/1/2.png',
614 | '2/1/3.png',
615 | '2/2/0.png',
616 | '2/2/1.png',
617 | '2/2/2.png',
618 | '2/2/3.png',
619 | '2/3/0.png',
620 | '2/3/1.png',
621 | '2/3/2.png',
622 | '2/3/3.png',
623 | '3/0/4.png',
624 | '3/0/5.png',
625 | '3/0/6.png',
626 | '3/0/7.png',
627 | '3/1/4.png',
628 | '3/1/5.png',
629 | '3/1/6.png',
630 | '3/1/7.png',
631 | '3/2/4.png',
632 | '3/2/5.png',
633 | '3/2/6.png',
634 | '3/2/7.png',
635 | '3/3/4.png',
636 | '3/3/5.png',
637 | '3/3/6.png',
638 | '3/3/7.png',
639 | '3/4/0.png',
640 | '3/4/1.png',
641 | '3/4/2.png',
642 | '3/4/3.png',
643 | '3/4/4.png',
644 | '3/4/5.png',
645 | '3/4/6.png',
646 | '3/4/7.png',
647 | '3/5/0.png',
648 | '3/5/1.png',
649 | '3/5/2.png',
650 | '3/5/3.png',
651 | '3/5/4.png',
652 | '3/5/5.png',
653 | '3/5/6.png',
654 | '3/5/7.png',
655 | '3/6/0.png',
656 | '3/6/1.png',
657 | '3/6/2.png',
658 | '3/6/3.png',
659 | '3/6/4.png',
660 | '3/6/5.png',
661 | '3/6/6.png',
662 | '3/6/7.png',
663 | '3/7/0.png',
664 | '3/7/1.png',
665 | '3/7/2.png',
666 | '3/7/3.png',
667 | '3/7/4.png',
668 | '3/7/5.png',
669 | '3/7/6.png',
670 | '3/7/7.png',
671 | ))
672 | )
673 |
674 |
675 | class TestWarpSlice(unittest.TestCase):
676 | def setUp(self):
677 | self.inputfile = os.path.join(TEST_ASSET_DIR, 'bluemarble-spanning-ll.tif')
678 |
679 | def test_simple(self):
680 | with NamedTemporaryDir() as outputdir:
681 | warp_slice(inputfile=self.inputfile, outputdir=outputdir,
682 | renderer=TouchRenderer(suffix='.png'))
683 | self.assertEqual(
684 | set(os.listdir(outputdir)),
685 | set((
686 | '2-0-0-26ef4e5b789cdc0646ca111264851a62.png',
687 | '2-0-1-a760093093243edf3557fddff32eba78.png',
688 | '2-0-2-ec87a838931d4d5d2e94a04644788a55.png',
689 | '2-1-0-3a60adfe5e110f70397d518d0bebc5fd.png',
690 | '2-1-1-fd0f72e802c90f4c3a2cbe25b7975d1.png',
691 | # The following are the borders
692 | '2-0-2-ec87a838931d4d5d2e94a04644788a55.png',
693 | '2-0-3-ec87a838931d4d5d2e94a04644788a55.png',
694 | '2-1-2-ec87a838931d4d5d2e94a04644788a55.png',
695 | '2-1-3-ec87a838931d4d5d2e94a04644788a55.png',
696 | '2-2-0-ec87a838931d4d5d2e94a04644788a55.png',
697 | '2-2-1-ec87a838931d4d5d2e94a04644788a55.png',
698 | '2-2-2-ec87a838931d4d5d2e94a04644788a55.png',
699 | '2-2-3-ec87a838931d4d5d2e94a04644788a55.png',
700 | '2-3-0-ec87a838931d4d5d2e94a04644788a55.png',
701 | '2-3-1-ec87a838931d4d5d2e94a04644788a55.png',
702 | '2-3-2-ec87a838931d4d5d2e94a04644788a55.png',
703 | '2-3-3-ec87a838931d4d5d2e94a04644788a55.png',
704 | ))
705 | )
706 |
--------------------------------------------------------------------------------
/tests/test_mbtiles.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from __future__ import (absolute_import, division, print_function,
4 | unicode_literals)
5 |
6 | import errno
7 | import os
8 | from tempfile import NamedTemporaryFile
9 | import unittest
10 |
11 | from gdal2mbtiles.mbtiles import (InvalidFileError, MetadataKeyError,
12 | MetadataValueError, Metadata, MBTiles)
13 |
14 |
15 | class TestMBTiles(unittest.TestCase):
16 | def setUp(self):
17 | self.tempfile = NamedTemporaryFile(suffix='.mbtiles')
18 | self.filename = self.tempfile.name
19 | self.version = '1.0'
20 | self.metadata = dict(
21 | name='transparent',
22 | type=Metadata.latest().TYPES.BASELAYER,
23 | version='1.0.0',
24 | description='Transparent World 2012',
25 | )
26 |
27 | def tearDown(self):
28 | try:
29 | self.tempfile.close()
30 | except OSError as e:
31 | if e.errno != errno.ENOENT:
32 | raise
33 |
34 | def test_open(self):
35 | with MBTiles.create(filename=self.filename,
36 | metadata=self.metadata,
37 | version=self.version):
38 | pass
39 |
40 | mbtiles = MBTiles(filename=self.filename)
41 |
42 | # Version detection
43 | self.assertEqual(mbtiles.version, self.version)
44 |
45 | # File are auto-opened
46 | self.assertFalse(mbtiles.closed)
47 | conn = mbtiles._conn
48 |
49 | # Open again
50 | self.assertNotEqual(mbtiles.open(), conn)
51 |
52 | # Close
53 | mbtiles.close()
54 | self.assertTrue(mbtiles.closed)
55 |
56 | def test_open_invalid(self):
57 | # Empty file
58 | self.assertRaises(InvalidFileError,
59 | MBTiles, filename=self.filename)
60 |
61 | # Python file
62 | self.assertRaises(InvalidFileError,
63 | MBTiles, filename=__file__)
64 |
65 | # Missing file
66 | self.assertRaises(IOError,
67 | MBTiles, filename='/dev/missing')
68 | self.assertRaises(IOError,
69 | MBTiles, filename='/missing')
70 |
71 | def test_create(self):
72 | # Create when filename does not exist
73 | os.remove(self.filename)
74 | mbtiles1 = MBTiles.create(filename=self.filename,
75 | metadata=self.metadata,
76 | version=self.version)
77 | self.assertFalse(mbtiles1.closed)
78 |
79 | # Create again when it exists
80 | mbtiles2 = MBTiles.create(filename=self.filename,
81 | metadata=self.metadata,
82 | version=self.version)
83 | self.assertFalse(mbtiles2.closed)
84 |
85 | self.assertNotEqual(mbtiles1, mbtiles2)
86 |
87 | # Create without version
88 | mbtiles3 = MBTiles.create(filename=self.filename,
89 | metadata=self.metadata)
90 | self.assertEqual(mbtiles3.version, self.version)
91 |
92 | def test_tiles(self):
93 | mbtiles = MBTiles.create(filename=':memory:',
94 | metadata=self.metadata,
95 | version=self.version)
96 | data = 'PNG image'
97 | hashed = hash(data)
98 |
99 | # Get missing tile
100 | self.assertEqual(mbtiles.get(x=0, y=0, z=0), None)
101 |
102 | # Insert tile
103 | mbtiles.insert(x=0, y=0, z=0, hashed=hashed, data=data)
104 |
105 | # Get inserted tile
106 | self.assertEqual(mbtiles.get(x=0, y=0, z=0), data)
107 |
108 | # Link tile
109 | mbtiles.insert(x=1, y=1, z=1, hashed=hashed)
110 |
111 | # Get linked tile
112 | self.assertEqual(mbtiles.get(x=1, y=1, z=1), data)
113 |
114 | def test_out_of_order_tile(self):
115 | mbtiles = MBTiles.create(filename=':memory:',
116 | metadata=self.metadata,
117 | version=self.version)
118 | data = 'PNG image'
119 | hashed = hash(data)
120 |
121 | # Link tile to nonexistent data
122 | mbtiles.insert(x=1, y=1, z=1, hashed=hashed)
123 |
124 | # Get linked tile
125 | self.assertEqual(mbtiles.get(x=1, y=1, z=1), None)
126 |
127 | # Add nonexistent data
128 | mbtiles.insert(x=0, y=0, z=0, hashed=hashed, data=data)
129 |
130 | # Get tile again
131 | self.assertEqual(mbtiles.get(x=1, y=1, z=1), data)
132 |
133 | def test_autocommit(self):
134 | mbtiles = MBTiles.create(filename=self.filename,
135 | metadata=self.metadata,
136 | version=self.version)
137 | data = 'PNG image'
138 | hashed = hash(data)
139 |
140 | # Insert tile
141 | mbtiles.insert(x=0, y=0, z=0, hashed=hashed, data=data)
142 | self.assertEqual(mbtiles.get(x=0, y=0, z=0), data)
143 |
144 | # Reopen
145 | mbtiles.open()
146 | self.assertEqual(mbtiles.get(x=0, y=0, z=0), data)
147 |
148 |
149 | class TestMetadata(unittest.TestCase):
150 | def setUp(self):
151 | self.filename = ':memory:'
152 | self.version = '1.0'
153 | self.metadata = dict(
154 | name='transparent',
155 | type=Metadata.latest().TYPES.BASELAYER,
156 | version='1.0.0',
157 | description='Transparent World 2012',
158 | )
159 |
160 | def test_simple(self):
161 | mbtiles = MBTiles.create(filename=self.filename,
162 | metadata=self.metadata,
163 | version=self.version)
164 | metadata = mbtiles.metadata
165 |
166 | # Set
167 | metadata['test'] = ''
168 | self.assertEqual(metadata['test'], '')
169 |
170 | # Set again
171 | metadata['test'] = 'Tileset'
172 | self.assertEqual(metadata['test'], 'Tileset')
173 |
174 | # Get missing
175 | self.assertRaises(MetadataKeyError, metadata.__getitem__, 'missing')
176 | self.assertEqual(metadata.get('missing'), None)
177 | self.assertEqual(metadata.get('missing', False), False)
178 |
179 | # Contains
180 | self.assertTrue('test' in metadata)
181 | self.assertFalse('missing' in metadata)
182 |
183 | # Delete
184 | del metadata['test']
185 | self.assertFalse('test' in metadata)
186 |
187 | # Delete mandatory
188 | self.assertRaises(MetadataKeyError,
189 | metadata.__delitem__, 'name')
190 |
191 | # Pop
192 | metadata['test'] = 'Tileset'
193 | self.assertEqual(metadata.pop('test'), 'Tileset')
194 |
195 | # Pop missing
196 | self.assertRaises(MetadataKeyError, metadata.pop, 'test')
197 | self.assertEqual(metadata.pop('test', None), None)
198 |
199 | # Update
200 | data = dict(list(self.metadata.items()),
201 | name='Tileset',
202 | description='This is a test tileset.')
203 | metadata.update(data)
204 |
205 | # Keys
206 | self.assertEqual(set(metadata.keys()), set(data.keys()))
207 |
208 | # Values
209 | self.assertEqual(set(metadata.values()), set(data.values()))
210 |
211 | # Items
212 | self.assertEqual(set(metadata.items()), set(data.items()))
213 |
214 | # Compare with dictionary
215 | self.assertEqual(metadata, data)
216 |
217 | def test_validate_1_0(self):
218 | version = '1.0'
219 | metadata = dict(
220 | name='transparent',
221 | type=Metadata.all()[version].TYPES.BASELAYER,
222 | version='1.0.0',
223 | )
224 |
225 | self.assertRaises(MetadataKeyError,
226 | MBTiles.create, filename=self.filename, metadata={},
227 | version=version)
228 | metadata.update(dict(
229 | description='Transparent World 2012',
230 | ))
231 |
232 | with MBTiles.create(filename=self.filename,
233 | metadata=metadata) as mbtiles:
234 | self.assertEqual(mbtiles.version, version)
235 |
236 | with MBTiles.create(filename=self.filename,
237 | metadata=metadata,
238 | version=version) as mbtiles:
239 | metadata = mbtiles.metadata
240 | self.assertRaises(MetadataKeyError,
241 | metadata.__delitem__, 'name')
242 | self.assertRaises(MetadataKeyError,
243 | metadata.__delitem__, 'type')
244 | self.assertRaises(MetadataKeyError,
245 | metadata.__delitem__, 'version')
246 | self.assertRaises(MetadataKeyError,
247 | metadata.__delitem__, 'description')
248 |
249 | metadata['type'] = metadata.TYPES.OVERLAY
250 | self.assertEqual(metadata['type'], 'overlay')
251 | metadata['type'] = metadata.TYPES.BASELAYER
252 | self.assertEqual(metadata['type'], 'baselayer')
253 | self.assertRaises(MetadataValueError,
254 | metadata.__setitem__, 'type', 'invalid')
255 |
256 | def test_validate_1_1(self):
257 | version = '1.1'
258 | metadata = dict(
259 | name='transparent',
260 | type=Metadata.all()[version].TYPES.BASELAYER,
261 | version='1.0.0',
262 | description='Transparent World 2012',
263 | )
264 |
265 | self.assertRaises(MetadataKeyError,
266 | MBTiles.create, filename=self.filename,
267 | metadata=self.metadata, version=version)
268 | metadata.update(dict(
269 | format=Metadata.all()[version].FORMATS.PNG,
270 | bounds='-180.0,-85,180,85',
271 | ))
272 |
273 | with MBTiles.create(filename=self.filename,
274 | metadata=metadata,
275 | version=version) as mbtiles:
276 | metadata = mbtiles.metadata
277 | self.assertRaises(MetadataKeyError,
278 | metadata.__delitem__, 'name')
279 | self.assertRaises(MetadataKeyError,
280 | metadata.__delitem__, 'type')
281 | self.assertRaises(MetadataKeyError,
282 | metadata.__delitem__, 'version')
283 | self.assertRaises(MetadataKeyError,
284 | metadata.__delitem__, 'description')
285 | self.assertRaises(MetadataKeyError,
286 | metadata.__delitem__, 'format')
287 |
288 | metadata['type'] = metadata.TYPES.OVERLAY
289 | self.assertEqual(metadata['type'], 'overlay')
290 | metadata['type'] = metadata.TYPES.BASELAYER
291 | self.assertEqual(metadata['type'], 'baselayer')
292 | self.assertRaises(MetadataValueError,
293 | metadata.__setitem__, 'type', 'invalid')
294 |
295 | metadata['format'] = metadata.FORMATS.PNG
296 | self.assertEqual(metadata['format'], 'png')
297 | metadata['format'] = metadata.FORMATS.JPG
298 | self.assertEqual(metadata['format'], 'jpg')
299 | self.assertRaises(MetadataValueError,
300 | metadata.__setitem__, 'format', 'invalid')
301 |
302 | metadata['bounds'] = '-1,-1,1,1'
303 | metadata['bounds'] = '-1.0,-1.0,1.0,1.0'
304 | metadata['bounds'] = '-1.0,-1.0,1.0,1.0'
305 | # left < -180
306 | self.assertRaises(MetadataValueError,
307 | metadata.__setitem__, 'bounds', '-180.1,-1,1,1')
308 | # bottom < -90
309 | self.assertRaises(MetadataValueError,
310 | metadata.__setitem__, 'bounds', '-1,-90.1,1,1')
311 | # right > 180
312 | self.assertRaises(MetadataValueError,
313 | metadata.__setitem__, 'bounds', '-1,-1,180.1,1')
314 | # top > 90
315 | self.assertRaises(MetadataValueError,
316 | metadata.__setitem__, 'bounds', '-1,-1,1,90.1')
317 | # left == right
318 | self.assertRaises(MetadataValueError,
319 | metadata.__setitem__, 'bounds', '1,-1,1,1')
320 | # left > right
321 | self.assertRaises(MetadataValueError,
322 | metadata.__setitem__, 'bounds', '1.1,-1,1,1')
323 | # bottom == top
324 | self.assertRaises(MetadataValueError,
325 | metadata.__setitem__, 'bounds', '-1,1,1,1')
326 | # bottom > top
327 | self.assertRaises(MetadataValueError,
328 | metadata.__setitem__, 'bounds', '-1,1.1,1,1')
329 |
330 | def test_validate_1_2(self):
331 | version = '1.2'
332 | metadata = dict(
333 | name='transparent',
334 | type=Metadata.all()[version].TYPES.BASELAYER,
335 | version='1.0.0',
336 | description='Transparent World 2012',
337 | )
338 |
339 | self.assertRaises(MetadataKeyError,
340 | MBTiles.create, filename=self.filename,
341 | metadata=self.metadata, version=version)
342 | metadata.update(dict(
343 | format=Metadata.all()[version].FORMATS.PNG,
344 | bounds='-180.0,-85,180,85',
345 | attribution='Brought to you by the letter A and the number 1.',
346 | ))
347 |
348 | with MBTiles.create(filename=self.filename,
349 | metadata=metadata) as mbtiles:
350 | self.assertEqual(mbtiles.version, version)
351 |
352 | with MBTiles.create(filename=self.filename,
353 | metadata=metadata,
354 | version=version) as mbtiles:
355 | metadata = mbtiles.metadata
356 | self.assertRaises(MetadataKeyError,
357 | metadata.__delitem__, 'name')
358 | self.assertRaises(MetadataKeyError,
359 | metadata.__delitem__, 'type')
360 | self.assertRaises(MetadataKeyError,
361 | metadata.__delitem__, 'version')
362 | self.assertRaises(MetadataKeyError,
363 | metadata.__delitem__, 'description')
364 | self.assertRaises(MetadataKeyError,
365 | metadata.__delitem__, 'format')
366 |
367 | metadata['type'] = metadata.TYPES.OVERLAY
368 | self.assertEqual(metadata['type'], 'overlay')
369 | metadata['type'] = metadata.TYPES.BASELAYER
370 | self.assertEqual(metadata['type'], 'baselayer')
371 | self.assertRaises(MetadataValueError,
372 | metadata.__setitem__, 'type', 'invalid')
373 |
374 | metadata['format'] = metadata.FORMATS.PNG
375 | self.assertEqual(metadata['format'], 'png')
376 | metadata['format'] = metadata.FORMATS.JPG
377 | self.assertEqual(metadata['format'], 'jpg')
378 | self.assertRaises(MetadataValueError,
379 | metadata.__setitem__, 'format', 'invalid')
380 |
381 | metadata['bounds'] = '-1,-1,1,1'
382 | metadata['bounds'] = '-1.0,-1.0,1.0,1.0'
383 | metadata['bounds'] = '-1.0,-1.0,1.0,1.0'
384 | # left < -180
385 | self.assertRaises(MetadataValueError,
386 | metadata.__setitem__, 'bounds', '-180.1,-1,1,1')
387 | # bottom < -90
388 | self.assertRaises(MetadataValueError,
389 | metadata.__setitem__, 'bounds', '-1,-90.1,1,1')
390 | # right > 180
391 | self.assertRaises(MetadataValueError,
392 | metadata.__setitem__, 'bounds', '-1,-1,180.1,1')
393 | # top > 90
394 | self.assertRaises(MetadataValueError,
395 | metadata.__setitem__, 'bounds', '-1,-1,1,90.1')
396 | # left == right
397 | self.assertRaises(MetadataValueError,
398 | metadata.__setitem__, 'bounds', '1,-1,1,1')
399 | # left > right
400 | self.assertRaises(MetadataValueError,
401 | metadata.__setitem__, 'bounds', '1.1,-1,1,1')
402 | # bottom == top
403 | self.assertRaises(MetadataValueError,
404 | metadata.__setitem__, 'bounds', '-1,1,1,1')
405 | # bottom > top
406 | self.assertRaises(MetadataValueError,
407 | metadata.__setitem__, 'bounds', '-1,1.1,1,1')
408 |
409 | def test_autocommit(self):
410 | with NamedTemporaryFile(suffix='.mbtiles') as tempfile:
411 | mbtiles = MBTiles.create(filename=tempfile.name,
412 | metadata=self.metadata,
413 | version=self.version)
414 |
415 | # Insert metadata
416 | mbtiles.metadata['test'] = 'Tileset'
417 | self.assertEqual(mbtiles.metadata['test'], 'Tileset')
418 |
419 | # Reopen
420 | mbtiles.open()
421 | self.assertEqual(mbtiles.metadata['test'], 'Tileset')
422 |
423 | # Delete metadata
424 | del mbtiles.metadata['test']
425 | self.assertRaises(KeyError, mbtiles.metadata.__getitem__, 'test')
426 |
427 | # Reopen
428 | mbtiles.open()
429 | self.assertRaises(KeyError, mbtiles.metadata.__getitem__, 'test')
430 |
--------------------------------------------------------------------------------
/tests/test_renderers.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from __future__ import (absolute_import, division, print_function,
4 | unicode_literals)
5 |
6 | import unittest
7 | import platform
8 |
9 | import pytest
10 |
11 | from gdal2mbtiles.renderers import JpegRenderer, PngRenderer, TouchRenderer
12 | from gdal2mbtiles.gd_types import rgba
13 | from gdal2mbtiles.utils import intmd5
14 | from gdal2mbtiles.vips import VImageAdapter
15 |
16 | # https://bugs.python.org/issue1322
17 | if platform.system() == 'Linux':
18 | import distro
19 | DISTRIBUTION = distro.linux_distribution()
20 | else:
21 | DISTRIBUTION = platform.system_alias(
22 | platform.system(),
23 | platform.release(),
24 | platform.version()
25 | )
26 |
27 | PNG8_OS_HASHES = {
28 | ('Ubuntu', '16.04', 'xenial'): 106831624867432276165545554861383631224,
29 | ('Ubuntu', '18.04', 'bionic'): 93909651943814796643456367818041361877,
30 | ('Ubuntu', '20.04', 'focal'): 226470660062402177473163372260043882022,
31 | }
32 |
33 | require_png8_os_hash = pytest.mark.skipif(
34 | DISTRIBUTION not in PNG8_OS_HASHES,
35 | reason="Result of png8 not specified for OS version"
36 | )
37 |
38 |
39 | class TestJpegRenderer(unittest.TestCase):
40 | def test_simple(self):
41 | renderer = JpegRenderer()
42 |
43 | # Black 1×1 image
44 | image = VImageAdapter.new_rgba(width=1, height=1,
45 | ink=rgba(r=0, g=0, b=0, a=255))
46 |
47 | black = renderer.render(image=image)
48 | black_md5 = intmd5(black)
49 |
50 | # Transparent 1×1 image
51 | image = VImageAdapter.new_rgba(width=1, height=1,
52 | ink=rgba(r=0, g=0, b=0, a=0))
53 |
54 | transparent = renderer.render(image=image)
55 | self.assertEqual(intmd5(transparent), black_md5)
56 |
57 | def test_suffix(self):
58 | # Default
59 | renderer = JpegRenderer()
60 | self.assertEqual(renderer.suffix, '.jpeg')
61 |
62 | # Specified
63 | renderer = JpegRenderer(suffix='.JPEG')
64 | self.assertEqual(renderer.suffix, '.JPEG')
65 |
66 |
67 | class TestPngRenderer(unittest.TestCase):
68 | def setUp(self):
69 | # Transparent 1×1 image
70 | self.image = VImageAdapter.new_rgba(width=1, height=1,
71 | ink=rgba(r=0, g=0, b=0, a=0))
72 |
73 | def test_simple(self):
74 | renderer = PngRenderer(png8=False, optimize=False)
75 | contents = renderer.render(image=self.image)
76 | self.assertEqual(intmd5(contents),
77 | 89446660811628514001822794642426893173)
78 |
79 | def test_compression(self):
80 | renderer = PngRenderer(compression=1, png8=False, optimize=False)
81 | contents = renderer.render(image=self.image)
82 | self.assertEqual(intmd5(contents),
83 | 227024021824580215543073313661866089265)
84 |
85 | def test_interlace(self):
86 | renderer = PngRenderer(interlace=1, png8=False, optimize=False)
87 | contents = renderer.render(image=self.image)
88 | self.assertEqual(intmd5(contents),
89 | 197686704564132731296723533976357306757)
90 |
91 | def test_optimize(self):
92 | renderer = PngRenderer(png8=False, optimize=2)
93 | contents = renderer.render(image=self.image)
94 | self.assertEqual(intmd5(contents),
95 | 227024021824580215543073313661866089265)
96 |
97 | # Default is PNG8=False and optimize=2
98 | renderer = PngRenderer()
99 | contents = renderer.render(image=self.image)
100 | self.assertEqual(intmd5(contents),
101 | 89446660811628514001822794642426893173)
102 |
103 | @require_png8_os_hash
104 | def test_png8(self):
105 | content_hash = PNG8_OS_HASHES[DISTRIBUTION]
106 | renderer = PngRenderer(png8=True, optimize=False)
107 | contents = renderer.render(image=self.image)
108 | self.assertEqual(intmd5(contents), content_hash)
109 |
110 | @require_png8_os_hash
111 | def test_png8_optimize(self):
112 | content_hash = PNG8_OS_HASHES[DISTRIBUTION]
113 | renderer = PngRenderer(png8=True, optimize=2)
114 | contents = renderer.render(image=self.image)
115 | # same hash as test_png8 since optipng treats it as already optimised
116 | self.assertEqual(intmd5(contents), content_hash)
117 |
118 | def test_suffix(self):
119 | # Default
120 | renderer = PngRenderer()
121 | self.assertEqual(renderer.suffix, '.png')
122 |
123 | # Specified
124 | renderer = PngRenderer(suffix='.PNG')
125 | self.assertEqual(renderer.suffix, '.PNG')
126 |
127 |
128 | class TestTouchRenderer(unittest.TestCase):
129 | def test_simple(self):
130 | renderer = TouchRenderer()
131 | contents = renderer.render(image=None)
132 | self.assertEqual(contents, b'')
133 |
134 | def test_suffix(self):
135 | # Default
136 | renderer = TouchRenderer()
137 | self.assertEqual(renderer.suffix, '')
138 |
139 | # Specified
140 | renderer = TouchRenderer(suffix='.bin')
141 | self.assertEqual(renderer.suffix, '.bin')
142 |
--------------------------------------------------------------------------------
/tests/test_scripts.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from __future__ import (absolute_import, division, print_function,
4 | unicode_literals)
5 |
6 | import os
7 | import pytest
8 | from subprocess import CalledProcessError, check_call
9 | import sys
10 | from tempfile import NamedTemporaryFile
11 | import unittest
12 |
13 | from gdal2mbtiles.mbtiles import MBTiles
14 |
15 |
16 | TEST_ASSET_DIR = os.path.dirname(__file__)
17 |
18 |
19 | class TestGdal2mbtilesScript(unittest.TestCase):
20 | def setUp(self):
21 | self.repo_dir = os.path.join(TEST_ASSET_DIR, os.path.pardir)
22 | self.script = os.path.join(self.repo_dir, 'gdal2mbtiles', 'main.py')
23 |
24 | self.environ = os.environ.copy()
25 |
26 | # Make sure you can get to the local gdal2mbtiles module
27 | pythonpath = self.environ.get('PYTHONPATH', [])
28 | if pythonpath:
29 | pythonpath = pythonpath.split(os.path.pathsep)
30 | pythonpath = os.path.pathsep.join([self.repo_dir] + pythonpath)
31 | self.environ['PYTHONPATH'] = pythonpath
32 |
33 | self.inputfile = os.path.join(TEST_ASSET_DIR, 'upsampling.tif')
34 | self.rgbfile = os.path.join(TEST_ASSET_DIR, 'bluemarble.tif')
35 | self.spanningfile = os.path.join(TEST_ASSET_DIR, 'bluemarble-spanning-ll.tif')
36 |
37 | def test_simple(self):
38 | with NamedTemporaryFile(suffix='.mbtiles') as output:
39 | command = [sys.executable, self.script, self.inputfile, output.name]
40 | check_call(command, env=self.environ)
41 | with MBTiles(output.name) as mbtiles:
42 | # 4×4 at resolution 2
43 | cursor = mbtiles._conn.execute('SELECT COUNT(*) FROM tiles')
44 | self.assertEqual(cursor.fetchone(), (1,))
45 |
46 | def test_metadata(self):
47 | with NamedTemporaryFile(suffix='.mbtiles') as output:
48 | command = [sys.executable, self.script, self.inputfile, output.name]
49 | check_call(command, env=self.environ)
50 |
51 | # Dataset (upsampling.tif) bounds in EPSG:4326
52 | dataset_bounds = '-180.0,-90.0,180.0,90.0'
53 |
54 | with MBTiles(output.name) as mbtiles:
55 | # Default metadata
56 | cursor = mbtiles._conn.execute('SELECT * FROM metadata')
57 | self.assertEqual(dict(cursor.fetchall()),
58 | {
59 | 'name': os.path.basename(self.inputfile),
60 | 'description': '',
61 | 'format': 'png',
62 | 'type': 'overlay',
63 | 'version': '1.0.0',
64 | 'bounds': dataset_bounds,
65 | 'x-minzoom': '0',
66 | 'x-maxzoom': '0',
67 | })
68 |
69 | command = [sys.executable, self.script,
70 | '--name', 'test',
71 | '--description', 'Unit test',
72 | '--format', 'jpg',
73 | '--layer-type', 'baselayer',
74 | '--version', '2.0.1',
75 | self.inputfile, output.name]
76 | check_call(command, env=self.environ)
77 | with MBTiles(output.name) as mbtiles:
78 | # Default metadata
79 | cursor = mbtiles._conn.execute('SELECT * FROM metadata')
80 | self.assertEqual(dict(cursor.fetchall()),
81 | {
82 | 'name': 'test',
83 | 'description': 'Unit test',
84 | 'format': 'jpg',
85 | 'type': 'baselayer',
86 | 'version': '2.0.1',
87 | 'bounds': dataset_bounds,
88 | 'x-minzoom': '0',
89 | 'x-maxzoom': '0',
90 | })
91 |
92 | def test_warp(self):
93 | null = open('/dev/null', 'r+')
94 |
95 | with NamedTemporaryFile(suffix='.mbtiles') as output:
96 | # Valid
97 | command = [sys.executable, self.script,
98 | '--spatial-reference', '4326',
99 | '--resampling', 'bilinear',
100 | self.rgbfile, output.name]
101 | check_call(command, env=self.environ)
102 |
103 | # Invalid spatial reference
104 | command = [sys.executable, self.script,
105 | '--spatial-reference', '9999',
106 | self.inputfile, output.name]
107 | self.assertRaises(CalledProcessError,
108 | check_call, command, env=self.environ,
109 | stderr=null)
110 |
111 | # Invalid resampling
112 | command = [sys.executable, self.script,
113 | '--resampling', 'montecarlo',
114 | self.inputfile, output.name]
115 | self.assertRaises(CalledProcessError,
116 | check_call, command, env=self.environ,
117 | stderr=null)
118 |
119 | def test_render(self):
120 | null = open('/dev/null', 'r+')
121 |
122 | with NamedTemporaryFile(suffix='.mbtiles') as output:
123 | # Valid
124 | command = [sys.executable, self.script,
125 | '--min-resolution', '1',
126 | '--max-resolution', '3',
127 | self.rgbfile, output.name]
128 | check_call(command, env=self.environ)
129 | with MBTiles(output.name) as mbtiles:
130 | cursor = mbtiles._conn.execute(
131 | """
132 | SELECT zoom_level, COUNT(*) FROM tiles
133 | GROUP BY zoom_level
134 | """
135 | )
136 | self.assertEqual(
137 | dict(cursor.fetchall()),
138 | {1: 4, # 2×2 at resolution 1
139 | 2: 16, # 4×4 at resolution 2
140 | 3: 64} # 8×8 at resolution 3
141 | )
142 |
143 | # Min resolution greater than input resolution with no max
144 | command = [sys.executable, self.script,
145 | '--min-resolution', '3',
146 | self.inputfile, output.name]
147 | self.assertRaises(
148 | CalledProcessError,
149 | check_call, command, env=self.environ, stderr=null
150 | )
151 |
152 | # Min resolution greater than max resolution
153 | command = [sys.executable, self.script,
154 | '--min-resolution', '2',
155 | '--max-resolution', '1',
156 | self.inputfile, output.name]
157 | self.assertRaises(
158 | CalledProcessError,
159 | check_call, command, env=self.environ, stderr=null
160 | )
161 |
162 | # Max resolution less than input resolution with no min
163 | command = [sys.executable, self.script,
164 | '--max-resolution', '0',
165 | self.rgbfile, output.name]
166 | self.assertRaises(
167 | CalledProcessError,
168 | check_call, command, env=self.environ, stderr=null
169 | )
170 |
171 | def test_fill_borders(self):
172 | with NamedTemporaryFile(suffix='.mbtiles') as output:
173 | # fill-borders
174 | command = [sys.executable, self.script,
175 | '--fill-borders',
176 | self.spanningfile, output.name]
177 | check_call(command, env=self.environ)
178 |
179 | # Dataset (bluemarble-spanning-ll.tif) bounds in EPSG:4326
180 | dataset_bounds = '-180.0,-90.0,0.0,0.0'
181 |
182 | with MBTiles(output.name) as mbtiles:
183 | # Default metadata
184 | cursor = mbtiles._conn.execute('SELECT * FROM metadata')
185 | self.assertTrue(dict(cursor.fetchall()),
186 | dict(name=os.path.basename(self.inputfile),
187 | description='',
188 | format='png',
189 | type='overlay',
190 | version='1.0.0',
191 | bounds=dataset_bounds))
192 | # 16 tiles
193 | cursor = cursor.execute('SELECT COUNT(*) FROM tiles')
194 | self.assertTrue(cursor.fetchone(), [16])
195 |
196 | # --no-fill-borders
197 | command = [sys.executable, self.script,
198 | '--no-fill-borders',
199 | self.spanningfile, output.name]
200 | check_call(command, env=self.environ)
201 | with MBTiles(output.name) as mbtiles:
202 | # 4 tiles, since the borders were not created
203 | cursor = mbtiles._conn.execute('SELECT COUNT(*) FROM tiles')
204 | self.assertTrue(cursor.fetchone(), [4])
205 |
206 | def test_colors(self):
207 | null = open('/dev/null', 'r+')
208 |
209 | with NamedTemporaryFile(suffix='.mbtiles') as output:
210 | # Valid
211 | command = [sys.executable, self.script,
212 | '--coloring', 'palette',
213 | '--color', '0:#00f',
214 | '--color', '1:green',
215 | self.inputfile, output.name]
216 | check_call(command, env=self.environ)
217 |
218 | # Invalid color
219 | command = [sys.executable, self.script,
220 | '--coloring', 'palette',
221 | '--color', 'invalid',
222 | self.inputfile, output.name]
223 | self.assertRaises(CalledProcessError,
224 | check_call, command, env=self.environ,
225 | stderr=null)
226 |
227 | command = [sys.executable, self.script,
228 | '--coloring', 'palette',
229 | '--color', '0:1',
230 | self.inputfile, output.name]
231 | self.assertRaises(CalledProcessError,
232 | check_call, command, env=self.environ,
233 | stderr=null)
234 |
235 | command = [sys.executable, self.script,
236 | '--coloring', 'palette',
237 | '--color', 's:#000',
238 | self.inputfile, output.name]
239 | self.assertRaises(CalledProcessError,
240 | check_call, command, env=self.environ,
241 | stderr=null)
242 |
243 | # Missing --color
244 | command = [sys.executable, self.script,
245 | '--coloring', 'palette',
246 | self.inputfile, output.name]
247 | self.assertRaises(CalledProcessError,
248 | check_call, command, env=self.environ,
249 | stderr=null)
250 |
251 | # Invalid --coloring
252 | command = [sys.executable, self.script,
253 | '--coloring', 'invalid',
254 | self.inputfile, output.name]
255 | self.assertRaises(CalledProcessError,
256 | check_call, command, env=self.environ,
257 | stderr=null)
258 |
259 | # Missing --coloring
260 | command = [sys.executable, self.script,
261 | '--color', '0:#00f',
262 | self.inputfile, output.name]
263 | self.assertRaises(CalledProcessError,
264 | check_call, command, env=self.environ,
265 | stderr=null)
266 |
267 | # Valid multi-band
268 | command = [sys.executable, self.script,
269 | '--coloring', 'gradient',
270 | '--color', '0:#00f',
271 | '--color', '1:green',
272 | '--colorize-band', '2',
273 | self.inputfile, output.name]
274 | check_call(command, env=self.environ)
275 |
276 | # Invalid band
277 | command = [sys.executable, self.script,
278 | '--coloring', 'palette',
279 | '--color', '0:#00f',
280 | '--color', '1:green',
281 | '--colorize-band', '-2',
282 | self.inputfile, output.name]
283 | self.assertRaises(CalledProcessError,
284 | check_call, command, env=self.environ,
285 | stderr=null)
286 |
--------------------------------------------------------------------------------
/tests/test_spatial_reference.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | import pytest
4 |
5 | from numpy import array
6 | from numpy.testing import assert_array_almost_equal
7 |
8 | from gdal2mbtiles.constants import (EPSG_WEB_MERCATOR,
9 | EPSG3857_EXTENTS)
10 | from gdal2mbtiles.gdal import SpatialReference
11 |
12 |
13 | @pytest.fixture
14 | def epsg_3857_from_proj4():
15 | """
16 | Return a gdal spatial reference object with
17 | 3857 crs using the ImportFromProj4 method.
18 | """
19 | spatial_ref = SpatialReference()
20 | spatial_ref.ImportFromProj4('+init=epsg:3857')
21 | return spatial_ref
22 |
23 |
24 | @pytest.fixture
25 | def epsg_3857_from_epsg():
26 | """
27 | Return a gdal spatial reference object with
28 | 3857 crs using the FromEPSG method.
29 | """
30 | spatial_ref = SpatialReference.FromEPSG(EPSG_WEB_MERCATOR)
31 | return spatial_ref
32 |
33 |
34 | def test_epsg_3857_proj4(epsg_3857_from_proj4):
35 | extents = epsg_3857_from_proj4.GetWorldExtents()
36 | extents = array(extents)
37 | assert_array_almost_equal(extents, EPSG3857_EXTENTS, decimal=3)
38 |
39 |
40 | def test_epsg_3857_from_epsg(epsg_3857_from_epsg):
41 | extents = epsg_3857_from_epsg.GetWorldExtents()
42 | extents = array(extents)
43 | assert_array_almost_equal(extents, EPSG3857_EXTENTS, decimal=3)
44 |
--------------------------------------------------------------------------------
/tests/test_storages.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from __future__ import (absolute_import, division, print_function,
4 | unicode_literals)
5 |
6 | import errno
7 | import os
8 | from shutil import rmtree
9 | from tempfile import NamedTemporaryFile
10 | import unittest
11 |
12 | from gdal2mbtiles.mbtiles import Metadata
13 | from gdal2mbtiles.renderers import PngRenderer, TouchRenderer
14 | from gdal2mbtiles.storages import (MbtilesStorage,
15 | NestedFileStorage, SimpleFileStorage)
16 | from gdal2mbtiles.gd_types import rgba
17 | from gdal2mbtiles.utils import intmd5, NamedTemporaryDir, recursive_listdir
18 | from gdal2mbtiles.vips import VImageAdapter
19 |
20 |
21 | class TestSimpleFileStorage(unittest.TestCase):
22 | def setUp(self):
23 | self.tempdir = NamedTemporaryDir()
24 | self.outputdir = self.tempdir.__enter__()
25 | self.renderer = TouchRenderer(suffix='.png')
26 | self.storage = SimpleFileStorage(outputdir=self.outputdir,
27 | renderer=self.renderer)
28 |
29 | def tearDown(self):
30 | self.tempdir.__exit__(None, None, None)
31 |
32 | def test_create(self):
33 | # Make a new directory if it doesn't exist
34 | os.rmdir(self.outputdir)
35 | storage = SimpleFileStorage(outputdir=self.outputdir,
36 | renderer=self.renderer)
37 | self.assertEqual(storage.outputdir, self.outputdir)
38 | self.assertTrue(os.path.isdir(self.outputdir))
39 |
40 | # Make a duplicate directory
41 | SimpleFileStorage(outputdir=self.outputdir,
42 | renderer=self.renderer)
43 | self.assertTrue(os.path.isdir(self.outputdir))
44 |
45 | def test_filepath(self):
46 | self.assertEqual(self.storage.filepath(x=0, y=1, z=2,
47 | hashed=0xdeadbeef),
48 | '2-0-1-deadbeef' + self.renderer.suffix)
49 |
50 | def test_get_hash(self):
51 | image = VImageAdapter.new_rgba(width=1, height=1,
52 | ink=rgba(r=0, g=0, b=0, a=0))
53 | self.assertEqual(self.storage.get_hash(image=image),
54 | int('f1d3ff8443297732862df21dc4e57262', base=16))
55 |
56 | def test_save(self):
57 | image = VImageAdapter.new_rgba(width=1, height=1,
58 | ink=rgba(r=0, g=0, b=0, a=0))
59 | self.storage.save(x=0, y=1, z=2, image=image)
60 | self.storage.save(x=1, y=0, z=2, image=image)
61 | self.assertEqual(set(os.listdir(self.outputdir)),
62 | set([
63 | '2-0-1-f1d3ff8443297732862df21dc4e57262.png',
64 | '2-1-0-f1d3ff8443297732862df21dc4e57262.png'
65 | ]))
66 |
67 | # Is this a real file?
68 | self.assertFalse(
69 | os.path.islink(os.path.join(
70 | self.outputdir, '2-0-1-f1d3ff8443297732862df21dc4e57262.png'
71 | ))
72 | )
73 |
74 | # Does the symlinking work?
75 | self.assertEqual(
76 | os.readlink(os.path.join(
77 | self.outputdir, '2-1-0-f1d3ff8443297732862df21dc4e57262.png'
78 | )),
79 | '2-0-1-f1d3ff8443297732862df21dc4e57262.png'
80 | )
81 |
82 | def test_symlink(self):
83 | # Same directory
84 | src = 'source'
85 | dst = 'destination'
86 | self.storage.symlink(src=src, dst=dst)
87 | self.assertEqual(os.listdir(self.outputdir),
88 | [dst])
89 | self.assertEqual(os.readlink(os.path.join(self.outputdir, dst)),
90 | src)
91 |
92 | # Subdirs
93 | subdir = os.path.join(self.outputdir, 'subdir')
94 | os.mkdir(subdir)
95 | self.storage.symlink(src=src, dst=os.path.join(subdir, dst))
96 | self.assertEqual(os.listdir(subdir),
97 | [dst])
98 | self.assertEqual(os.readlink(os.path.join(subdir, dst)),
99 | os.path.join(os.path.pardir, src))
100 |
101 | def test_save_border(self):
102 | # Western hemisphere is border
103 | self.storage.save_border(x=0, y=0, z=1)
104 | self.storage.save_border(x=0, y=1, z=1)
105 | self.assertEqual(set(sorted(os.listdir(self.outputdir))),
106 | set(sorted([
107 | '1-0-0-ec87a838931d4d5d2e94a04644788a55.png',
108 | '1-0-1-ec87a838931d4d5d2e94a04644788a55.png',
109 | ])))
110 |
111 | # Is this a real file?
112 | self.assertFalse(
113 | os.path.islink(os.path.join(
114 | self.outputdir, '1-0-0-ec87a838931d4d5d2e94a04644788a55.png'
115 | ))
116 | )
117 |
118 | # Does the symlinking work?
119 | self.assertEqual(
120 | os.readlink(os.path.join(
121 | self.outputdir, '1-0-1-ec87a838931d4d5d2e94a04644788a55.png'
122 | )),
123 | '1-0-0-ec87a838931d4d5d2e94a04644788a55.png'
124 | )
125 |
126 |
127 | class TestNestedFileStorage(unittest.TestCase):
128 | def setUp(self):
129 | self.tempdir = NamedTemporaryDir()
130 | self.outputdir = self.tempdir.__enter__()
131 | self.renderer = TouchRenderer(suffix='.png')
132 | self.storage = NestedFileStorage(outputdir=self.outputdir,
133 | renderer=self.renderer)
134 |
135 | def tearDown(self):
136 | self.tempdir.__exit__(None, None, None)
137 |
138 | def test_create(self):
139 | # Make a new directory if it doesn't exist
140 | os.rmdir(self.outputdir)
141 | storage = NestedFileStorage(outputdir=self.outputdir,
142 | renderer=self.renderer)
143 | self.assertEqual(storage.outputdir, self.outputdir)
144 | self.assertTrue(os.path.isdir(self.outputdir))
145 |
146 | # Make a duplicate directory
147 | NestedFileStorage(outputdir=self.outputdir,
148 | renderer=self.renderer)
149 | self.assertTrue(os.path.isdir(self.outputdir))
150 |
151 | def test_filepath(self):
152 | self.assertEqual(self.storage.filepath(x=0, y=1, z=2,
153 | hashed=0xdeadbeef),
154 | '2/0/1' + self.renderer.suffix)
155 |
156 | def test_makedirs(self):
157 | # Cache should be empty
158 | self.assertFalse(self.storage.madedirs)
159 |
160 | self.storage.makedirs(x=0, y=1, z=2)
161 | self.assertEqual(set(recursive_listdir(self.outputdir)),
162 | set(['2/',
163 | '2/0/']))
164 |
165 | # Is cache populated?
166 | self.assertTrue(self.storage.madedirs[2][0])
167 |
168 | # Delete and readd without clearing cache
169 | rmtree(os.path.join(self.outputdir, '2'))
170 | self.assertEqual(os.listdir(self.outputdir), [])
171 | self.storage.makedirs(x=0, y=1, z=2)
172 | self.assertEqual(os.listdir(self.outputdir), [])
173 |
174 | def test_save(self):
175 | image = VImageAdapter.new_rgba(width=1, height=1,
176 | ink=rgba(r=0, g=0, b=0, a=0))
177 | self.storage.save(x=0, y=1, z=2, image=image)
178 | self.storage.save(x=1, y=0, z=2, image=image)
179 | self.storage.save(x=1, y=0, z=3, image=image)
180 | self.assertEqual(set(recursive_listdir(self.outputdir)),
181 | set(['2/',
182 | '2/0/',
183 | '2/0/1.png',
184 | '2/1/',
185 | '2/1/0.png',
186 | '3/',
187 | '3/1/',
188 | '3/1/0.png']))
189 |
190 | # Is this a real file?
191 | self.assertFalse(
192 | os.path.islink(os.path.join(self.outputdir, '2', '0', '1.png'))
193 | )
194 |
195 | # Does the symlinking work?
196 | self.assertEqual(
197 | os.readlink(os.path.join(self.outputdir, '2', '1', '0.png')),
198 | os.path.join(os.path.pardir, '0', '1.png')
199 | )
200 | self.assertEqual(
201 | os.readlink(os.path.join(self.outputdir, '3', '1', '0.png')),
202 | os.path.join(os.path.pardir, os.path.pardir, '2', '0', '1.png')
203 | )
204 |
205 | def test_save_border(self):
206 | # Western hemisphere is border
207 | self.storage.save_border(x=0, y=0, z=1)
208 | self.storage.save_border(x=0, y=1, z=1)
209 | self.storage.save_border(x=0, y=1, z=2)
210 | self.assertEqual(set(recursive_listdir(self.outputdir)),
211 | set([
212 | '1/',
213 | '1/0/',
214 | '1/0/0.png',
215 | '1/0/1.png',
216 | '2/',
217 | '2/0/',
218 | '2/0/1.png',
219 | ]))
220 |
221 | # Is this a real file?
222 | self.assertFalse(
223 | os.path.islink(os.path.join(
224 | self.outputdir, '1/0/0.png'
225 | ))
226 | )
227 |
228 | # Does the symlinking work?
229 | self.assertEqual(
230 | os.readlink(os.path.join(
231 | self.outputdir, '1/0/1.png'
232 | )),
233 | '0.png'
234 | )
235 | self.assertEqual(
236 | os.readlink(os.path.join(
237 | self.outputdir, '2/0/1.png'
238 | )),
239 | os.path.join(os.path.pardir, os.path.pardir, '1', '0', '0.png')
240 | )
241 |
242 |
243 | class TestMbtilesStorage(unittest.TestCase):
244 | def setUp(self):
245 | self.tempfile = NamedTemporaryFile()
246 | # Use the PngRenderer because we want to know that callback
247 | # works properly.
248 | self.renderer = PngRenderer(png8=False, optimize=False)
249 | self.metadata = dict(
250 | name='transparent',
251 | type=Metadata.latest().TYPES.BASELAYER,
252 | version='1.0.0',
253 | description='Transparent World 2012',
254 | format=Metadata.latest().FORMATS.PNG,
255 | )
256 | self.storage = MbtilesStorage.create(renderer=self.renderer,
257 | filename=':memory:',
258 | metadata=self.metadata)
259 |
260 | def tearDown(self):
261 | try:
262 | self.tempfile.close()
263 | except OSError as e:
264 | if e.errno != errno.ENOENT:
265 | raise
266 |
267 | def test_create(self):
268 | # Make a new file if it doesn't exist
269 | os.remove(self.tempfile.name)
270 | storage = MbtilesStorage.create(renderer=self.renderer,
271 | filename=self.tempfile.name,
272 | metadata=self.metadata)
273 | self.assertEqual(storage.filename, self.tempfile.name)
274 | self.assertEqual(storage.mbtiles.metadata, self.metadata)
275 | self.assertTrue(os.path.isfile(self.tempfile.name))
276 |
277 | # Make a duplicate file
278 | MbtilesStorage.create(renderer=self.renderer,
279 | filename=self.tempfile.name,
280 | metadata=self.metadata)
281 | self.assertEqual(storage.filename, self.tempfile.name)
282 | self.assertTrue(os.path.isfile(self.tempfile.name))
283 |
284 | def test_get_hash(self):
285 | image = VImageAdapter.new_rgba(width=1, height=1,
286 | ink=rgba(r=0, g=0, b=0, a=0))
287 | self.assertEqual(self.storage.get_hash(image=image),
288 | int('f1d3ff8443297732862df21dc4e57262', base=16))
289 |
290 | def test_save(self):
291 | # We must create this on disk
292 | self.storage = MbtilesStorage.create(renderer=self.renderer,
293 | filename=self.tempfile.name,
294 | metadata=self.metadata)
295 |
296 | # Transparent 1×1 image
297 | image = VImageAdapter.new_rgba(width=1, height=1,
298 | ink=rgba(r=0, g=0, b=0, a=0))
299 |
300 | # Save it twice, assuming that MBTiles will deduplicate
301 | self.storage.save(x=0, y=1, z=2, image=image)
302 | self.storage.save(x=1, y=0, z=2, image=image)
303 |
304 | # Assert that things were saved properly
305 | self.assertEqual(
306 | [(z, x, y, intmd5(data))
307 | for z, x, y, data in self.storage.mbtiles.all()],
308 | [
309 | (2, 0, 1, 89446660811628514001822794642426893173),
310 | (2, 1, 0, 89446660811628514001822794642426893173),
311 | ]
312 | )
313 |
314 | # Close the existing database.
315 | self.storage.mbtiles.close()
316 |
317 | # Re-open the created file
318 | storage = MbtilesStorage(renderer=self.renderer,
319 | filename=self.tempfile.name)
320 |
321 | # Read out of the backend
322 | self.assertEqual(
323 | [(z, x, y, intmd5(data))
324 | for z, x, y, data in storage.mbtiles.all()],
325 | [
326 | (2, 0, 1, 89446660811628514001822794642426893173),
327 | (2, 1, 0, 89446660811628514001822794642426893173),
328 | ]
329 | )
330 |
331 | def test_save_border(self):
332 | # Western hemisphere is border
333 | self.storage.save_border(x=0, y=0, z=1)
334 | self.storage.save_border(x=0, y=1, z=1)
335 |
336 | # Assert that things were saved properly
337 | self.assertEqual(
338 | [(z, x, y, intmd5(data))
339 | for z, x, y, data in self.storage.mbtiles.all()],
340 | [
341 | (1, 0, 0, 182760986852492185208562855341207287999),
342 | (1, 0, 1, 182760986852492185208562855341207287999),
343 | ]
344 | )
345 |
--------------------------------------------------------------------------------
/tests/test_types.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from __future__ import (absolute_import, division, print_function,
4 | unicode_literals)
5 |
6 | import unittest
7 |
8 | from gdal2mbtiles.gd_types import rgba
9 |
10 |
11 | class TestRgba(unittest.TestCase):
12 | def test_create(self):
13 | self.assertEqual(rgba(0, 0, 0),
14 | rgba(0, 0, 0, 255))
15 |
16 | def test_webcolor_named(self):
17 | self.assertEqual(rgba.webcolor('red'),
18 | rgba(255, 0, 0, 255))
19 | self.assertEqual(rgba.webcolor('RED'),
20 | rgba(255, 0, 0, 255))
21 |
22 | # http://en.wikipedia.org/wiki/The_Colour_of_Magic
23 | self.assertRaises(ValueError, rgba.webcolor, 'octarine')
24 |
25 | def test_webcolor_hex(self):
26 | # Abbreviated
27 | self.assertEqual(rgba.webcolor('#0f0'),
28 | rgba(0, 255, 0, 255))
29 | self.assertEqual(rgba.webcolor('#0F0'),
30 | rgba(0, 255, 0, 255))
31 |
32 | # Full
33 | self.assertEqual(rgba.webcolor('#0000ff'),
34 | rgba(0, 0, 255, 255))
35 | self.assertEqual(rgba.webcolor('#0000FF'),
36 | rgba(0, 0, 255, 255))
37 |
38 | # No hash in front
39 | self.assertRaises(ValueError, rgba.webcolor, '0000ff')
40 |
--------------------------------------------------------------------------------
/tests/upsampling.tif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ecometrica/gdal2mbtiles/df06cc5c226f5684a0bf98c87f8639b07020b2e1/tests/upsampling.tif
--------------------------------------------------------------------------------
/tox.ini:
--------------------------------------------------------------------------------
1 | [tox]
2 | envlist = py3, pinned-vips
3 | skip_missing_interpreters = true
4 |
5 | [testenv]
6 | whitelist_externals = env
7 |
8 | setenv =
9 | PYTHONPATH={toxinidir}:{env:PYTHONPATH:}
10 | TOX_ENVBINDIR={envbindir}
11 | LIBRARY_PATH=/usr/local/lib
12 | CPATH=/usr/local/include
13 | CPLUS_INCLUDE_PATH=/usr/include/gdal
14 | C_INCLUDE_PATH=/usr/include/gdal
15 |
16 | passenv =
17 | GDAL_VERSION
18 |
19 | deps =
20 | pyvips
21 |
22 | extras =
23 | tests
24 |
25 | install_command =
26 | pip install {opts} {packages}
27 |
28 | commands_pre =
29 | pip install \
30 | --global-option=build_ext \
31 | --global-option=--gdal-config=/usr/bin/gdal-config \
32 | --global-option=-I/usr/include/gdal GDAL=={env:GDAL_VERSION}
33 |
34 | commands =
35 | pytest {posargs}
36 |
37 | [testenv:pinned-vips]
38 | deps =
39 | pyvips==2.1.8
40 |
--------------------------------------------------------------------------------