├── .gitignore ├── .travis.yml ├── CHANGES ├── LICENSE ├── MANIFEST.in ├── README.rst ├── landez ├── __init__.py ├── cache.py ├── data_test │ ├── data │ │ ├── world_merc.dbf │ │ ├── world_merc.index │ │ ├── world_merc.prj │ │ ├── world_merc.shp │ │ ├── world_merc.shx │ │ └── world_merc_license.txt │ └── stylesheet.xml ├── filters.py ├── proj.py ├── sources.py ├── tests.py ├── tiles.py └── util.py ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | #python 2 | *.pyc 3 | #OSX 4 | .DS_Store 5 | 6 | # Thumbnails 7 | ._* 8 | 9 | # Files that might appear on external disk 10 | .Spotlight-V100 11 | .Trashes 12 | 13 | /build/ 14 | /dist/ 15 | /landez.egg-info/ 16 | 17 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | sudo: required 3 | language: python 4 | python: 5 | - 2.7 6 | - 3.5 7 | 8 | before_install: 9 | - deactivate 10 | - sudo apt-get install -y python-software-properties 11 | - if [[ $TRAVIS_PYTHON_VERSION == 3.5 ]]; then virtualenv --system-site-packages venv -p python3; fi 12 | - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then virtualenv --system-site-packages venv; fi 13 | - source venv/bin/activate 14 | - pip install -r requirements.txt 15 | - if [[ $TRAVIS_PYTHON_VERSION == 3.5 ]]; then 16 | sudo apt-get install python3-mapnik; 17 | fi 18 | - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then 19 | sudo apt-add-repository --yes ppa:mapnik/v2.2.0; 20 | sudo apt-get update -qq; 21 | sudo apt-get install -y mapnik-utils python-mapnik; 22 | fi 23 | - python --version 24 | install: 25 | - python setup.py develop 26 | before_script: 27 | - pip install nose 28 | - pip install coverage 29 | script: 30 | - nosetests --with-coverage --cover-package=landez 31 | after_success: 32 | - pip install coveralls 33 | - coveralls 34 | -------------------------------------------------------------------------------- /CHANGES: -------------------------------------------------------------------------------- 1 | ========= 2 | CHANGELOG 3 | ========= 4 | 5 | 2.5.1.dev0 6 | ================== 7 | 8 | * 9 | 10 | 11 | 2.5.0 (2019-04-16) 12 | ================== 13 | 14 | * Add support of Python 3. 15 | 16 | 17 | 2.4.1 (2019-03-13) 18 | ================== 19 | 20 | * Do not try to get tiles again when tiles doesn't exist. 21 | 22 | 23 | 2.4.0 (2017-03-02) 24 | ================== 25 | 26 | * Do not crash when overlay tile data is not a valid image 27 | * Correctly generate metadata for zoom levels 28 | * Add support for tms mbtiles 29 | * Correct tile box calculation for case when floating point value is an integer 30 | * Correctly generate metadata for zoom levels 31 | * Use the full path to construct the cache directory, as otherwise different 32 | tiles sets on the same server are considered to be the same one 33 | * Added a name metadata to prevent Maptiler crash 34 | 35 | 36 | 2.3.0 (2014-11-18) 37 | ================== 38 | 39 | * Add headers to WMS sources if specified (thanks @sempixel!) 40 | 41 | 42 | 2.2.0 (2014-09-22) 43 | ================== 44 | 45 | * Add delay between tiles downloads retries (by @kiorky) 46 | * Add option to ignore errors during MBTiles creation (e.g. download errors) 47 | 48 | 49 | 2.1.1 (2013-08-27) 50 | ================== 51 | 52 | * Do not hard-code ``grid();`` JSONP callback. 53 | 54 | 2.1.0 (2013-08-27) 55 | ================== 56 | 57 | * Add TMS support (ebrehault) 58 | * Add default subdomains argument for TileSource 59 | * Add ability to set HTTP headers for tiles 60 | * Fix file corruption on Windows (by @osuchw) 61 | 62 | 2.0.3 (2013-05-03) 63 | ================== 64 | 65 | * Fix Mapnik signature on render() 66 | 67 | 2.0.2 (2012-06-21) 68 | ================== 69 | 70 | * Prevent the whole image to be converted to grayscale 71 | * Explicitly check http status code at tiles download 72 | 73 | 2.0.1 (2012-05-29) 74 | ================== 75 | 76 | * Fix infinite loop on blending layers 77 | 78 | 2.0.0 (2012-05-25) 79 | ================== 80 | 81 | * Rework cache mechanism 82 | * Jpeg tiles support (#14) 83 | * Remove use of temporary files 84 | * Image post-processing (#11) 85 | 86 | 2.0.0-alpha (2012-05-23) 87 | ======================== 88 | 89 | * Refactoring of whole stack 90 | 91 | 1.8.2 (2012-03-27) 92 | ================== 93 | 94 | * Fix Mapnik rendering 95 | 96 | 1.8.1 (2012-02-24) 97 | ================== 98 | 99 | * Fix MBTiles cache cleaning 100 | 101 | 1.8 (2012-02-24) 102 | ================ 103 | 104 | * WMS support 105 | * Tiles compositing 106 | 107 | 1.7 (2012-02-17) 108 | ================ 109 | 110 | * Catch Sqlite exceptions 111 | 112 | 1.6 (2012-02-08) 113 | ================ 114 | 115 | * UTF-Grid support for MBTiles files 116 | 117 | 1.5 (2011-12-07) 118 | ================ 119 | 120 | * Subdomain support for tiles servers 121 | * Low level tiles manipulation 122 | * Use i18n 123 | 124 | 1.4 (2011-10-17) 125 | ================ 126 | 127 | * Remove extra logging message of mbutil 128 | 129 | 1.3 (2011-09-23) 130 | ================ 131 | 132 | * Export set of tiles into single image 133 | 134 | 1.2 (2011-06-21) 135 | ================ 136 | 137 | * Raise exception if no tiles in coverages 138 | 139 | 1.1 (2012-04-18) 140 | ================ 141 | 142 | * Move internals to landez module 143 | * Split projection into separate module 144 | 145 | 1.0 (2011-04-18) 146 | ================ 147 | 148 | * Initial working version 149 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst CHANGES LICENSE 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | *Landez* manipulates tiles, builds MBTiles, does tiles compositing and arrange tiles together into single images. 2 | 3 | Tiles can either be obtained from a remote tile service URL, from a local Mapnik stylesheet, 4 | a WMS server or from MBTiles files. 5 | 6 | For building MBTiles, Landez embeds *mbutil* from Mapbox https://github.com/mapbox/mbutil at the final stage. 7 | The land covered is specified using a list of bounding boxes and zoom levels. 8 | 9 | 10 | .. image:: https://pypip.in/v/landez/badge.png 11 | :target: https://pypi.python.org/pypi/landez 12 | 13 | .. image:: https://pypip.in/d/landez/badge.png 14 | :target: https://pypi.python.org/pypi/landez 15 | 16 | .. image:: https://travis-ci.org/makinacorpus/landez.png 17 | :target: https://travis-ci.org/makinacorpus/landez 18 | 19 | .. image:: https://coveralls.io/repos/makinacorpus/landez/badge.png 20 | :target: https://coveralls.io/r/makinacorpus/landez 21 | 22 | 23 | ======= 24 | INSTALL 25 | ======= 26 | 27 | *Landez* is pure python and has no external dependency. :: 28 | 29 | sudo easy_install landez 30 | 31 | However, it requires `mapnik` if the tiles are rendered locally. :: 32 | 33 | sudo aptitude install python-mapnik 34 | 35 | And `PIL` to blend tiles together or export arranged tiles into images. :: 36 | 37 | sudo aptitude install python-imaging 38 | 39 | ===== 40 | USAGE 41 | ===== 42 | 43 | Building MBTiles files 44 | ====================== 45 | 46 | Remote tiles 47 | ------------ 48 | 49 | Using a remote tile service (OpenStreetMap.org by default): 50 | :: 51 | 52 | import logging 53 | from landez import MBTilesBuilder 54 | 55 | logging.basicConfig(level=logging.DEBUG) 56 | 57 | mb = MBTilesBuilder(cache=False) 58 | mb.add_coverage(bbox=(-180.0, -90.0, 180.0, 90.0), 59 | zoomlevels=[0, 1]) 60 | mb.run() 61 | 62 | Please respect `Tile usage policies ` 63 | 64 | Local rendering 65 | --------------- 66 | 67 | Using mapnik to render tiles: 68 | 69 | :: 70 | 71 | import logging 72 | from landez import MBTilesBuilder 73 | 74 | logging.basicConfig(level=logging.DEBUG) 75 | 76 | mb = MBTilesBuilder(stylefile="yourstyle.xml", filepath="dest.mbtiles") 77 | mb.add_coverage(bbox=(-180.0, -90.0, 180.0, 90.0), 78 | zoomlevels=[0, 1]) 79 | mb.run() 80 | 81 | 82 | And with UTFGrids: 83 | 84 | :: 85 | 86 | import logging 87 | from landez import MBTilesBuilder 88 | 89 | logging.basicConfig(level=logging.DEBUG) 90 | 91 | mb = MBTilesBuilder(stylefile="yourstyle.xml", 92 | grid_fields=["field1", "field2", "field3", ...] , 93 | filepath="dest.mbtiles") 94 | mb.add_coverage(bbox=(-180, -90, 180, 90), 95 | zoomlevels=[0, 1, 2, 3]) 96 | mb.run() 97 | 98 | 99 | From an other MBTiles file 100 | -------------------------- 101 | :: 102 | 103 | import logging 104 | from landez import MBTilesBuilder 105 | 106 | logging.basicConfig(level=logging.DEBUG) 107 | 108 | mb = MBTilesBuilder(mbtiles_file="yourfile.mbtiles", filepath="dest.mbtiles") 109 | mb.add_coverage(bbox=(-180.0, -90.0, 180.0, 90.0), 110 | zoomlevels=[0, 1]) 111 | mb.run() 112 | 113 | 114 | From a WMS server 115 | ----------------- 116 | :: 117 | 118 | mb = MBTilesBuilder(wms_server="http://yourserver.com/geoserver/wms", 119 | wms_layers=["ign:departements"], 120 | wms_options=dict(format="image/png", 121 | transparent=True), 122 | filepath="dest.mbtiles") 123 | mb.add_coverage(bbox=([-0.9853,43.6435.1126,44.0639])) 124 | mb.run() 125 | 126 | 127 | 128 | Blend tiles together 129 | ==================== 130 | 131 | Merge multiple sources of tiles (URL, WMS, MBTiles, Mapnik stylesheet) together. *(requires python PIL)* 132 | 133 | For example, build a new MBTiles by blending tiles of a MBTiles on top of OpenStreetMap tiles : 134 | 135 | :: 136 | 137 | mb = MBTilesBuilder(filepath="merged.mbtiles") 138 | overlay = TilesManager(mbtiles_file="carto.mbtiles") 139 | mb.add_layer(overlay) 140 | mb.run() 141 | 142 | Or composite a WMS layer with OpenStreetMap using transparency (40%): 143 | 144 | :: 145 | 146 | mb = MBTilesBuilder(wms_server="http://yourserver.com/geoserver/wms", 147 | wms_layers=["img:orthophoto"]) 148 | overlay = TilesManager(remote=True) 149 | mb.add_layer(overlay, 0.4) 150 | mb.run() 151 | 152 | 153 | Export Images 154 | ============= 155 | 156 | Assemble and arrange tiles together into a single image. *(requires python PIL)* 157 | 158 | Specify tiles sources in the exact same way as for building MBTiles files. 159 | 160 | :: 161 | 162 | import logging 163 | from landez import ImageExporter 164 | 165 | logging.basicConfig(level=logging.DEBUG) 166 | 167 | ie = ImageExporter(mbtiles_file="yourfile.mbtiles") 168 | ie.export_image(bbox=(-180.0, -90.0, 180.0, 90.0), zoomlevel=3, imagepath="image.png") 169 | 170 | 171 | Add post-processing filters 172 | =========================== 173 | 174 | Convert map tiles to gray scale, more suitable for information overlay : 175 | 176 | :: 177 | 178 | from landez.filters import GrayScale 179 | 180 | ie = ImageExporter() 181 | ie.add_filter(GrayScale()) 182 | 183 | Replace a specific color by transparent pixels (i.e. color to alpha, *a-la-Gimp*) : 184 | 185 | :: 186 | 187 | from landez.filters import ColorToAlpha 188 | 189 | overlay = TileManager() 190 | overlay.add_filter(ColorToAlpha('#ffffff')) # white will be transparent 191 | 192 | ie = ImageExporter() 193 | ie.add_layer(overlay) 194 | ... 195 | 196 | 197 | Extract MBTiles content 198 | ======================= 199 | 200 | :: 201 | 202 | from landez.sources import MBTilesReader 203 | 204 | mbreader = MBTilesReader("yourfile.mbtiles") 205 | 206 | # Metadata 207 | print mbreader.metadata() 208 | 209 | # Zoom levels 210 | print mbreader.zoomlevels() 211 | 212 | # Image tile 213 | with open('tile.png', 'wb') as out: 214 | out.write(mbreader.tile(z, x, y)) 215 | 216 | # UTF-Grid tile 217 | print mbreader.grid(z, x, y, 'callback') 218 | 219 | 220 | 221 | Manipulate tiles 222 | ================ 223 | 224 | :: 225 | 226 | from landez import MBTilesBuilder 227 | 228 | # From a TMS tile server 229 | # tm = TilesManager(tiles_url="http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png") 230 | 231 | # From a MBTiles file 232 | tm = TilesManager(mbtiles_file="yourfile.mbtiles") 233 | 234 | tiles = tm.tileslist(bbox=(-180.0, -90.0, 180.0, 90.0), 235 | zoomlevels=[0, 1]) 236 | for tile in tiles: 237 | tilecontent = tm.tile(tile) # download, extract or take from cache 238 | ... 239 | 240 | Cache tiles are stored using TMS scheme by default (with ``y`` value flipped). It can be changed to WMTS (a.k.a ``xyz``) : 241 | 242 | :: 243 | 244 | tm = TilesManager(your_sources_options, cache=True, cache_scheme="wmts") 245 | 246 | 247 | Run tests 248 | ========= 249 | 250 | Run tests with nosetests (if you are working in a virtualenv, don't forget to install nose in it!): 251 | 252 | :: 253 | 254 | cd landez 255 | nosetests 256 | 257 | The Mapnik stylesheet for the test about grid content comes from 258 | 259 | 260 | ======= 261 | AUTHORS 262 | ======= 263 | 264 | * Mathieu Leplatre 265 | * Sergej Tatarincev 266 | * Éric Bréhault 267 | * Waldemar Osuch 268 | * Isabelle Vallet 269 | * Thanks to mbutil authors 270 | 271 | 272 | .. image:: http://depot.makina-corpus.org/public/logo.gif 273 | :target: http://www.makina-corpus.com 274 | 275 | ======= 276 | LICENSE 277 | ======= 278 | 279 | * Lesser GNU Public License 280 | -------------------------------------------------------------------------------- /landez/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | 4 | """ Default tiles URL """ 5 | DEFAULT_TILES_URL = "http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png" 6 | """ Default tiles subdomains """ 7 | DEFAULT_TILES_SUBDOMAINS = list("abc") 8 | """ Base temporary folder """ 9 | DEFAULT_TMP_DIR = os.path.join(tempfile.gettempdir(), 'landez') 10 | """ Default output MBTiles file """ 11 | DEFAULT_FILEPATH = os.path.join(os.getcwd(), "tiles.mbtiles") 12 | """ Default tile size in pixels (*useless* in remote rendering) """ 13 | DEFAULT_TILE_SIZE = 256 14 | """ Default tile format (mime-type) """ 15 | DEFAULT_TILE_FORMAT = 'image/png' 16 | DEFAULT_TILE_SCHEME = 'wmts' 17 | """ Number of retries for remove tiles downloading """ 18 | DOWNLOAD_RETRIES = 10 19 | """ Path to fonts for Mapnik rendering """ 20 | TRUETYPE_FONTS_PATH = '/usr/share/fonts/truetype/' 21 | 22 | from .tiles import * 23 | from .sources import * 24 | -------------------------------------------------------------------------------- /landez/cache.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import logging 4 | import shutil 5 | from gettext import gettext as _ 6 | from .util import flip_y 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | class Cache(object): 12 | def __init__(self, **kwargs): 13 | self.extension = kwargs.get('extension', '.png') 14 | self._scheme = 'tms' 15 | 16 | def tile_file(self, z_x_y): 17 | (z, x, y) = z_x_y 18 | tile_dir = os.path.join("%s" % z, "%s" % x) 19 | y = flip_y(y, z) 20 | tile_name = "%s%s" % (y, self.extension) 21 | return tile_dir, tile_name 22 | 23 | @property 24 | def scheme(self): 25 | return self._scheme 26 | 27 | def read(self, z_x_y): 28 | raise NotImplementedError 29 | 30 | def save(self, body, z_x_y): 31 | raise NotImplementedError 32 | 33 | def remove(self, z_x_y): 34 | raise NotImplementedError 35 | 36 | def clean(self): 37 | raise NotImplementedError 38 | 39 | 40 | class Dummy(Cache): 41 | def read(self, z_x_y): 42 | return None 43 | 44 | def save(self, body, z_x_y): 45 | pass 46 | 47 | def remove(self, z_x_y): 48 | pass 49 | 50 | def clean(self): 51 | pass 52 | 53 | 54 | class Disk(Cache): 55 | def __init__(self, basename, folder, **kwargs): 56 | super(Disk, self).__init__(**kwargs) 57 | self._basename = None 58 | self._basefolder = folder 59 | self.folder = folder 60 | self.basename = basename 61 | 62 | @property 63 | def basename(self): 64 | return self._basename 65 | 66 | @basename.setter 67 | def basename(self, basename): 68 | self._basename = basename 69 | subfolder = re.sub(r'[^a-z^A-Z^0-9^_]+', '', basename.replace("/","_").lower()) 70 | self.folder = os.path.join(self._basefolder, subfolder) 71 | 72 | @Cache.scheme.setter 73 | def scheme(self, scheme): 74 | assert scheme in ('wmts', 'xyz', 'tms'), "Unknown scheme %s" % scheme 75 | self._scheme = 'xyz' if (scheme == 'wmts') else scheme 76 | 77 | def tile_file(self, z_x_y): 78 | (z, x, y) = z_x_y 79 | tile_dir = os.path.join("%s" % z, "%s" % x) 80 | if (self.scheme != 'xyz'): 81 | y = flip_y(y, z) 82 | tile_name = "%s%s" % (y, self.extension) 83 | return tile_dir, tile_name 84 | 85 | def tile_fullpath(self, z_x_y): 86 | (z, x, y) = z_x_y 87 | tile_dir, tile_name = self.tile_file((z, x, y)) 88 | tile_abs_dir = os.path.join(self.folder, tile_dir) 89 | return os.path.join(tile_abs_dir, tile_name) 90 | 91 | def remove(self, z_x_y): 92 | (z, x, y) = z_x_y 93 | tile_abs_uri = self.tile_fullpath((z, x, y)) 94 | os.remove(tile_abs_uri) 95 | parent = os.path.dirname(tile_abs_uri) 96 | i = 0 97 | while i <= 3: # try to remove 3 levels (cache/z/x/) 98 | try: 99 | os.rmdir(parent) 100 | parent = os.path.dirname(parent) 101 | i += 1 102 | except OSError: 103 | break 104 | 105 | def read(self, z_x_y): 106 | (z, x, y) = z_x_y 107 | tile_abs_uri = self.tile_fullpath((z, x, y)) 108 | if os.path.exists(tile_abs_uri): 109 | logger.debug(_("Found %s") % tile_abs_uri) 110 | return open(tile_abs_uri, 'rb').read() 111 | return None 112 | 113 | def save(self, body, z_x_y): 114 | (z, x, y) = z_x_y 115 | tile_abs_uri = self.tile_fullpath((z, x, y)) 116 | tile_abs_dir = os.path.dirname(tile_abs_uri) 117 | if not os.path.isdir(tile_abs_dir): 118 | os.makedirs(tile_abs_dir) 119 | logger.debug(_("Save %s bytes to %s") % (len(body), tile_abs_uri)) 120 | open(tile_abs_uri, 'wb').write(body) 121 | 122 | def clean(self): 123 | logger.debug(_("Clean-up %s") % self.folder) 124 | try: 125 | shutil.rmtree(self.folder) 126 | except OSError: 127 | logger.warn(_("%s was missing or read-only.") % self.folder) 128 | -------------------------------------------------------------------------------- /landez/data_test/data/world_merc.dbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makinacorpus/landez/6e5c71ded6071158e7943df204cd7bd1ed623a30/landez/data_test/data/world_merc.dbf -------------------------------------------------------------------------------- /landez/data_test/data/world_merc.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makinacorpus/landez/6e5c71ded6071158e7943df204cd7bd1ed623a30/landez/data_test/data/world_merc.index -------------------------------------------------------------------------------- /landez/data_test/data/world_merc.prj: -------------------------------------------------------------------------------- 1 | PROJCS["Google Maps Global Mercator",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]],PROJECTION["Mercator_2SP"],PARAMETER["standard_parallel_1",0],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1]] -------------------------------------------------------------------------------- /landez/data_test/data/world_merc.shp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makinacorpus/landez/6e5c71ded6071158e7943df204cd7bd1ed623a30/landez/data_test/data/world_merc.shp -------------------------------------------------------------------------------- /landez/data_test/data/world_merc.shx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makinacorpus/landez/6e5c71ded6071158e7943df204cd7bd1ed623a30/landez/data_test/data/world_merc.shx -------------------------------------------------------------------------------- /landez/data_test/data/world_merc_license.txt: -------------------------------------------------------------------------------- 1 | world_merc 2 | ========== 3 | 4 | 'world_merc.shp' is a version of TM_WORLD_BORDERS_SIMPL-0.3.shp 5 | downloaded from http://thematicmapping.org/downloads/world_borders.php. 6 | 7 | Coodinates near 180 degress longitude were clipped to faciliate reprojection 8 | to Google mercator (EPSG:900913). 9 | 10 | Details from original readme are below: 11 | 12 | ------------- 13 | 14 | TM_WORLD_BORDERS-0.1.ZIP 15 | 16 | Provided by Bjorn Sandvik, thematicmapping.org 17 | 18 | Use this dataset with care, as several of the borders are disputed. 19 | 20 | The original shapefile (world_borders.zip, 3.2 MB) was downloaded from the Mapping Hacks website: 21 | http://www.mappinghacks.com/data/ 22 | 23 | The dataset was derived by Schuyler Erle from public domain sources. 24 | Sean Gilles did some clean up and made some enhancements. 25 | 26 | 27 | COLUMN TYPE DESCRIPTION 28 | 29 | Shape Polygon Country/area border as polygon(s) 30 | FIPS String(2) FIPS 10-4 Country Code 31 | ISO2 String(2) ISO 3166-1 Alpha-2 Country Code 32 | ISO3 String(3) ISO 3166-1 Alpha-3 Country Code 33 | UN Short Integer(3) ISO 3166-1 Numeric-3 Country Code 34 | NAME String(50) Name of country/area 35 | AREA Long Integer(7) Land area, FAO Statistics (2002) 36 | POP2005 Double(10,0) Population, World Polulation Prospects (2005) 37 | REGION Short Integer(3) Macro geographical (continental region), UN Statistics 38 | SUBREGION Short Integer(3) Geogrpahical sub-region, UN Statistics 39 | LON FLOAT (7,3) Longitude 40 | LAT FLOAT (6,3) Latitude 41 | 42 | 43 | CHANGELOG VERSION 0.3 - 30 July 2008 44 | 45 | - Corrected spelling mistake (United Arab Emirates) 46 | - Corrected population number for Japan 47 | - Adjusted long/lat values for India, Italy and United Kingdom 48 | 49 | 50 | CHANGELOG VERSION 0.2 - 1 April 2008 51 | 52 | - Made new ZIP archieves. No change in dataset. 53 | 54 | 55 | CHANGELOG VERSION 0.1 - 13 March 2008 56 | 57 | - Polygons representing each country were merged into one feature 58 | - ≈land Islands was extracted from Finland 59 | - Hong Kong was extracted from China 60 | - Holy See (Vatican City) was added 61 | - Gaza Strip and West Bank was merged into "Occupied Palestinean Territory" 62 | - Saint-Barthelemy was extracted from Netherlands Antilles 63 | - Saint-Martin (Frensh part) was extracted from Guadeloupe 64 | - Svalbard and Jan Mayen was merged into "Svalbard and Jan Mayen Islands" 65 | - Timor-Leste was extracted from Indonesia 66 | - Juan De Nova Island was merged with "French Southern & Antarctic Land" 67 | - Baker Island, Howland Island, Jarvis Island, Johnston Atoll, Midway Islands 68 | and Wake Island was merged into "United States Minor Outlying Islands" 69 | - Glorioso Islands, Parcel Islands, Spartly Islands was removed 70 | (almost uninhabited and missing ISO-3611-1 code) 71 | 72 | - Added ISO-3166-1 codes (alpha-2, alpha-3, numeric-3). Source: 73 | https://www.cia.gov/library/publications/the-world-factbook/appendix/appendix-d.html 74 | http://unstats.un.org/unsd/methods/m49/m49alpha.htm 75 | http://www.fysh.org/~katie/development/geography.txt 76 | - AREA column has been replaced with data from UNdata: 77 | Land area, 1000 hectares, 2002, FAO Statistics 78 | - POPULATION column (POP2005) has been replaced with data from UNdata: 79 | Population, 2005, Medium variant, World Population Prospects: The 2006 Revision 80 | - Added region and sub-region codes from UN Statistics Division. Source: 81 | http://unstats.un.org/unsd/methods/m49/m49regin.htm 82 | - Added LAT, LONG values for each country 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /landez/data_test/stylesheet.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 11 | 12 | 13 | style 14 | 15 | data/world_merc.shp 16 | shape 17 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /landez/filters.py: -------------------------------------------------------------------------------- 1 | class Filter(object): 2 | @property 3 | def basename(self): 4 | return self.__class__.__name__ 5 | 6 | def process(self, image): 7 | return image 8 | 9 | @classmethod 10 | def string2rgba(cls, colorstring): 11 | """ Convert #RRGGBBAA to an (R, G, B, A) tuple """ 12 | colorstring = colorstring.strip() 13 | if colorstring[0] == '#': 14 | colorstring = colorstring[1:] 15 | if len(colorstring) < 6: 16 | raise ValueError("input #%s is not in #RRGGBB format" % colorstring) 17 | r, g, b = colorstring[:2], colorstring[2:4], colorstring[4:6] 18 | a = 'ff' 19 | if len(colorstring) > 6: 20 | a = colorstring[6:8] 21 | r, g, b, a = [int(n, 16) for n in (r, g, b, a)] 22 | return (r, g, b, a) 23 | 24 | 25 | class GrayScale(Filter): 26 | def process(self, image): 27 | return image.convert('L') 28 | 29 | 30 | class ColorToAlpha(Filter): 31 | def __init__(self, color): 32 | self.color = color 33 | 34 | @property 35 | def basename(self): 36 | return super(ColorToAlpha, self).basename + self.color 37 | 38 | def process(self, image): 39 | # Code taken from Phatch - Photo Batch Processor 40 | # Copyright (C) 2007-2010 www.stani.be 41 | 42 | from PIL import Image, ImageMath 43 | 44 | def difference1(source, color): 45 | """When source is bigger than color""" 46 | return (source - color) / (255.0 - color) 47 | 48 | def difference2(source, color): 49 | """When color is bigger than source""" 50 | return (color - source) / color 51 | 52 | def color_to_alpha(image, color=None): 53 | image = image.convert('RGBA') 54 | 55 | color = map(float, Filter.string2rgba(self.color)) 56 | img_bands = [band.convert("F") for band in image.split()] 57 | 58 | # Find the maximum difference rate between source and color. I had to use two 59 | # difference functions because ImageMath.eval only evaluates the expression 60 | # once. 61 | alpha = ImageMath.eval( 62 | """float( 63 | max( 64 | max( 65 | max( 66 | difference1(red_band, cred_band), 67 | difference1(green_band, cgreen_band) 68 | ), 69 | difference1(blue_band, cblue_band) 70 | ), 71 | max( 72 | max( 73 | difference2(red_band, cred_band), 74 | difference2(green_band, cgreen_band) 75 | ), 76 | difference2(blue_band, cblue_band) 77 | ) 78 | ) 79 | )""", 80 | difference1=difference1, 81 | difference2=difference2, 82 | red_band = img_bands[0], 83 | green_band = img_bands[1], 84 | blue_band = img_bands[2], 85 | cred_band = color[0], 86 | cgreen_band = color[1], 87 | cblue_band = color[2] 88 | ) 89 | # Calculate the new image colors after the removal of the selected color 90 | new_bands = [ 91 | ImageMath.eval( 92 | "convert((image - color) / alpha + color, 'L')", 93 | image = img_bands[i], 94 | color = color[i], 95 | alpha = alpha 96 | ) 97 | for i in xrange(3) 98 | ] 99 | # Add the new alpha band 100 | new_bands.append(ImageMath.eval( 101 | "convert(alpha_band * alpha, 'L')", 102 | alpha = alpha, 103 | alpha_band = img_bands[3] 104 | )) 105 | return Image.merge('RGBA', new_bands) 106 | 107 | return color_to_alpha(image, self.color) 108 | -------------------------------------------------------------------------------- /landez/proj.py: -------------------------------------------------------------------------------- 1 | from math import pi, sin, log, exp, atan, tan, ceil 2 | from gettext import gettext as _ 3 | from . import DEFAULT_TILE_SIZE 4 | 5 | DEG_TO_RAD = pi/180 6 | RAD_TO_DEG = 180/pi 7 | MAX_LATITUDE = 85.0511287798 8 | EARTH_RADIUS = 6378137 9 | 10 | 11 | def minmax (a,b,c): 12 | a = max(a,b) 13 | a = min(a,c) 14 | return a 15 | 16 | 17 | class InvalidCoverageError(Exception): 18 | """ Raised when coverage bounds are invalid """ 19 | pass 20 | 21 | 22 | class GoogleProjection(object): 23 | 24 | NAME = 'EPSG:3857' 25 | 26 | """ 27 | Transform Lon/Lat to Pixel within tiles 28 | Originally written by OSM team : http://svn.openstreetmap.org/applications/rendering/mapnik/generate_tiles.py 29 | """ 30 | def __init__(self, tilesize=DEFAULT_TILE_SIZE, levels = [0], scheme='wmts'): 31 | if not levels: 32 | raise InvalidCoverageError(_("Wrong zoom levels.")) 33 | self.Bc = [] 34 | self.Cc = [] 35 | self.zc = [] 36 | self.Ac = [] 37 | self.levels = levels 38 | self.maxlevel = max(levels) + 1 39 | self.tilesize = tilesize 40 | self.scheme = scheme 41 | c = tilesize 42 | for d in range(self.maxlevel): 43 | e = c/2; 44 | self.Bc.append(c/360.0) 45 | self.Cc.append(c/(2 * pi)) 46 | self.zc.append((e,e)) 47 | self.Ac.append(c) 48 | c *= 2 49 | 50 | def project_pixels(self,ll,zoom): 51 | d = self.zc[zoom] 52 | e = round(d[0] + ll[0] * self.Bc[zoom]) 53 | f = minmax(sin(DEG_TO_RAD * ll[1]),-0.9999,0.9999) 54 | g = round(d[1] + 0.5*log((1+f)/(1-f))*-self.Cc[zoom]) 55 | return (e,g) 56 | 57 | def unproject_pixels(self,px,zoom): 58 | e = self.zc[zoom] 59 | f = (px[0] - e[0])/self.Bc[zoom] 60 | g = (px[1] - e[1])/-self.Cc[zoom] 61 | h = RAD_TO_DEG * ( 2 * atan(exp(g)) - 0.5 * pi) 62 | if self.scheme == 'tms': 63 | h = - h 64 | return (f,h) 65 | 66 | def tile_at(self, zoom, position): 67 | """ 68 | Returns a tuple of (z, x, y) 69 | """ 70 | x, y = self.project_pixels(position, zoom) 71 | return (zoom, int(x/self.tilesize), int(y/self.tilesize)) 72 | 73 | def tile_bbox(self, z_x_y): 74 | """ 75 | Returns the WGS84 bbox of the specified tile 76 | """ 77 | (z, x, y) = z_x_y 78 | topleft = (x * self.tilesize, (y + 1) * self.tilesize) 79 | bottomright = ((x + 1) * self.tilesize, y * self.tilesize) 80 | nw = self.unproject_pixels(topleft, z) 81 | se = self.unproject_pixels(bottomright, z) 82 | return nw + se 83 | 84 | def project(self, lng_lat): 85 | """ 86 | Returns the coordinates in meters from WGS84 87 | """ 88 | (lng, lat) = lng_lat 89 | x = lng * DEG_TO_RAD 90 | lat = max(min(MAX_LATITUDE, lat), -MAX_LATITUDE) 91 | y = lat * DEG_TO_RAD 92 | y = log(tan((pi / 4) + (y / 2))) 93 | return (x*EARTH_RADIUS, y*EARTH_RADIUS) 94 | 95 | def unproject(self, x_y): 96 | """ 97 | Returns the coordinates from position in meters 98 | """ 99 | (x, y) = x_y 100 | lng = x/EARTH_RADIUS * RAD_TO_DEG 101 | lat = 2 * atan(exp(y/EARTH_RADIUS)) - pi/2 * RAD_TO_DEG 102 | return (lng, lat) 103 | 104 | def tileslist(self, bbox): 105 | if len(bbox) != 4: 106 | raise InvalidCoverageError(_("Wrong format of bounding box.")) 107 | xmin, ymin, xmax, ymax = bbox 108 | if abs(xmin) > 180 or abs(xmax) > 180 or \ 109 | abs(ymin) > 90 or abs(ymax) > 90: 110 | raise InvalidCoverageError(_("Some coordinates exceed [-180,+180], [-90, 90].")) 111 | 112 | if xmin >= xmax or ymin >= ymax: 113 | raise InvalidCoverageError(_("Bounding box format is (xmin, ymin, xmax, ymax)")) 114 | 115 | ll0 = (xmin, ymax) # left top 116 | ll1 = (xmax, ymin) # right bottom 117 | 118 | l = [] 119 | for z in self.levels: 120 | px0 = self.project_pixels(ll0,z) 121 | px1 = self.project_pixels(ll1,z) 122 | 123 | for x in range(int(px0[0]/self.tilesize), 124 | int(ceil(px1[0]/self.tilesize))): 125 | if (x < 0) or (x >= 2**z): 126 | continue 127 | for y in range(int(px0[1]/self.tilesize), 128 | int(ceil(px1[1]/self.tilesize))): 129 | if (y < 0) or (y >= 2**z): 130 | continue 131 | if self.scheme == 'tms': 132 | y = ((2**z-1) - y) 133 | l.append((z, x, y)) 134 | return l 135 | -------------------------------------------------------------------------------- /landez/sources.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import zlib 4 | import sqlite3 5 | import logging 6 | import json 7 | from gettext import gettext as _ 8 | from pkg_resources import parse_version 9 | import requests 10 | try: 11 | from urllib.parse import urlparse, urlencode 12 | from urllib.request import urlopen, Request 13 | except ImportError: 14 | from urlparse import urlparse 15 | from urllib import urlencode 16 | from urllib2 import urlopen, Request 17 | from tempfile import NamedTemporaryFile 18 | from .util import flip_y 19 | 20 | 21 | has_mapnik = False 22 | try: 23 | import mapnik 24 | has_mapnik = True 25 | except ImportError: 26 | pass 27 | 28 | 29 | from . import DEFAULT_TILE_FORMAT, DEFAULT_TILE_SIZE, DEFAULT_TILE_SCHEME, DOWNLOAD_RETRIES 30 | from .proj import GoogleProjection 31 | 32 | 33 | logger = logging.getLogger(__name__) 34 | 35 | 36 | class ExtractionError(Exception): 37 | """ Raised when extraction of tiles from specified MBTiles has failed """ 38 | pass 39 | 40 | 41 | class InvalidFormatError(Exception): 42 | """ Raised when reading of MBTiles content has failed """ 43 | pass 44 | 45 | 46 | class DownloadError(Exception): 47 | """ Raised when download at tiles URL fails DOWNLOAD_RETRIES times """ 48 | pass 49 | 50 | 51 | class TileSource(object): 52 | def __init__(self, tilesize=None): 53 | if tilesize is None: 54 | tilesize = DEFAULT_TILE_SIZE 55 | self.tilesize = tilesize 56 | self.basename = '' 57 | 58 | def tile(self, z, x, y): 59 | raise NotImplementedError 60 | 61 | def metadata(self): 62 | return dict() 63 | 64 | 65 | class MBTilesReader(TileSource): 66 | def __init__(self, filename, tilesize=None): 67 | super(MBTilesReader, self).__init__(tilesize) 68 | self.filename = filename 69 | self.basename = os.path.basename(self.filename) 70 | self._con = None 71 | self._cur = None 72 | 73 | def _query(self, sql, *args): 74 | """ Executes the specified `sql` query and returns the cursor """ 75 | if not self._con: 76 | logger.debug(_("Open MBTiles file '%s'") % self.filename) 77 | self._con = sqlite3.connect(self.filename) 78 | self._cur = self._con.cursor() 79 | sql = ' '.join(sql.split()) 80 | logger.debug(_("Execute query '%s' %s") % (sql, args)) 81 | try: 82 | self._cur.execute(sql, *args) 83 | except (sqlite3.OperationalError, sqlite3.DatabaseError)as e: 84 | raise InvalidFormatError(_("%s while reading %s") % (e, self.filename)) 85 | return self._cur 86 | 87 | def metadata(self): 88 | rows = self._query('SELECT name, value FROM metadata') 89 | rows = [(row[0], row[1]) for row in rows] 90 | return dict(rows) 91 | 92 | def zoomlevels(self): 93 | rows = self._query('SELECT DISTINCT(zoom_level) FROM tiles ORDER BY zoom_level') 94 | return [int(row[0]) for row in rows] 95 | 96 | def tile(self, z, x, y): 97 | logger.debug(_("Extract tile %s") % ((z, x, y),)) 98 | tms_y = flip_y(int(y), int(z)) 99 | rows = self._query('''SELECT tile_data FROM tiles 100 | WHERE zoom_level=? AND tile_column=? AND tile_row=?;''', (z, x, tms_y)) 101 | t = rows.fetchone() 102 | if not t: 103 | raise ExtractionError(_("Could not extract tile %s from %s") % ((z, x, y), self.filename)) 104 | return t[0] 105 | 106 | def grid(self, z, x, y, callback=None): 107 | tms_y = flip_y(int(y), int(z)) 108 | rows = self._query('''SELECT grid FROM grids 109 | WHERE zoom_level=? AND tile_column=? AND tile_row=?;''', (z, x, tms_y)) 110 | t = rows.fetchone() 111 | if not t: 112 | raise ExtractionError(_("Could not extract grid %s from %s") % ((z, x, y), self.filename)) 113 | grid_json = json.loads(zlib.decompress(t[0])) 114 | 115 | rows = self._query('''SELECT key_name, key_json FROM grid_data 116 | WHERE zoom_level=? AND tile_column=? AND tile_row=?;''', (z, x, tms_y)) 117 | # join up with the grid 'data' which is in pieces when stored in mbtiles file 118 | grid_json['data'] = {} 119 | grid_data = rows.fetchone() 120 | while grid_data: 121 | grid_json['data'][grid_data[0]] = json.loads(grid_data[1]) 122 | grid_data = rows.fetchone() 123 | serialized = json.dumps(grid_json) 124 | if callback is not None: 125 | return '%s(%s);' % (callback, serialized) 126 | return serialized 127 | 128 | def find_coverage(self, zoom): 129 | """ 130 | Returns the bounding box (minx, miny, maxx, maxy) of an adjacent 131 | group of tiles at this zoom level. 132 | """ 133 | # Find a group of adjacent available tiles at this zoom level 134 | rows = self._query('''SELECT tile_column, tile_row FROM tiles 135 | WHERE zoom_level=? 136 | ORDER BY tile_column, tile_row;''', (zoom,)) 137 | t = rows.fetchone() 138 | xmin, ymin = t 139 | previous = t 140 | while t and t[0] - previous[0] <= 1: 141 | # adjacent, go on 142 | previous = t 143 | t = rows.fetchone() 144 | xmax, ymax = previous 145 | # Transform (xmin, ymin) (xmax, ymax) to pixels 146 | S = self.tilesize 147 | bottomleft = (xmin * S, (ymax + 1) * S) 148 | topright = ((xmax + 1) * S, ymin * S) 149 | # Convert center to (lon, lat) 150 | proj = GoogleProjection(S, [zoom]) # WGS84 151 | return proj.unproject_pixels(bottomleft, zoom) + proj.unproject_pixels(topright, zoom) 152 | 153 | 154 | class TileDownloader(TileSource): 155 | def __init__(self, url, headers=None, subdomains=None, tilesize=None): 156 | super(TileDownloader, self).__init__(tilesize) 157 | self.tiles_url = url 158 | self.tiles_subdomains = subdomains or ['a', 'b', 'c'] 159 | parsed = urlparse(self.tiles_url) 160 | self.basename = parsed.netloc+parsed.path 161 | self.headers = headers or {} 162 | 163 | def tile(self, z, x, y): 164 | """ 165 | Download the specified tile from `tiles_url` 166 | """ 167 | logger.debug(_("Download tile %s") % ((z, x, y),)) 168 | # Render each keyword in URL ({s}, {x}, {y}, {z}, {size} ... ) 169 | size = self.tilesize 170 | s = self.tiles_subdomains[(x + y) % len(self.tiles_subdomains)]; 171 | try: 172 | url = self.tiles_url.format(**locals()) 173 | except KeyError as e: 174 | raise DownloadError(_("Unknown keyword %s in URL") % e) 175 | 176 | logger.debug(_("Retrieve tile at %s") % url) 177 | r = DOWNLOAD_RETRIES 178 | sleeptime = 1 179 | while r > 0: 180 | try: 181 | request = requests.get(url, headers=self.headers) 182 | if request.status_code == 200: 183 | return request.content 184 | raise DownloadError(_("Status code : %s, url : %s") % (request.status_code, url)) 185 | except requests.exceptions.ConnectionError as e: 186 | logger.debug(_("Download error, retry (%s left). (%s)") % (r, e)) 187 | r -= 1 188 | time.sleep(sleeptime) 189 | # progressivly sleep longer to wait for this tile 190 | if (sleeptime <= 10) and (r % 2 == 0): 191 | sleeptime += 1 # increase wait 192 | raise DownloadError(_("Cannot download URL %s") % url) 193 | 194 | 195 | class WMSReader(TileSource): 196 | def __init__(self, url, layers, headers=None, tilesize=None, **kwargs): 197 | super(WMSReader, self).__init__(tilesize) 198 | self.basename = '-'.join(layers) 199 | self.url = url 200 | self.headers = headers or {} 201 | self.wmsParams = dict( 202 | service='WMS', 203 | request='GetMap', 204 | version='1.1.1', 205 | styles='', 206 | format=DEFAULT_TILE_FORMAT, 207 | scheme=DEFAULT_TILE_SCHEME, 208 | transparent=False, 209 | layers=','.join(layers), 210 | width=self.tilesize, 211 | height=self.tilesize, 212 | ) 213 | self.wmsParams.update(**kwargs) 214 | projectionKey = 'srs' 215 | if parse_version(self.wmsParams['version']) >= parse_version('1.3'): 216 | projectionKey = 'crs' 217 | self.wmsParams[projectionKey] = GoogleProjection.NAME 218 | 219 | def tile(self, z, x, y): 220 | logger.debug(_("Request WMS tile %s") % ((z, x, y),)) 221 | proj = GoogleProjection(self.tilesize, [z]) 222 | bbox = proj.tile_bbox((z, x, y)) 223 | bbox = proj.project(bbox[:2]) + proj.project(bbox[2:]) 224 | bbox = ','.join(map(str, bbox)) 225 | # Build WMS request URL 226 | encodedparams = urlencode(self.wmsParams) 227 | url = "%s?%s" % (self.url, encodedparams) 228 | url += "&bbox=%s" % bbox # commas are not encoded 229 | try: 230 | logger.debug(_("Download '%s'") % url) 231 | request = requests.get(url, headers=self.headers) 232 | assert request.headers == self.wmsParams['format'], "Invalid WMS response type : %s" % self.headers 233 | return request.content 234 | except (AssertionError, IOError): 235 | raise ExtractionError 236 | 237 | 238 | class MapnikRenderer(TileSource): 239 | def __init__(self, stylefile, tilesize=None): 240 | super(MapnikRenderer, self).__init__(tilesize) 241 | assert has_mapnik, _("Cannot render tiles without mapnik !") 242 | self.stylefile = stylefile 243 | self.basename = os.path.basename(self.stylefile) 244 | self._mapnik = None 245 | self._prj = None 246 | 247 | def tile(self, z, x, y): 248 | """ 249 | Render the specified tile with Mapnik 250 | """ 251 | logger.debug(_("Render tile %s") % ((z, x, y),)) 252 | proj = GoogleProjection(self.tilesize, [z]) 253 | return self.render(proj.tile_bbox((z, x, y))) 254 | 255 | def _prepare_rendering(self, bbox, width=None, height=None): 256 | if not self._mapnik: 257 | self._mapnik = mapnik.Map(width, height) 258 | # Load style XML 259 | mapnik.load_map(self._mapnik, self.stylefile, True) 260 | # Obtain projection 261 | self._prj = mapnik.Projection(self._mapnik.srs) 262 | 263 | # Convert to map projection 264 | assert len(bbox) == 4, _("Provide a bounding box tuple (minx, miny, maxx, maxy)") 265 | c0 = self._prj.forward(mapnik.Coord(bbox[0], bbox[1])) 266 | c1 = self._prj.forward(mapnik.Coord(bbox[2], bbox[3])) 267 | 268 | # Bounding box for the tile 269 | bbox = mapnik.Box2d(c0.x, c0.y, c1.x, c1.y) 270 | self._mapnik.resize(width, height) 271 | self._mapnik.zoom_to_box(bbox) 272 | self._mapnik.buffer_size = 128 273 | 274 | def render(self, bbox, width=None, height=None): 275 | """ 276 | Render the specified tile with Mapnik 277 | """ 278 | width = width or self.tilesize 279 | height = height or self.tilesize 280 | self._prepare_rendering(bbox, width=width, height=height) 281 | 282 | # Render image with default Agg renderer 283 | tmpfile = NamedTemporaryFile(delete=False) 284 | im = mapnik.Image(width, height) 285 | mapnik.render(self._mapnik, im) 286 | im.save(tmpfile.name, 'png256') # TODO: mapnik output only to file? 287 | tmpfile.close() 288 | content = open(tmpfile.name, 'rb').read() 289 | os.unlink(tmpfile.name) 290 | return content 291 | 292 | def grid(self, z, x, y, fields, layer): 293 | """ 294 | Render the specified grid with Mapnik 295 | """ 296 | logger.debug(_("Render grid %s") % ((z, x, y),)) 297 | proj = GoogleProjection(self.tilesize, [z]) 298 | return self.render_grid(proj.tile_bbox((z, x, y)), fields, layer) 299 | 300 | def render_grid(self, bbox, grid_fields, layer, width=None, height=None): 301 | """ 302 | Render the specified grid with Mapnik 303 | """ 304 | width = width or self.tilesize 305 | height = height or self.tilesize 306 | self._prepare_rendering(bbox, width=width, height=height) 307 | 308 | grid = mapnik.Grid(width, height) 309 | mapnik.render_layer(self._mapnik, grid, layer=layer, fields=grid_fields) 310 | grid = grid.encode() 311 | return json.dumps(grid) 312 | 313 | -------------------------------------------------------------------------------- /landez/tests.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import mock 4 | import unittest 5 | import shutil 6 | import tempfile 7 | import json 8 | import sqlite3 9 | 10 | from .tiles import (TilesManager, MBTilesBuilder, ImageExporter, 11 | EmptyCoverageError, DownloadError) 12 | from .proj import InvalidCoverageError 13 | from .cache import Disk 14 | from .sources import MBTilesReader 15 | 16 | 17 | class TestTilesManager(unittest.TestCase): 18 | def test_format(self): 19 | mb = TilesManager() 20 | self.assertEqual(mb.tile_format, 'image/png') 21 | self.assertEqual(mb.cache.extension, '.png') 22 | # Format from WMS options 23 | mb = TilesManager(wms_server='dumb', wms_layers=['dumber'], 24 | wms_options={'format': 'image/jpeg'}) 25 | 26 | self.assertEqual(mb.tile_format, 'image/jpeg') 27 | self.assertEqual(mb.cache.extension, '.jpeg') 28 | # Format from URL extension 29 | mb = TilesManager(tiles_url='http://tileserver/{z}/{x}/{y}.jpg') 30 | self.assertEqual(mb.tile_format, 'image/jpeg') 31 | self.assertEqual(mb.cache.extension, '.jpeg') 32 | mb = TilesManager(tiles_url='http://tileserver/{z}/{x}/{y}.png') 33 | self.assertEqual(mb.tile_format, 'image/png') 34 | self.assertEqual(mb.cache.extension, '.png') 35 | # No extension in URL 36 | mb = TilesManager(tiles_url='http://tileserver/tiles/') 37 | self.assertEqual(mb.tile_format, 'image/png') 38 | self.assertEqual(mb.cache.extension, '.png') 39 | mb = TilesManager(tile_format='image/gif', 40 | tiles_url='http://tileserver/tiles/') 41 | self.assertEqual(mb.tile_format, 'image/gif') 42 | self.assertEqual(mb.cache.extension, '.gif') 43 | 44 | def test_tileslist(self): 45 | mb = TilesManager() 46 | # World at level 0 47 | l = mb.tileslist((-180.0, -90.0, 180.0, 90.0), [0]) 48 | self.assertEqual(l, [(0, 0, 0)]) 49 | # World at levels [0, 1] 50 | l = mb.tileslist((-180.0, -90.0, 180.0, 90.0), [0, 1]) 51 | self.assertEqual(l, [(0, 0, 0), 52 | (1, 0, 0), (1, 0, 1), (1, 1, 0), (1, 1, 1)]) 53 | # Incorrect bounds 54 | self.assertRaises(InvalidCoverageError, mb.tileslist, (-91.0, -180.0), [0]) 55 | self.assertRaises(InvalidCoverageError, mb.tileslist, (-90.0, -180.0, 180.0, 90.0), []) 56 | self.assertRaises(InvalidCoverageError, mb.tileslist, (-91.0, -180.0, 180.0, 90.0), [0]) 57 | self.assertRaises(InvalidCoverageError, mb.tileslist, (-91.0, -180.0, 181.0, 90.0), [0]) 58 | self.assertRaises(InvalidCoverageError, mb.tileslist, (-90.0, 180.0, 180.0, 90.0), [0]) 59 | self.assertRaises(InvalidCoverageError, mb.tileslist, (-30.0, -90.0, -50.0, 90.0), [0]) 60 | 61 | def test_tileslist_at_z1_x0_y0(self): 62 | mb = TilesManager() 63 | l = mb.tileslist((-180.0, 1, -1, 90.0), [1]) 64 | self.assertEqual(l, [(1, 0, 0)]) 65 | 66 | def test_tileslist_at_z1_x0_y0_tms(self): 67 | mb = TilesManager(tile_scheme='tms') 68 | l = mb.tileslist((-180.0, 1, -1, 90.0), [1]) 69 | 70 | self.assertEqual(l, [(1, 0, 1)]) 71 | 72 | def test_download_tile(self): 73 | mb = TilesManager(cache=False) 74 | tile = (1, 1, 1) 75 | # Unknown URL keyword 76 | mb = TilesManager(tiles_url="http://{X}.tile.openstreetmap.org/{z}/{x}/{y}.png") 77 | self.assertRaises(DownloadError, mb.tile, (1, 1, 1)) 78 | # With subdomain keyword 79 | mb = TilesManager(tiles_url="http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png") 80 | content = mb.tile(tile) 81 | self.assertTrue(content is not None) 82 | # No subdomain keyword 83 | mb = TilesManager(tiles_url="http://tile.openstreetmap.org/{z}/{x}/{y}.png") 84 | content = mb.tile(tile) 85 | self.assertTrue(content is not None) 86 | # Subdomain in available range 87 | mb = TilesManager(tiles_url="http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", 88 | tiles_subdomains=list("abc")) 89 | for y in range(3): 90 | content = mb.tile((10, 0, y)) 91 | self.assertTrue(content is not None) 92 | # Subdomain out of range 93 | mb = TilesManager(tiles_subdomains=list("abcz")) 94 | self.assertRaises(DownloadError, mb.tile, (10, 1, 2)) 95 | # Invalid URL 96 | mb = TilesManager(tiles_url="http://{s}.osm.com") 97 | self.assertRaises(DownloadError, mb.tile, (10, 1, 2)) 98 | 99 | 100 | class TestMBTilesBuilder(unittest.TestCase): 101 | temp_cache = os.path.join(tempfile.gettempdir(), 'landez/stileopenstreetmaporg_z_x_ypng') 102 | temp_dir = os.path.join(tempfile.gettempdir(), 'landez/tiles') 103 | 104 | def tearDown(self): 105 | try: 106 | shutil.rmtree(self.temp_cache) 107 | except OSError: 108 | pass 109 | try: 110 | shutil.rmtree(self.temp_dir) 111 | except OSError: 112 | pass 113 | try: 114 | os.remove('tiles.mbtiles') 115 | except OSError: 116 | pass 117 | 118 | def test_init(self): 119 | mb = MBTilesBuilder() 120 | self.assertEqual(mb.filepath, os.path.join(os.getcwd(), 'tiles.mbtiles')) 121 | self.assertEqual(mb.cache.folder, self.temp_cache) 122 | self.assertEqual(mb.tmp_dir, self.temp_dir) 123 | 124 | mb = MBTilesBuilder(filepath='/foo/bar/toto.mb') 125 | self.assertEqual(mb.cache.folder, self.temp_cache) 126 | self.assertEqual(mb.tmp_dir, os.path.join(tempfile.gettempdir(), 'landez/toto')) 127 | 128 | def test_run(self): 129 | mb = MBTilesBuilder(filepath='big.mbtiles') 130 | # Fails if no coverage 131 | self.assertRaises(EmptyCoverageError, mb.run, True) 132 | # Runs well from web tiles 133 | mb.add_coverage(bbox=(-180.0, -90.0, 180.0, 90.0), zoomlevels=[0, 1]) 134 | mb.run(force=True) 135 | self.assertEqual(mb.nbtiles, 5) 136 | # Read from other mbtiles 137 | mb2 = MBTilesBuilder(filepath='small.mbtiles', mbtiles_file=mb.filepath, cache=False) 138 | mb2.add_coverage(bbox=(-180.0, 1, -1, 90.0), zoomlevels=[1]) 139 | mb2.run(force=True) 140 | self.assertEqual(mb2.nbtiles, 1) 141 | os.remove('small.mbtiles') 142 | os.remove('big.mbtiles') 143 | 144 | def test_run_with_errors(self): 145 | if os.path.exists('tiles.mbtiles'): 146 | os.remove('tiles.mbtiles') 147 | mb = MBTilesBuilder(tiles_url='http://foo.bar') 148 | mb.add_coverage(bbox=(-180.0, -90.0, 180.0, 90.0), zoomlevels=[0, 1]) 149 | self.assertRaises(DownloadError, mb.run) 150 | mb = MBTilesBuilder(tiles_url='http://foo.bar', ignore_errors=True) 151 | mb.add_coverage(bbox=(-180.0, -90.0, 180.0, 90.0), zoomlevels=[0, 1]) 152 | mb.run() 153 | 154 | @mock.patch('requests.get') 155 | def test_run_jpeg(self, mock_get): 156 | mock_get.return_value.content = b'jpeg' 157 | mock_get.return_value.status_code = 200 158 | output = 'mq.mbtiles' 159 | mb = MBTilesBuilder(filepath=output, 160 | tiles_url='https://proxy-ign.openstreetmap.fr/94GjiyqD/bdortho/{z}/{x}/{y}.jpg') 161 | mb.add_coverage(bbox=(1.3, 43.5, 1.6, 43.7), zoomlevels=[10]) 162 | mb.run(force=True) 163 | self.assertEqual(mb.nbtiles, 4) 164 | # Check result 165 | reader = MBTilesReader(output) 166 | self.assertTrue(reader.metadata().get('format'), 'jpeg') 167 | os.remove(output) 168 | 169 | def test_clean_gather(self): 170 | mb = MBTilesBuilder() 171 | mb._clean_gather() 172 | self.assertEqual(mb.tmp_dir, self.temp_dir) 173 | self.assertFalse(os.path.exists(mb.tmp_dir)) 174 | mb._gather((1, 1, 1)) 175 | self.assertTrue(os.path.exists(mb.tmp_dir)) 176 | mb._clean_gather() 177 | self.assertFalse(os.path.exists(mb.tmp_dir)) 178 | 179 | def test_grid_content(self): 180 | here = os.path.abspath(os.path.dirname(__file__)) 181 | mb = MBTilesBuilder( 182 | stylefile=os.path.join(here, "data_test", "stylesheet.xml"), 183 | grid_fields=["NAME"], 184 | grid_layer=0, 185 | filepath='foo.mbtiles', 186 | cache=False 187 | ) 188 | 189 | mb.add_coverage(bbox=(-180, -90, 180, 90), zoomlevels=[2]) 190 | mb.run() 191 | 192 | mbtiles_path = os.path.join(os.getcwd(), 'foo.mbtiles') 193 | mbtiles = sqlite3.connect(mbtiles_path).cursor() 194 | grid = mbtiles.execute("SELECT grid FROM grids WHERE zoom_level=2 AND tile_column=1 AND tile_row=1") 195 | produced_data = json.loads(mb.grid((2, 1, 1)))['data']['39']['NAME'] 196 | expected_data = 'Costa Rica' 197 | os.remove('foo.mbtiles') 198 | self.assertEqual(produced_data, expected_data) 199 | 200 | def test_zoomlevels(self): 201 | mb = MBTilesBuilder() 202 | mb.add_coverage(bbox=(-180.0, -90.0, 180.0, 90.0), zoomlevels=[0, 1]) 203 | mb.add_coverage(bbox=(-180.0, -90.0, 180.0, 90.0), zoomlevels=[11, 12]) 204 | mb.add_coverage(bbox=(-180.0, -90.0, 180.0, 90.0), zoomlevels=[5]) 205 | self.assertEqual(mb.zoomlevels[0], 0) 206 | self.assertEqual(mb.zoomlevels[1], 1) 207 | self.assertEqual(mb.zoomlevels[2], 5) 208 | self.assertEqual(mb.zoomlevels[3], 11) 209 | self.assertEqual(mb.zoomlevels[4], 12) 210 | 211 | 212 | class TestImageExporter(unittest.TestCase): 213 | def test_gridtiles(self): 214 | mb = ImageExporter() 215 | # At zoom level 0 216 | grid = mb.grid_tiles((-180.0, -90.0, 180.0, 90.0), 0) 217 | self.assertEqual(grid, [[(0, 0)]]) 218 | # At zoom level 1 219 | grid = mb.grid_tiles((-180.0, -90.0, 180.0, 90.0), 1) 220 | self.assertEqual(grid, [[(0, 0), (1, 0)], 221 | [(0, 1), (1, 1)]]) 222 | 223 | def test_exportimage(self): 224 | from PIL import Image 225 | output = "image.png" 226 | ie = ImageExporter() 227 | ie.export_image((-180.0, -90.0, 180.0, 90.0), 2, output) 228 | i = Image.open(output) 229 | self.assertEqual((1024, 1024), i.size) 230 | os.remove(output) 231 | # Test from other mbtiles 232 | mb = MBTilesBuilder(filepath='toulouse.mbtiles') 233 | mb.add_coverage(bbox=(1.3, 43.5, 1.6, 43.7), zoomlevels=[12]) 234 | mb.run() 235 | ie = ImageExporter(mbtiles_file=mb.filepath) 236 | ie.export_image((1.3, 43.5, 1.6, 43.7), 12, output) 237 | os.remove('toulouse.mbtiles') 238 | i = Image.open(output) 239 | self.assertEqual((1280, 1024), i.size) 240 | os.remove(output) 241 | 242 | 243 | class TestCache(unittest.TestCase): 244 | temp_path = os.path.join(tempfile.gettempdir(), 'landez/stileopenstreetmaporg_z_x_ypng') 245 | 246 | def clean(self): 247 | try: 248 | shutil.rmtree(self.temp_path) 249 | except OSError: 250 | pass 251 | 252 | def test_folder(self): 253 | c = Disk('foo', '/tmp/') 254 | self.assertEqual(c.folder, '/tmp/foo') 255 | c.basename = 'bar' 256 | self.assertEqual(c.folder, '/tmp/bar') 257 | 258 | def test_remove(self): 259 | mb = TilesManager() 260 | mb.cache.save(b'toto', (1, 1, 1)) 261 | self.assertTrue(os.path.exists('/tmp/landez/stileopenstreetmaporg_z_x_ypng/1/1/0.png')) 262 | mb.cache.remove((1, 1, 1)) 263 | self.assertFalse(os.path.exists('/tmp/landez/stileopenstreetmaporg_z_x_ypng/1/1/0.png')) 264 | mb.cache.clean() 265 | self.assertFalse(os.path.exists(mb.cache.folder)) 266 | 267 | def test_clean(self): 268 | mb = TilesManager() 269 | self.assertEqual(mb.cache.folder, self.temp_path) 270 | # Missing dir 271 | self.assertFalse(os.path.exists(mb.cache.folder)) 272 | mb.cache.clean() 273 | # Empty dir 274 | os.makedirs(mb.cache.folder) 275 | self.assertTrue(os.path.exists(mb.cache.folder)) 276 | mb.cache.clean() 277 | self.assertFalse(os.path.exists(mb.cache.folder)) 278 | 279 | def test_cache_scheme_WMTS(self): 280 | tm = TilesManager(tiles_url="http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", cache=True, cache_scheme='wmts') 281 | self.assertEqual(tm.cache.scheme, 'xyz') 282 | 283 | def test_cache_with_bad_scheme(self): 284 | with self.assertRaises(AssertionError): 285 | TilesManager(tiles_url="http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", cache=True, cache_scheme='badscheme') 286 | 287 | def test_cache_is_stored_at_WMTS_format(self): 288 | tm = TilesManager(tiles_url="http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", cache=True, cache_scheme='wmts') 289 | tilecontent = tm.tile((12, 2064, 1495)) 290 | self.assertTrue(os.path.exists(os.path.join(self.temp_path, '12', '2064', '1495.png'))) 291 | 292 | def test_cache_is_stored_at_TMS_format(self): 293 | tm = TilesManager(tiles_url="http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", cache=True, cache_scheme='tms') 294 | tilecontent = tm.tile((12, 2064, 1495)) 295 | self.assertTrue(os.path.exists(os.path.join(self.temp_path, '12', '2064', '2600.png'))) 296 | 297 | def setUp(self): 298 | self.clean() 299 | 300 | def tearDown(self): 301 | self.clean() 302 | 303 | 304 | class TestLayers(unittest.TestCase): 305 | def test_cache_folder(self): 306 | mb = TilesManager(tiles_url='http://server') 307 | self.assertEqual(mb.cache.folder, '/tmp/landez/server') 308 | over = TilesManager(tiles_url='http://toto') 309 | self.assertEqual(over.cache.folder, '/tmp/landez/toto') 310 | mb.add_layer(over) 311 | self.assertEqual(mb.cache.folder, '/tmp/landez/servertoto10') 312 | mb.add_layer(over, 0.5) 313 | self.assertEqual(mb.cache.folder, '/tmp/landez/servertoto10toto05') 314 | 315 | 316 | class TestFilters(unittest.TestCase): 317 | def test_cache_folder(self): 318 | from .filters import ColorToAlpha 319 | mb = TilesManager(tiles_url='http://server') 320 | self.assertEqual(mb.cache.folder, '/tmp/landez/server') 321 | mb.add_filter(ColorToAlpha('#ffffff')) 322 | self.assertEqual(mb.cache.folder, '/tmp/landez/servercolortoalphaffffff') 323 | 324 | 325 | if __name__ == '__main__': 326 | logging.basicConfig(level=logging.DEBUG) 327 | unittest.main() 328 | -------------------------------------------------------------------------------- /landez/tiles.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import logging 4 | from gettext import gettext as _ 5 | import json 6 | import mimetypes 7 | import uuid 8 | 9 | from io import BytesIO, StringIO 10 | 11 | from mbutil import disk_to_mbtiles 12 | 13 | from . import (DEFAULT_TILES_URL, DEFAULT_TILES_SUBDOMAINS, 14 | DEFAULT_TMP_DIR, DEFAULT_FILEPATH, DEFAULT_TILE_SIZE, 15 | DEFAULT_TILE_FORMAT, DEFAULT_TILE_SCHEME) 16 | from .proj import GoogleProjection 17 | from .cache import Disk, Dummy 18 | from .sources import (MBTilesReader, TileDownloader, WMSReader, 19 | MapnikRenderer, ExtractionError, DownloadError) 20 | 21 | has_pil = False 22 | try: 23 | import Image 24 | import ImageEnhance 25 | has_pil = True 26 | except ImportError: 27 | try: 28 | from PIL import Image, ImageEnhance 29 | has_pil = True 30 | except ImportError: 31 | pass 32 | 33 | 34 | logger = logging.getLogger(__name__) 35 | 36 | 37 | 38 | class EmptyCoverageError(Exception): 39 | """ Raised when coverage (tiles list) is empty """ 40 | pass 41 | 42 | 43 | class TilesManager(object): 44 | 45 | def __init__(self, **kwargs): 46 | """ 47 | Manipulates tiles in general. Gives ability to list required tiles on a 48 | bounding box, download them, render them, extract them from other mbtiles... 49 | 50 | Keyword arguments: 51 | cache -- use a local cache to share tiles between runs (default True) 52 | 53 | tiles_dir -- Local folder containing existing tiles if cache is 54 | True, or where temporary tiles will be written otherwise 55 | (default DEFAULT_TMP_DIR) 56 | 57 | tiles_url -- remote URL to download tiles (*default DEFAULT_TILES_URL*) 58 | tiles_headers -- HTTP headers to send (*default empty*) 59 | 60 | stylefile -- mapnik stylesheet file (*to render tiles locally*) 61 | 62 | mbtiles_file -- A MBTiles file providing tiles (*to extract its tiles*) 63 | 64 | wms_server -- A WMS server url (*to request tiles*) 65 | wms_layers -- The list of layers to be requested 66 | wms_options -- WMS parameters to be requested (see ``landez.reader.WMSReader``) 67 | 68 | tile_size -- default tile size (default DEFAULT_TILE_SIZE) 69 | tile_format -- default tile format (default DEFAULT_TILE_FORMAT) 70 | tile_scheme -- default tile format (default DEFAULT_TILE_SCHEME) 71 | """ 72 | self.tile_size = kwargs.get('tile_size', DEFAULT_TILE_SIZE) 73 | self.tile_format = kwargs.get('tile_format', DEFAULT_TILE_FORMAT) 74 | self.tile_scheme = kwargs.get('tile_scheme', DEFAULT_TILE_SCHEME) 75 | 76 | # Tiles Download 77 | self.tiles_url = kwargs.get('tiles_url', DEFAULT_TILES_URL) 78 | self.tiles_subdomains = kwargs.get('tiles_subdomains', DEFAULT_TILES_SUBDOMAINS) 79 | self.tiles_headers = kwargs.get('tiles_headers') 80 | 81 | # Tiles rendering 82 | self.stylefile = kwargs.get('stylefile') 83 | 84 | # Grids rendering 85 | self.grid_fields = kwargs.get('grid_fields', []) 86 | self.grid_layer = kwargs.get('grid_layer', 0) 87 | 88 | # MBTiles reading 89 | self.mbtiles_file = kwargs.get('mbtiles_file') 90 | 91 | # WMS requesting 92 | self.wms_server = kwargs.get('wms_server') 93 | self.wms_layers = kwargs.get('wms_layers', []) 94 | self.wms_options = kwargs.get('wms_options', {}) 95 | 96 | if self.mbtiles_file: 97 | self.reader = MBTilesReader(self.mbtiles_file, self.tile_size) 98 | elif self.wms_server: 99 | assert self.wms_layers, _("Requires at least one layer (see ``wms_layers`` parameter)") 100 | self.reader = WMSReader(self.wms_server, self.wms_layers, self.tiles_headers, 101 | self.tile_size, **self.wms_options) 102 | if 'format' in self.wms_options: 103 | self.tile_format = self.wms_options['format'] 104 | logger.info(_("Tile format set to %s") % self.tile_format) 105 | elif self.stylefile: 106 | self.reader = MapnikRenderer(self.stylefile, self.tile_size) 107 | else: 108 | mimetype, encoding = mimetypes.guess_type(self.tiles_url) 109 | if mimetype and mimetype != self.tile_format: 110 | self.tile_format = mimetype 111 | logger.info(_("Tile format set to %s") % self.tile_format) 112 | self.reader = TileDownloader(self.tiles_url, headers=self.tiles_headers, 113 | subdomains=self.tiles_subdomains, tilesize=self.tile_size) 114 | 115 | # Tile files extensions 116 | self._tile_extension = mimetypes.guess_extension(self.tile_format, strict=False) 117 | assert self._tile_extension, _("Unknown format %s") % self.tile_format 118 | if self._tile_extension in ('.jpe', '.jpg'): 119 | self._tile_extension = '.jpeg' 120 | 121 | # Cache 122 | tiles_dir = kwargs.get('tiles_dir', DEFAULT_TMP_DIR) 123 | if kwargs.get('cache', True): 124 | self.cache = Disk(self.reader.basename, tiles_dir, extension=self._tile_extension) 125 | if kwargs.get('cache_scheme'): 126 | self.cache.scheme = kwargs.get('cache_scheme') 127 | else: 128 | self.cache = Dummy(extension=self._tile_extension) 129 | 130 | # Overlays 131 | self._layers = [] 132 | # Filters 133 | self._filters = [] 134 | # Number of tiles rendered/downloaded here 135 | self.rendered = 0 136 | 137 | def tileslist(self, bbox, zoomlevels): 138 | """ 139 | Build the tiles list within the bottom-left/top-right bounding 140 | box (minx, miny, maxx, maxy) at the specified zoom levels. 141 | Return a list of tuples (z,x,y) 142 | """ 143 | proj = GoogleProjection(self.tile_size, zoomlevels, self.tile_scheme) 144 | return proj.tileslist(bbox) 145 | 146 | def add_layer(self, tilemanager, opacity=1.0): 147 | """ 148 | Add a layer to be blended (alpha-composite) on top of the tile. 149 | tilemanager -- a `TileManager` instance 150 | opacity -- transparency factor for compositing 151 | """ 152 | assert has_pil, _("Cannot blend layers without python PIL") 153 | assert self.tile_size == tilemanager.tile_size, _("Cannot blend layers whose tile size differs") 154 | assert 0 <= opacity <= 1, _("Opacity should be between 0.0 (transparent) and 1.0 (opaque)") 155 | self.cache.basename += '%s%.1f' % (tilemanager.cache.basename, opacity) 156 | self._layers.append((tilemanager, opacity)) 157 | 158 | def add_filter(self, filter_): 159 | """ Add an image filter for post-processing """ 160 | assert has_pil, _("Cannot add filters without python PIL") 161 | self.cache.basename += filter_.basename 162 | self._filters.append(filter_) 163 | 164 | def tile(self, z_x_y): 165 | """ 166 | Return the tile (binary) content of the tile and seed the cache. 167 | """ 168 | (z, x, y) = z_x_y 169 | logger.debug(_("tile method called with %s") % ([z, x, y])) 170 | 171 | output = self.cache.read((z, x, y)) 172 | if output is None: 173 | output = self.reader.tile(z, x, y) 174 | # Blend layers 175 | if len(self._layers) > 0: 176 | logger.debug(_("Will blend %s layer(s)") % len(self._layers)) 177 | output = self._blend_layers(output, (z, x, y)) 178 | # Apply filters 179 | for f in self._filters: 180 | image = f.process(self._tile_image(output)) 181 | output = self._image_tile(image) 182 | # Save result to cache 183 | self.cache.save(output, (z, x, y)) 184 | 185 | self.rendered += 1 186 | return output 187 | 188 | def grid(self, z_x_y): 189 | """ Return the UTFGrid content """ 190 | # sources.py -> MapnikRenderer -> grid 191 | (z, x, y) = z_x_y 192 | content = self.reader.grid(z, x, y, self.grid_fields, self.grid_layer) 193 | return content 194 | 195 | 196 | def _blend_layers(self, imagecontent, z_x_y): 197 | """ 198 | Merge tiles of all layers into the specified tile path 199 | """ 200 | (z, x, y) = z_x_y 201 | result = self._tile_image(imagecontent) 202 | # Paste each layer 203 | for (layer, opacity) in self._layers: 204 | try: 205 | # Prepare tile of overlay, if available 206 | overlay = self._tile_image(layer.tile((z, x, y))) 207 | except (IOError, DownloadError, ExtractionError)as e: 208 | logger.warn(e) 209 | continue 210 | # Extract alpha mask 211 | overlay = overlay.convert("RGBA") 212 | r, g, b, a = overlay.split() 213 | overlay = Image.merge("RGB", (r, g, b)) 214 | a = ImageEnhance.Brightness(a).enhance(opacity) 215 | overlay.putalpha(a) 216 | mask = Image.merge("L", (a,)) 217 | result.paste(overlay, (0, 0), mask) 218 | # Read result 219 | return self._image_tile(result) 220 | 221 | def _tile_image(self, data): 222 | """ 223 | Tile binary content as PIL Image. 224 | """ 225 | image = Image.open(BytesIO(data)) 226 | return image.convert('RGBA') 227 | 228 | def _image_tile(self, image): 229 | out = StringIO() 230 | image.save(out, self._tile_extension[1:]) 231 | return out.getvalue() 232 | 233 | 234 | class MBTilesBuilder(TilesManager): 235 | def __init__(self, **kwargs): 236 | """ 237 | A MBTiles builder for a list of bounding boxes and zoom levels. 238 | 239 | filepath -- output MBTiles file (default DEFAULT_FILEPATH) 240 | tmp_dir -- temporary folder for gathering tiles (default DEFAULT_TMP_DIR/filepath) 241 | ignore_errors -- ignore errors during MBTiles creation (e.g. download errors) 242 | """ 243 | super(MBTilesBuilder, self).__init__(**kwargs) 244 | self.filepath = kwargs.get('filepath', DEFAULT_FILEPATH) 245 | self.ignore_errors = kwargs.get('ignore_errors', False) 246 | # Gather tiles for mbutil 247 | basename, ext = os.path.splitext(os.path.basename(self.filepath)) 248 | self.tmp_dir = kwargs.get('tmp_dir', DEFAULT_TMP_DIR) 249 | self.tmp_dir = os.path.join(self.tmp_dir, basename) 250 | self.tile_format = kwargs.get('tile_format', DEFAULT_TILE_FORMAT) 251 | 252 | # Number of tiles in total 253 | self.nbtiles = 0 254 | self._bboxes = [] 255 | 256 | def add_coverage(self, bbox, zoomlevels): 257 | """ 258 | Add a coverage to be included in the resulting mbtiles file. 259 | """ 260 | self._bboxes.append((bbox, zoomlevels)) 261 | 262 | @property 263 | def zoomlevels(self): 264 | """ 265 | Return the list of covered zoom levels, in ascending order 266 | """ 267 | zooms = set() 268 | for coverage in self._bboxes: 269 | for zoom in coverage[1]: 270 | zooms.add(zoom) 271 | return sorted(zooms) 272 | 273 | @property 274 | def bounds(self): 275 | """ 276 | Return the bounding box of covered areas 277 | """ 278 | return self._bboxes[0][0] #TODO: merge all coverages 279 | 280 | def run(self, force=False): 281 | """ 282 | Build a MBTile file. 283 | 284 | force -- overwrite if MBTiles file already exists. 285 | """ 286 | if os.path.exists(self.filepath): 287 | if force: 288 | logger.warn(_("%s already exists. Overwrite.") % self.filepath) 289 | os.remove(self.filepath) 290 | else: 291 | # Already built, do not do anything. 292 | logger.info(_("%s already exists. Nothing to do.") % self.filepath) 293 | return 294 | 295 | # Clean previous runs 296 | self._clean_gather() 297 | 298 | # If no coverage added, use bottom layer metadata 299 | if len(self._bboxes) == 0 and len(self._layers) > 0: 300 | bottomlayer = self._layers[0] 301 | metadata = bottomlayer.reader.metadata() 302 | if 'bounds' in metadata: 303 | logger.debug(_("Use bounds of bottom layer %s") % bottomlayer) 304 | bbox = map(float, metadata.get('bounds', '').split(',')) 305 | zoomlevels = range(int(metadata.get('minzoom', 0)), int(metadata.get('maxzoom', 0))) 306 | self.add_coverage(bbox=bbox, zoomlevels=zoomlevels) 307 | 308 | # Compute list of tiles 309 | tileslist = set() 310 | for bbox, levels in self._bboxes: 311 | logger.debug(_("Compute list of tiles for bbox %s on zooms %s.") % (bbox, levels)) 312 | bboxlist = self.tileslist(bbox, levels) 313 | logger.debug(_("Add %s tiles.") % len(bboxlist)) 314 | tileslist = tileslist.union(bboxlist) 315 | logger.debug(_("%s tiles in total.") % len(tileslist)) 316 | self.nbtiles = len(tileslist) 317 | if not self.nbtiles: 318 | raise EmptyCoverageError(_("No tiles are covered by bounding boxes : %s") % self._bboxes) 319 | logger.debug(_("%s tiles to be packaged.") % self.nbtiles) 320 | 321 | # Go through whole list of tiles and gather them in tmp_dir 322 | self.rendered = 0 323 | for (z, x, y) in tileslist: 324 | try: 325 | self._gather((z, x, y)) 326 | except Exception as e: 327 | logger.warn(e) 328 | if not self.ignore_errors: 329 | raise 330 | 331 | logger.debug(_("%s tiles were missing.") % self.rendered) 332 | 333 | # Some metadata 334 | middlezoom = self.zoomlevels[len(self.zoomlevels) // 2] 335 | lat = self.bounds[1] + (self.bounds[3] - self.bounds[1])/2 336 | lon = self.bounds[0] + (self.bounds[2] - self.bounds[0])/2 337 | metadata = {} 338 | metadata['name'] = str(uuid.uuid4()) 339 | metadata['format'] = self._tile_extension[1:] 340 | metadata['minzoom'] = self.zoomlevels[0] 341 | metadata['maxzoom'] = self.zoomlevels[-1] 342 | metadata['bounds'] = '%s,%s,%s,%s' % tuple(self.bounds) 343 | metadata['center'] = '%s,%s,%s' % (lon, lat, middlezoom) 344 | #display informations from the grids on hover 345 | content_to_display = '' 346 | for field_name in self.grid_fields: 347 | content_to_display += "{{{ %s }}}
" % field_name 348 | metadata['template'] = '{{#__location__}}{{/__location__}} {{#__teaser__}} \ 349 | %s {{/__teaser__}}{{#__full__}}{{/__full__}}' % content_to_display 350 | metadatafile = os.path.join(self.tmp_dir, 'metadata.json') 351 | with open(metadatafile, 'w') as output: 352 | json.dump(metadata, output) 353 | 354 | # TODO: add UTF-Grid of last layer, if any 355 | 356 | # Package it! 357 | logger.info(_("Build MBTiles file '%s'.") % self.filepath) 358 | extension = self.tile_format.split("image/")[-1] 359 | disk_to_mbtiles( 360 | self.tmp_dir, 361 | self.filepath, 362 | format=extension, 363 | scheme=self.cache.scheme 364 | ) 365 | 366 | try: 367 | os.remove("%s-journal" % self.filepath) # created by mbutil 368 | except OSError as e: 369 | pass 370 | self._clean_gather() 371 | 372 | def _gather(self, z_x_y): 373 | (z, x, y) = z_x_y 374 | files_dir, tile_name = self.cache.tile_file((z, x, y)) 375 | tmp_dir = os.path.join(self.tmp_dir, files_dir) 376 | if not os.path.isdir(tmp_dir): 377 | os.makedirs(tmp_dir) 378 | tilecontent = self.tile((z, x, y)) 379 | tilepath = os.path.join(tmp_dir, tile_name) 380 | with open(tilepath, 'wb') as f: 381 | f.write(tilecontent) 382 | if len(self.grid_fields) > 0: 383 | gridcontent = self.grid((z, x, y)) 384 | gridpath = "%s.%s" % (os.path.splitext(tilepath)[0], 'grid.json') 385 | with open(gridpath, 'w') as f: 386 | f.write(gridcontent) 387 | 388 | def _clean_gather(self): 389 | logger.debug(_("Clean-up %s") % self.tmp_dir) 390 | try: 391 | shutil.rmtree(self.tmp_dir) 392 | #Delete parent folder only if empty 393 | try: 394 | parent = os.path.dirname(self.tmp_dir) 395 | os.rmdir(parent) 396 | logger.debug(_("Clean-up parent %s") % parent) 397 | except OSError: 398 | pass 399 | except OSError: 400 | pass 401 | 402 | 403 | class ImageExporter(TilesManager): 404 | def __init__(self, **kwargs): 405 | """ 406 | Arrange the tiles and join them together to build a single big image. 407 | """ 408 | super(ImageExporter, self).__init__(**kwargs) 409 | 410 | def grid_tiles(self, bbox, zoomlevel): 411 | """ 412 | Return a grid of (x, y) tuples representing the juxtaposition 413 | of tiles on the specified ``bbox`` at the specified ``zoomlevel``. 414 | """ 415 | tiles = self.tileslist(bbox, [zoomlevel]) 416 | grid = {} 417 | for (z, x, y) in tiles: 418 | if not grid.get(y): 419 | grid[y] = [] 420 | grid[y].append(x) 421 | sortedgrid = [] 422 | for y in sorted(grid.keys(), reverse=self.tile_scheme == 'tms'): 423 | sortedgrid.append([(x, y) for x in sorted(grid[y])]) 424 | return sortedgrid 425 | 426 | def export_image(self, bbox, zoomlevel, imagepath): 427 | """ 428 | Writes to ``imagepath`` the tiles for the specified bounding box and zoomlevel. 429 | """ 430 | assert has_pil, _("Cannot export image without python PIL") 431 | grid = self.grid_tiles(bbox, zoomlevel) 432 | width = len(grid[0]) 433 | height = len(grid) 434 | widthpix = width * self.tile_size 435 | heightpix = height * self.tile_size 436 | 437 | result = Image.new("RGBA", (widthpix, heightpix)) 438 | offset = (0, 0) 439 | for i, row in enumerate(grid): 440 | for j, (x, y) in enumerate(row): 441 | offset = (j * self.tile_size, i * self.tile_size) 442 | img = self._tile_image(self.tile((zoomlevel, x, y))) 443 | result.paste(img, offset) 444 | logger.info(_("Save resulting image to '%s'") % imagepath) 445 | result.save(imagepath) 446 | -------------------------------------------------------------------------------- /landez/util.py: -------------------------------------------------------------------------------- 1 | def flip_y(y, z): 2 | return 2 ** z - 1 - y 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/mapbox/mbutil.git@master#egg=mbutil 2 | Pillow == 5.1.0 3 | requests == 2.21.0 4 | mock == 0.7.2 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf8 -*- 3 | import os 4 | from setuptools import setup, find_packages 5 | 6 | here = os.path.abspath(os.path.dirname(__file__)) 7 | 8 | setup( 9 | name='landez', 10 | version='2.5.1.dev0', 11 | author='Mathieu Leplatre', 12 | author_email='mathieu.leplatre@makina-corpus.com', 13 | url='https://github.com/makinacorpus/landez/', 14 | download_url="http://pypi.python.org/pypi/landez/", 15 | description="Landez is a python toolbox to manipulate map tiles.", 16 | long_description=open(os.path.join(here, 'README.rst')).read() + '\n\n' + 17 | open(os.path.join(here, 'CHANGES')).read(), 18 | license='LPGL, see LICENSE file.', 19 | install_requires = [ 20 | 'mbutil', 21 | 'requests', 22 | ], 23 | extras_require = { 24 | 'PIL': ["Pillow"], 25 | 'Mapnik': ["Mapnik >= 2.0.0"] 26 | }, 27 | packages=find_packages(), 28 | include_package_data=True, 29 | zip_safe=False, 30 | keywords=['MBTiles', 'Mapnik'], 31 | classifiers=['Programming Language :: Python :: 3.5', 32 | 'Natural Language :: English', 33 | 'Topic :: Utilities', 34 | 'Development Status :: 5 - Production/Stable'], 35 | ) 36 | --------------------------------------------------------------------------------