Postile/OpenMapTiles example

├── postile ├── __init__.py ├── static │ └── favicon.ico ├── sql.py ├── templates │ ├── index.html │ └── index-debug.html └── postile.py ├── MANIFEST.in ├── Dockerfile ├── LICENSE ├── .gitignore ├── setup.py ├── README.md └── assets └── graphics └── postile_logo.svg /postile/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.1.1' 2 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include postile/static/* 2 | include postile/templates/*.html 3 | -------------------------------------------------------------------------------- /postile/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Oslandia/postile/HEAD/postile/static/favicon.ico -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:buster-slim 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y git libprotobuf-dev libprotobuf-c-dev python3.6 python3-pip \ 5 | && rm -rf /var/lib/apt/lists/* 6 | 7 | RUN cd /opt \ 8 | && git clone https://github.com/oslandia/postile \ 9 | && cd postile \ 10 | && pip3 install cython \ 11 | && pip3 install . 12 | 13 | CMD ["/usr/local/bin/postile"] 14 | -------------------------------------------------------------------------------- /postile/sql.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module containing SQL raw requests 3 | """ 4 | 5 | 6 | single_layer = """ 7 | select st_asmvt(tile, '{layer}', 4096) 8 | from ( 9 | with tmp as ( 10 | select st_srid({geom}) as srid 11 | from {layer} 12 | limit 1 13 | ) select * from tmp 14 | , lateral ( 15 | select 16 | st_asmvtgeom( 17 | st_simplify( 18 | st_transform({geom}, {OUTPUT_SRID}) 19 | , {scale}, true 20 | ) 21 | , {bbox}) as mvtgeom 22 | {fields} 23 | from {layer} 24 | where st_transform({bbox}, tmp.srid) && {geom} 25 | ) _ where mvtgeom is not null 26 | ) as tile 27 | """ 28 | -------------------------------------------------------------------------------- /postile/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Postile/OpenMapTiles example 6 | 7 | 8 | 9 | 13 | 14 | 15 | 16 |

17 | 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Ludovic Delauné 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /postile/templates/index-debug.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Postile/OpenMapTiles example DEBUG mode 6 | 7 | 8 | 9 | 20 | 21 | 22 | 23 |

24 |

25 | tile debug
26 | collision debug
27 |

28 | 48 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import io 4 | from setuptools import setup, find_packages 5 | 6 | here = os.path.abspath(os.path.dirname(__file__)) 7 | 8 | requirements = ( 9 | 'cython', 10 | 'mercantile', 11 | 'sanic==19.6.3', 12 | 'sanic-cors==0.9.9.post1', 13 | 'pyyaml', 14 | 'asyncpg>=0.15.0', 15 | 'jinja2' 16 | ) 17 | 18 | 19 | def find_version(*file_paths): 20 | """ 21 | see https://github.com/pypa/sampleproject/blob/master/setup.py 22 | """ 23 | with io.open(os.path.join(here, *file_paths), 'r') as f: 24 | version_file = f.read() 25 | 26 | # The version line must have the form 27 | # __version__ = 'ver' 28 | version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", 29 | version_file, re.M) 30 | if version_match: 31 | return version_match.group(1) 32 | raise RuntimeError("Unable to find version string. " 33 | "Should be at the first line of __init__.py.") 34 | 35 | 36 | with open("README.md", "r") as fh: 37 | long_description = fh.read() 38 | 39 | 40 | setup( 41 | name='Postile', 42 | version=find_version('postile', '__init__.py'), 43 | description="Fast Vector Tile Server", 44 | long_description=long_description, 45 | long_description_content_type="text/markdown", 46 | url='https://github.com/ldgeo/postile', 47 | author='ldgeo', 48 | author_email='contact@oslandia.com', 49 | license='BSD-3', 50 | classifiers=[ 51 | 'Development Status :: 3 - Alpha', 52 | 'Intended Audience :: Developers', 53 | 'Programming Language :: Python :: 3.6', 54 | ], 55 | packages=find_packages(), 56 | install_requires=requirements, 57 | include_package_data=True, 58 | entry_points={ 59 | 'console_scripts': [ 60 | 'postile=postile.postile:main', 61 | ], 62 | }, 63 | ) 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 |

5 | 6 | Build Status 7 | 8 | 9 | Package version 10 | 11 |

12 | 13 | --- 14 | 15 | Fast Mapbox Vector Tile Server mainly suited for the [openmaptiles vector tile schema](https://github.com/openmaptiles/openmaptiles) 16 | 17 | ## Features 18 | 19 | - Support for PostGIS backend through a tm2source (as generated by [OpenMapTiles](https://github.com/openmaptiles/openmaptiles)) 20 | - Support for PostGIS single layers 21 | - Support for reading MBTiles 22 | - On-the-fly reprojection to web mercator EPSG:3857 (only for single layers) 23 | - Connection pooling and asynchronous requests thanks to [asyncpg](https://github.com/MagicStack/asyncpg) 24 | 25 | ## Requirements 26 | 27 | - Python `>= 3.6` 28 | - for PostGIS backend, recent `ST_AsMVT` function. At least PostGIS >= 2.4.0. 29 | 30 | 31 | ## Installation 32 | 33 | pip install cython 34 | pip install -e . 35 | postile --help 36 | 37 | ## Using a Docker container 38 | 39 | Start Postile with: 40 | 41 | docker run --network host oslandia/postile postile --help 42 | 43 | ## Example of serving postgis layers individually 44 | 45 | postile --pguser **** --pgpassword **** --pgdatabase mydb --pghost localhost --listen-port 8080 --cors 46 | 47 | Then layer `boundaries` can be served with: 48 | 49 | http://localhost:8080/boundaries/z/x/y.pbf?fields=id,name 50 | 51 | `fields` is optional, and when absent only geometries are encoded in the vector tile. 52 | 53 | ## Preview 54 | 55 | The root endpoint will display a built-in viewer with `mapbox-gl-js`. 56 | In `DEBUG` mode the same page will also add some checkboxes to show tile boundaries and collision boxes (for labels). 57 | 58 | 59 | --- 60 | *For a concrete example using OpenMapTiles schema see [this tutorial](https://github.com/ldgeo/postile-openmaptiles)* 61 | 62 | -------------------------------------------------------------------------------- /postile/postile.py: -------------------------------------------------------------------------------- 1 | """ 2 | Fast VectorTile server for PostGIS backend 3 | 4 | inspired by https://github.com/openmaptiles/postserve 5 | """ 6 | import io 7 | import os 8 | import socket 9 | import sys 10 | import re 11 | import argparse 12 | import sqlite3 13 | from pathlib import Path 14 | 15 | from sanic import Sanic 16 | from sanic.log import logger 17 | from sanic import response 18 | from sanic_cors import CORS 19 | 20 | import mercantile 21 | import yaml 22 | 23 | import asyncio 24 | import asyncpg 25 | 26 | from jinja2 import Environment, PackageLoader, select_autoescape 27 | 28 | from postile.sql import single_layer 29 | 30 | # https://github.com/openstreetmap/mapnik-stylesheets/blob/master/zoom-to-scale.txt 31 | # map width in meters for web mercator 3857 32 | MAP_WIDTH_IN_METRES = 40075016.68557849 33 | TILE_WIDTH_IN_PIXELS = 256.0 34 | STANDARDIZED_PIXEL_SIZE = 0.00028 35 | 36 | # prepare regexp to extract the query from a tm2 table subquery 37 | LAYERQUERY = re.compile(r'\s*$(?P.*)$\s+as\s+\w+\s*', re.IGNORECASE | re.DOTALL) 38 | 39 | # the de facto standard projection for web mapping applications 40 | # official EPSG code 41 | OUTPUT_SRID = 3857 42 | 43 | app = Sanic() 44 | 45 | # lower zooms can take a while to generate (ie zoom 0->4) 46 | app.config.RESPONSE_TIMEOUT = 60 * 2 47 | 48 | # where am i ? 49 | here = Path(os.path.abspath(os.path.dirname(__file__))) 50 | 51 | jinja_env = Environment( 52 | loader=PackageLoader('postile', 'templates'), 53 | autoescape=select_autoescape(['html', 'xml']) 54 | ) 55 | 56 | class Config: 57 | # postgresql DSN 58 | dsn = None 59 | # tm2source prepared query 60 | tm2query = None 61 | # sqlite3 connection 62 | db_sqlite = None 63 | # style configuration file 64 | style = None 65 | # database connection pool 66 | db_pg = None 67 | fonts = None 68 | 69 | 70 | @app.listener('before_server_start') 71 | async def setup_db_pg(app, loop): 72 | """ 73 | initiate postgresql connection 74 | """ 75 | if Config.dsn: 76 | try: 77 | Config.db_pg = await asyncpg.create_pool(Config.dsn, loop=loop) 78 | except socket.gaierror: 79 | print(f'Cannot establish connection to {Config.dsn}. \ 80 | Did you pass correct values to --pghost?') 81 | raise 82 | except asyncpg.exceptions.InvalidPasswordError: 83 | print(f'Cannot connect to {Config.dsn}. \ 84 | Please check values passed to --pguser and --pgpassword') 85 | raise 86 | 87 | 88 | @app.listener('after_server_stop') 89 | async def cleanup_db_pg(app, loop): 90 | if Config.dsn: 91 | await Config.db_pg.close() 92 | 93 | 94 | def zoom_to_scale_denom(zoom): 95 | map_width_in_pixels = TILE_WIDTH_IN_PIXELS * (2 ** zoom) 96 | return MAP_WIDTH_IN_METRES / (map_width_in_pixels * STANDARDIZED_PIXEL_SIZE) 97 | 98 | 99 | def resolution(zoom): 100 | """ 101 | Takes a web mercator zoom level and returns the pixel resolution for that 102 | scale according to the global TILE_WIDTH_IN_PIXELS size 103 | """ 104 | return MAP_WIDTH_IN_METRES / (TILE_WIDTH_IN_PIXELS * (2 ** zoom)) 105 | 106 | 107 | def prepared_query(filename): 108 | with io.open(filename, 'r') as stream: 109 | layers = yaml.load(stream, Loader=yaml.FullLoader) 110 | 111 | queries = [] 112 | for layer in layers['Layer']: 113 | # Remove whitespaces, subquery parenthesis and final alias 114 | query = LAYERQUERY.match(layer['Datasource']['table']).group('query') 115 | 116 | query = query.replace( 117 | layer['Datasource']['geometry_field'], 118 | "st_asmvtgeom({}, {{bbox}}) as mvtgeom" 119 | .format(layer['Datasource']['geometry_field']) 120 | ) 121 | query = query.replace('!bbox!', '{bbox}') 122 | query = query.replace('!scale_denominator!', "{scale_denominator}") 123 | query = query.replace('!pixel_width!', '{pixel_width}') 124 | query = query.replace('!pixel_height!', '{pixel_height}') 125 | 126 | query = """ 127 | select st_asmvt(tile, '{}', 4096, 'mvtgeom') 128 | from ({} where st_asmvtgeom({}, {{bbox}}) is not null) as tile 129 | """.format(layer['id'], query, layer['Datasource']['geometry_field']) 130 | 131 | queries.append(query) 132 | 133 | return " union all ".join(queries) 134 | 135 | 136 | @app.route('/style.json') 137 | async def get_jsonstyle(request): 138 | if not Config.style: 139 | return response.text('no style available', status=404) 140 | 141 | return await response.file( 142 | Config.style, 143 | headers={"Content-Type": "application/json"} 144 | ) 145 | 146 | @app.route('/fonts//.pbf') 147 | async def get_fonts(request, fontstack, frange): 148 | if not Config.fonts: 149 | return response.text('no fonts available', status=404) 150 | return await response.file( 151 | Path(Config.fonts) / fontstack / f'{frange}.pbf', 152 | headers={"Content-Type": "application/x-protobuf"} 153 | ) 154 | 155 | async def get_mbtiles(request, z, x, y): 156 | # Flip Y coordinate because MBTiles store tiles in TMS. 157 | coords = (x, (1 << z) - 1 - y, z) 158 | cursor = Config.db_sqlite.execute(""" 159 | SELECT tile_data 160 | FROM tiles 161 | WHERE tile_column=? and tile_row=? and zoom_level=? 162 | LIMIT 1 """, coords) 163 | 164 | tile = cursor.fetchone() 165 | if tile: 166 | return response.raw( 167 | tile[0], 168 | headers={"Content-Type": "application/x-protobuf", 169 | "Content-Encoding": "gzip"}) 170 | else: 171 | return response.raw(b'', 172 | headers={"Content-Type": "application/x-protobuf"}) 173 | 174 | async def get_tile_tm2(request, x, y, z): 175 | """ 176 | """ 177 | scale_denominator = zoom_to_scale_denom(z) 178 | 179 | # compute mercator bounds 180 | bounds = mercantile.xy_bounds(x, y, z) 181 | bbox = f"st_makebox2d(st_point({bounds.left}, {bounds.bottom}), st_point({bounds.right},{bounds.top}))" 182 | 183 | sql = Config.tm2query.format( 184 | bbox=bbox, 185 | scale_denominator=scale_denominator, 186 | pixel_width=256, 187 | pixel_height=256, 188 | ) 189 | logger.debug(sql) 190 | 191 | async with Config.db_pg.acquire() as conn: 192 | # join tiles into one bytes string except null tiles 193 | rows = await conn.fetch(sql) 194 | pbf = b''.join([row[0] for row in rows if row[0]]) 195 | 196 | return response.raw( 197 | pbf, 198 | headers={"Content-Type": "application/x-protobuf"} 199 | ) 200 | 201 | async def get_tile_postgis(request, x, y, z, layer): 202 | """ 203 | Direct access to a postgis layer 204 | """ 205 | if ' ' in layer: 206 | return response.text('bad layer name: {}'.format(layer), status=404) 207 | 208 | # get fields given in parameters 209 | fields = ',' + request.raw_args['fields'] if 'fields' in request.raw_args else '' 210 | # get geometry column name from query args else geom is used 211 | geom = request.raw_args.get('geom', 'geom') 212 | # compute mercator bounds 213 | bounds = mercantile.xy_bounds(x, y, z) 214 | 215 | # make bbox for filtering 216 | bbox = f"st_setsrid(st_makebox2d(st_point({bounds.left}, {bounds.bottom}), st_point({bounds.right},{bounds.top})), {OUTPUT_SRID})" 217 | 218 | # compute pixel resolution 219 | scale = resolution(z) 220 | 221 | sql = single_layer.format(**locals(), OUTPUT_SRID=OUTPUT_SRID) 222 | 223 | logger.debug(sql) 224 | 225 | async with Config.db_pg.acquire() as conn: 226 | rows = await conn.fetch(sql) 227 | pbf = b''.join([row[0] for row in rows if row[0]]) 228 | 229 | return response.raw( 230 | pbf, 231 | headers={"Content-Type": "application/x-protobuf"} 232 | ) 233 | 234 | def preview(request): 235 | """build and return a preview page 236 | """ 237 | if app.debug: 238 | template = jinja_env.get_template('index-debug.html') 239 | else: 240 | template = jinja_env.get_template('index.html') 241 | 242 | html_content = template.render(host=request.host, scheme=request.scheme) 243 | return response.html(html_content) 244 | 245 | def config_tm2(tm2file): 246 | """Adds specific routes for tm2 source and prepare the global SQL Query 247 | 248 | """ 249 | # build the SQL query for all layers found in TM2 file 250 | Config.tm2query = prepared_query(tm2file) 251 | # add route dedicated to tm2 queries 252 | app.add_route(get_tile_tm2, r'///.pbf', methods=['GET']) 253 | app.add_route(preview, r'/', methods=['GET']) 254 | 255 | 256 | def config_mbtiles(mbtiles): 257 | """Adds specific routes for mbtiles source 258 | 259 | """ 260 | Config.db_sqlite = sqlite3.connect(mbtiles) 261 | app.add_route(get_mbtiles, r'///.pbf', methods=['GET']) 262 | app.add_route(preview, r'/', methods=['GET']) 263 | 264 | 265 | def check_file_exists(filename): 266 | if not os.path.exists(filename): 267 | print(f'file does not exists: {filename}, quitting...') 268 | sys.exit(1) 269 | 270 | 271 | def main(): 272 | parser = argparse.ArgumentParser(description='Fast VectorTile server with PostGIS backend') 273 | parser.add_argument('--tm2', type=str, help='TM2 source file (yaml)') 274 | parser.add_argument('--mbtiles', type=str, help='read tiles from a mbtiles file') 275 | parser.add_argument('--style', type=str, help='GL Style to serve at /style.json') 276 | parser.add_argument('--pgdatabase', type=str, help='database name', default='osm') 277 | parser.add_argument('--pghost', type=str, help='postgres hostname', default='') 278 | parser.add_argument('--pgport', type=int, help='postgres port', default=5432) 279 | parser.add_argument('--pguser', type=str, help='postgres user', default='') 280 | parser.add_argument('--pgpassword', type=str, help='postgres password', default='') 281 | parser.add_argument('--listen', type=str, help='listen address', default='127.0.0.1') 282 | parser.add_argument('--listen-port', type=str, help='listen port', default=8080) 283 | parser.add_argument('--cors', action='store_true', help='make cross-origin AJAX possible') 284 | parser.add_argument('--debug', action='store_true', help='activate sanic debug mode') 285 | parser.add_argument('--fonts', type=str, help='fonts location') 286 | args = parser.parse_args() 287 | 288 | if len(sys.argv) == 1: 289 | # display help message when no args are passed. 290 | parser.print_help() 291 | sys.exit(1) 292 | 293 | if args.tm2: 294 | check_file_exists(args.tm2) 295 | config_tm2(args.tm2) 296 | elif args.mbtiles: 297 | check_file_exists(args.mbtiles) 298 | config_mbtiles(args.mbtiles) 299 | else: 300 | # no tm2 file given, switching to direct connection to postgis layers 301 | app.add_route(get_tile_postgis, r'////.pbf', methods=['GET']) 302 | if args.style: 303 | check_file_exists(args.style) 304 | Config.style = args.style 305 | 306 | if args.fonts: 307 | check_file_exists(args.fonts) 308 | Config.fonts = args.fonts 309 | 310 | # interpolate values for postgres connection 311 | if not args.mbtiles: 312 | Config.dsn = ( 313 | 'postgres://{pguser}:{pgpassword}@{pghost}:{pgport}/{pgdatabase}' 314 | .format(**args.__dict__) 315 | ) 316 | 317 | if args.cors: 318 | CORS(app) 319 | 320 | # add static route for the favicon 321 | app.static('/favicon.ico', str(here / 'static/favicon.ico')) 322 | 323 | app.run( 324 | host=args.listen, 325 | port=args.listen_port, 326 | debug=args.debug) 327 | 328 | 329 | if __name__ == '__main__': 330 | main() 331 | -------------------------------------------------------------------------------- /assets/graphics/postile_logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 355 | --------------------------------------------------------------------------------