├── test_generate_tiles.py ├── .gitignore ├── tiles2mbt.py ├── tile_list.py ├── redering_tile_server.py ├── tests.py ├── tiles2renderd.py ├── tile_server.py ├── utils.py ├── atlas.ini ├── README.md ├── tiles.py ├── map_utils.py ├── rendering_tile_server-sockets.py └── generate_tiles.py /test_generate_tiles.py: -------------------------------------------------------------------------------- 1 | from generate_tiles import time2hms 2 | 3 | 4 | def test_time2hms(): 5 | assert time2hms (0) == (0, 0, 0) 6 | assert time2hms (59) == (0, 0, 59) 7 | assert time2hms (60) == (0, 1, 0) 8 | assert time2hms(3600) == (1, 0, 0) 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | Elevation/0 2 | Elevation/1 3 | Elevation/2 4 | Elevation/3 5 | Elevation/4 6 | Elevation/5 7 | Elevation/6 8 | Elevation/7 9 | Elevation/8 10 | Elevation/9 11 | Elevation/10 12 | Elevation/11 13 | Elevation/12 14 | Elevation/13 15 | Elevation/14 16 | Elevation/15 17 | Elevation/16 18 | Elevation/17 19 | Elevation/18 20 | data/height/*.tif 21 | openstreetmap-carto.xml 22 | openstreetmap-carto/ 23 | tilemill/ 24 | *.log 25 | .kate-swp 26 | *~ 27 | data/height/*.vrt 28 | data/height/*.tif 29 | *.pyc 30 | *.pyo 31 | openstreetmap-carto* 32 | *.xml 33 | -------------------------------------------------------------------------------- /tiles2mbt.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import sys 4 | import os.path 5 | 6 | import map_utils 7 | 8 | sector = sys.argv[1] 9 | 10 | atlas = map_utils.Atlas([sector]) 11 | map = atlas.maps[sector] 12 | backend = map_utils.MBTilesBackend(f"{sector}.mbt", map.bbox) 13 | 14 | # backend.init() 15 | 16 | print('INSERTING TILES') 17 | for z in range(map.max_z + 1): 18 | for x in map.iterate_x(z): 19 | for y in map.iterate_y(z): 20 | try: 21 | data = open(os.path.join('Elevation', str(z), str(x), "%d.png" % y), 22 | 'rb').read() 23 | except FileNotFoundError: 24 | pass 25 | else: 26 | backend.store_raw(z, x, y, data) 27 | 28 | backend.commit() 29 | -------------------------------------------------------------------------------- /tile_list.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python2 2 | 3 | import sys 4 | 5 | import map_utils 6 | 7 | bbox= [ float (x) for x in sys.argv[1].split (',') ] 8 | 9 | minZoom= 0 10 | maxZoom= int (bbox.pop ()) 11 | 12 | gprj = map_utils.GoogleProjection(maxZoom+1) 13 | 14 | # print bbox, maxZoom 15 | 16 | ll0 = (bbox[0],bbox[3]) 17 | ll1 = (bbox[2],bbox[1]) 18 | 19 | image_size = 256.0 20 | 21 | for z in range(minZoom, maxZoom + 1): 22 | px0 = gprj.lon_lat2pixel(ll0, z) 23 | px1 = gprj.lon_lat2pixel(ll1, z) 24 | 25 | for x in range(int(px0[0] / image_size), 26 | int(px1[0] / image_size) + 1): 27 | # Validate x co-ordinate 28 | if (x < 0) or (x >= 2**z): 29 | continue 30 | 31 | for y in range(int(px0[1] / image_size), 32 | int(px1[1] / image_size) + 1): 33 | # Validate x co-ordinate 34 | if (y < 0) or (y >= 2**z): 35 | continue 36 | 37 | print("%d/%d/%d.png" % (z, x, y)) 38 | -------------------------------------------------------------------------------- /redering_tile_server.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | # I would like to use FastAPI, but this is not really an API ... is it? 4 | # yes! you can say 5 | # @app.get("/{z}/{x}/{y}.{ext}") 6 | # and FastAPI will resolve it for you! 7 | 8 | # but 9 | 10 | # FastAPI has QUITE some deps: 81MiB in total 11 | # not suitable for a phone, ... but we're not going to run this on a phone, are we? :) 12 | 13 | import os.path 14 | import sys 15 | 16 | from fastapi import FastAPI, status 17 | from fastapi.responses import FileResponse, JSONResponse 18 | 19 | app = FastAPI() 20 | # app.root_dir = '.' 21 | app.root_dir = '/home/mdione/src/projects/elevation/Elevation' 22 | # render_farm = 23 | 24 | 25 | # returning Files requires async 26 | @app.get("/{z}/{x}/{y}.{ext}") 27 | async def get_tile(z: int, x: int, y: int, ext: str): 28 | # TypeError: join() argument must be str, bytes, or os.PathLike object, not 'int' 29 | # tile_path = os.path.join(app.root_dir, z, x, f"{y}.{ext}") 30 | tile_path = f"{app.root_dir}/{z}/{x}/{y}.{ext}" 31 | if os.path.exists(tile_path): 32 | return FileResponse(tile_path) 33 | 34 | # beh, it doesn't 404's 35 | # return FileResponse(tile_path) 36 | return JSONResponse(status_code=status.HTTP_404_NOT_FOUND, content=None) 37 | 38 | if __name__ == '__main__': 39 | app.root_dir = sys.argv[1] 40 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import unittest 4 | import os 5 | import sqlite3 6 | 7 | import map_utils 8 | 9 | class TestMBTiles (unittest.TestCase): 10 | 11 | def setUp (self): 12 | # the bbox is not really important 13 | self.backend= map_utils.MBTilesBackend ('TestMBTiles', [10, 20, 30, 40]) 14 | session= sqlite3.connect ('TestMBTiles.mbt') 15 | self.session.set_trace_callback (print) 16 | self.db= session.cursor () 17 | 18 | 19 | def test_single_tile (self): 20 | data= open ('sea.png', 'rb').read () 21 | 22 | self.backend.store (0, 0, 0, data) 23 | self.backend.commit () 24 | 25 | self.assertTrue (self.backend.exists (0, 0, 0)) 26 | 27 | def test_two_seas_one_tile (self): 28 | data= open ('sea.png', 'rb').read () 29 | 30 | self.backend.store (0, 0, 0, data) 31 | self.backend.store (1, 1, 1, data) 32 | self.backend.commit () 33 | self.backend.close () 34 | 35 | # test two seas 36 | seas= self.db.execute ('select * from map;').fetchall () 37 | 38 | self.assertEqual (len (seas), 2) 39 | 40 | # test one id 41 | self.assertEqual (seas[0][3], seas[1][3]) 42 | 43 | # test one tile 44 | tile= self.db.execute ('select * from images;').fetchall () 45 | 46 | self.assertEqual (len (tile), 1) 47 | 48 | # test one id, again 49 | self.assertEqual (seas[0][3], tile[0][0]) 50 | 51 | # test one view 52 | tiles= self.db.execute ('select * from tiles;').fetchall () 53 | 54 | self.assertEqual (len (tiles), 2) 55 | 56 | def test_update (self): 57 | data1= open ('data1.png', 'rb').read () 58 | data2= open ('data2.png', 'rb').read () 59 | 60 | self.backend.store (0, 0, 0, data1) 61 | self.backend.commit () 62 | self.backend.store (0, 0, 0, data2) 63 | self.backend.commit () 64 | self.backend.close () 65 | 66 | data3= self.db.execute ('SELECT tile_data FROM tiles;').fetchall () 67 | self.assertEqual (data3[0][0], data2) 68 | 69 | def tearDown (self): 70 | self.backend.close () 71 | os.unlink ('TestMBTiles.mbt') 72 | 73 | if __name__ == '__main__': 74 | unittest.main(verbosity=2) 75 | -------------------------------------------------------------------------------- /tiles2renderd.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3 2 | 3 | import sys 4 | import os.path 5 | from PIL import Image 6 | 7 | # Mod_tile / renderd store the rendered tiles 8 | # in "meta tiles" in a special hashed directory structure. These combine 9 | # 8x8 actual tiles into a single metatile file. The metatiles are stored 10 | # in the following directory structure: 11 | 12 | # /[base_dir]/[TileSetName]/[Z]/[xxxxyyyy]/[xxxxyyyy]/[xxxxyyyy]/[xxxxyyyy]/[xxxxyyyy].png 13 | 14 | # Where base_dir is a configurable base path for all tiles. TileSetName 15 | # is the name of the style sheet rendered. Z is the zoom level. 16 | # [xxxxyyyy] is an 8 bit number, with the first 4 bits taken from the x 17 | # coordinate and the second 4 bits taken from the y coordinate. This 18 | # attempts to cluster 16x16 square of tiles together into a single sub 19 | # directory for more efficient access patterns. 20 | 21 | #for (i=0; i<5; i++) { 22 | #hash[i] = ((x & 0x0f) << 4) | (y & 0x0f); 23 | #x >>= 4; 24 | #y >>= 4; 25 | #} 26 | def xyz_to_cache (x, y, z): 27 | h = [] 28 | 29 | for i in range(5): 30 | h.append( ((x & 0x0f) << 4) | (y & 0x0f) ) 31 | x >>= 4 32 | y >>= 4 33 | 34 | return h 35 | 36 | 37 | def generate_meta(tileset, col, row, z): 38 | if z in (0, 1, 2): 39 | tiles_in_meta = 1 << z 40 | else: 41 | tiles_in_meta = 8 42 | 43 | tiles = [] 44 | tile_count = 0 45 | 46 | for x in range(col, col + tiles_in_meta): 47 | for y in range(row, row + tiles_in_meta): 48 | tile_file = os.path.join (tileset, str(z), str(x), "%d.png" % y) 49 | if os.path.exists(tile_file): 50 | tiles.append(tile_file) 51 | tile_count += 1 52 | else: 53 | tiles.append('sea.png') 54 | 55 | if tile_count>0: 56 | # do not generate if it's all sea 57 | # TODO: how to handle this on mod_tile? 58 | dst = Image.new ('RGB', (256*tiles_in_meta, 256*tiles_in_meta)) 59 | 60 | for index, tile in enumerate(tiles): 61 | src = Image.open(tile) 62 | 63 | x = index // tiles_in_meta 64 | y = index % tiles_in_meta 65 | 66 | 67 | dst.paste(src, (x*256, y*256)) 68 | 69 | crumbs = xyz_to_cache(col, row, z) 70 | dst_path = os.path.join('/var/lib/mod_tile', tileset.lower(), str(z), 71 | *[ str (c) for c in crumbs ])+'.png' 72 | 73 | os.makedirs(os.path.dirname(dst_path), exist_ok=True) 74 | print (z, col, row, dst_path) 75 | dst.save(dst_path) 76 | 77 | 78 | tileset = sys.argv[1] 79 | # cache_dir = '/var/cache/mod_tile/%s' % tile_dir.lower() 80 | 81 | # TODO: read bboxes 82 | for z in range(19): 83 | if z in (0, 1, 2): 84 | mx = 1 85 | else: 86 | mx = 1 << (z-3) 87 | for col in range(mx): 88 | for row in range(mx): 89 | generate_meta(tileset, col*8, row*8, z) 90 | -------------------------------------------------------------------------------- /tile_server.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | from http import HTTPStatus 4 | from http.server import HTTPServer, SimpleHTTPRequestHandler 5 | import os 6 | import os.path 7 | import re 8 | import stat 9 | import sys 10 | 11 | import map_utils 12 | 13 | 14 | class TileServer(SimpleHTTPRequestHandler): 15 | def __init__(self, request, client, server, atlas): 16 | self.atlas = atlas 17 | super().__init__(request, client, server) 18 | 19 | 20 | def do_GET(self): 21 | try: 22 | # _ gets '' because self.path is absolute 23 | _, z, x, y_ext = self.path.split('/') 24 | except ValueError: 25 | self.send_error(HTTPStatus.BAD_REQUEST, f"bad tile spec '{self.path}' .") 26 | else: 27 | # TODO: make sure ext matches the file type we return 28 | # TODO: support JPEG 29 | y, ext = os.path.splitext(y_ext) 30 | 31 | tile = map_utils.Tile(*[ int(i) for i in (z, x, y) ]) 32 | 33 | # TODO: implement 'depth first'; that is, sort maps somehow 34 | # will need a tree, probably, beware of overlapping maps 35 | found = False 36 | for name, backend in self.atlas.items(): # 37 | if tile in backend: 38 | found = True 39 | break 40 | 41 | if found: 42 | if backend.fs_based: 43 | full_path = os.path.join(name, self.path) 44 | size = os.stat(full_path).st_size 45 | else: 46 | backend.fetch(tile) 47 | size = len(tile.data) 48 | 49 | self.send_response(HTTPStatus.OK) 50 | self.send_header('Content-Type', 'image/png') 51 | self.send_header('Content-Length', size) 52 | self.end_headers() 53 | 54 | if backend.fs_based: 55 | # create a socket to use high level sendfile() 56 | sender = socket.fromfd(self.wfile.raw.fileno()) 57 | sender.sendfile(open(full_path)) 58 | else: 59 | self.wfile.write(tile.data) 60 | else: 61 | self.send_error(HTTPStatus.NOT_FOUND, 'Tile not found.') 62 | 63 | 64 | def main(): 65 | maps = sys.argv[1:] 66 | atlas = {} 67 | 68 | # splitext() returns the leading dot 69 | mbt_exts = re.compile(r'\.(mbt|mbtiles|sqlite|sqlitedb)') 70 | 71 | for map in maps: 72 | basename, ext = os.path.splitext(map) 73 | 74 | if mbt_exts.match(ext) is not None: 75 | atlas[basename] = map_utils.MBTilesBackend(map, None, ro=True) 76 | 77 | elif stat.S_ISDIR(os.stat(map).st_mode): 78 | atlas[basename] = map_utils.DiskBackend(map) 79 | 80 | # TODO: use aio 81 | server = HTTPServer(('', 4847), lambda *a: TileServer(*a, atlas)) 82 | 83 | server.serve_forever() 84 | 85 | 86 | if __name__ == '__main__': 87 | main() 88 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import bisect 2 | import multiprocessing 3 | import statistics 4 | 5 | 6 | try: 7 | NUM_CPUS = multiprocessing.cpu_count() 8 | except NotImplementedError: 9 | NUM_CPUS = 1 10 | 11 | 12 | class MedianTracker: 13 | '''A statistics class ''' 14 | def __init__(self): 15 | self.items = [] 16 | 17 | 18 | def add(self, item): 19 | # TODO: why do they have to be in order? 20 | index = bisect.bisect(self.items, item) 21 | 22 | self.items.insert(index, item) 23 | 24 | 25 | def median(self): 26 | if len(self.items) == 0: 27 | return 0 28 | 29 | return statistics.median(self.items) 30 | 31 | 32 | def floor(i: int, base: int=1) -> int: 33 | '''Round i down to the closest multiple of base.''' 34 | return base * (i // base) 35 | 36 | 37 | def pyramid_count(min_zoom, max_zoom): 38 | '''Return the amount of tiles of the pyramid between ZLs min and max.''' 39 | # each pyramid level (ZL) i has 4**i tiles 40 | return sum([ 4**i for i in range(max_zoom - min_zoom + 1) ]) 41 | 42 | 43 | def time2hms(seconds: float): 44 | '''Converts time t in seconds into H/M/S.''' 45 | remaining_seconds = int(seconds) 46 | hours, remaining_seconds = divmod(remaining_seconds, 3600) 47 | # minutes, seconds = divmod(remaining_seconds, 60) 48 | minutes = remaining_seconds // 60 49 | _, seconds = divmod(seconds, 60) 50 | 51 | return (hours, minutes, seconds) 52 | 53 | 54 | def log_grafana(text): 55 | return None 56 | requests.post('http://localhost:3000/api/annotations', 57 | auth=('admin', 'Voo}s0zaetaeNgai'), 58 | json=dict( 59 | dashboardUID='e907aca7-ea55-4d54-9ade-819b32087f74', 60 | time=int(datetime.datetime.now().timestamp()*1000), 61 | text=text, 62 | ) 63 | ) 64 | 65 | 66 | class SimpleQueue: 67 | '''Class based on a list that implements the minimum needed to look like a 68 | *.Queue. The advantage is that there is no (de)serializing here.''' 69 | 70 | def __init__(self, size): 71 | self.queue = [] 72 | 73 | 74 | def get(self, block=True, timeout=None): 75 | if block: 76 | waited = 0.0 77 | 78 | while len(self.queue) == 0 and (timeout is None or waited < timeout): 79 | sleep(0.1) 80 | waited += 0.1 81 | 82 | return self.queue.pop(0) 83 | 84 | 85 | def put(self, value, block=True, timeout=None): 86 | # ignore block and timeout, making it a unbound queue 87 | # TODO: revisit? 88 | self.queue.append(value) 89 | 90 | 91 | def qsize(self): 92 | return len(self.queue) 93 | 94 | 95 | def remove(self, value): 96 | try: 97 | self.queue.remove(value) 98 | except ValueError: 99 | # if it's not present, it means it's being rendered 100 | pass 101 | 102 | 103 | -------------------------------------------------------------------------------- /atlas.ini: -------------------------------------------------------------------------------- 1 | # hint: use http://boundingbox.klokantech.com/ 2 | [maps] 3 | # w, s, e, n, max_z 4 | Test1=-170,10,-110,80,2 5 | Test2=10,-40,170,40,2 6 | RenderTest=7:07,44:10,7:11,44:12,18 7 | # MRS_test=5.37124,43.28394,5.37125,43.28395,19 8 | MRS_test=5.355886,43.225567,5.392771,43.24861,18 9 | 10 | Europe=-10,35,30,60,11 11 | BigEurope=-10,35,40,85,11 12 | Nice=5,43,10,46,18 13 | Antibes=7,43:30,7:30,44:30,18 14 | Marseille-Aix-Toulon=5:10,43,6,43:40,18 15 | # Calanques=5.1916667,43.0194447,5.6289642,43.2874327,18 16 | Calanques=5.2758,43.1634,5.5409,43.2969,18 17 | 18 | Margalef=0.5,41,2.5,41.85,18 19 | 20 | perrito=-64.6738,-31.0117,-64.4602,-30.7684,6 21 | 22 | Roma=12.25,41.75,12.75,42,18 23 | 24 | Azores=-31.3,36.8,-25.38,39.75,18 25 | Azores1=-26,36.8,-25,38,18 26 | Azores2=-29,38.8,-27,39.15,18 27 | Azores3=-31.3,39:22,-31,39:32,18 28 | Lisboa=-9.5,38.6,-9,39,18 29 | 30 | Iberia=-10,35,5,45,14 31 | Tarragona=1.21,41.08,1.28,41.15,18 32 | Almeria=-2.48,36.8,-2.41,36.86,18 33 | Granada=-3.61,37.15,-3.53,37.02,18 34 | Cordoba.es=-4.81,37.86,-4.75,37.9,18 35 | Sevilla=-6.03,37.36,-5.96,37.41,18 36 | 37 | Mockba=37:35,55:44,37:40,55:47,19 38 | Cahkt-TTetepbypr=30:10,59:53,30:27,60,19 39 | 40 | Bruxelles=4.265126,50.790514,4.733418,50.922718,19 41 | 42 | Aosta=7,45:20,8,46,19 43 | 44 | Orcieres=6,44:20,6:40,45,18 45 | Tignes=6.848259,45.399896,7.099571,45.550602,18 46 | 47 | Praha=14:23,50:03,14:27,50:06,18 48 | 49 | Ardeche=4:20,44:10,4:50,44:30,18 50 | 51 | Barcelona=1.75,41,2.25,42,18 52 | Kobenhavn=12.44,55.58,12.68,55.75,18 53 | 54 | Bern=7.419318,46.92836,7.487983,46.997832,18 55 | Amsterdam=4.719685,52.283085,5.013226,52.431329,18 56 | 57 | Berlin=13.27013,52.49,13.50302,52.65654,18 58 | 59 | GreatGreatLondon=-0.6169,51.0828,0.2922,51.9113,15 60 | London=-0.429469,51.330733,0.02303,51.552433,18 61 | 62 | Argentina=-73.6145250,-55.6829560,-53.5902380,-21.7257530,14 63 | # GreatCordoba=-64.8797,-32.4923,-63.1322,-30.6095,14 64 | GranCordoba=-65.087,-31.972,-64.041,-30.769,14 65 | Cordoba.ar=-64.361316,-31.485815,-64.087277,-31.248769,18 66 | # Cordoba.ar=-64.550487,-31.500976,-64.099361,-31.200156,18 67 | CaminoChosMalal=-71.3006,-39.1106,-67.7685,-36.6769,14 68 | ChosMalal=-70.344402,-37.410511,-70.223895,-37.341212,18 69 | 70 | RD=-74.525,17.4447,-68.2628,20.1382,14 71 | SantoDomingo=-69.994556,18.418402,-69.819804,18.512189,18 72 | 73 | MRS-dark=5.783,44.055,6.949,44.498,16 74 | Alpilles=4.6754,43.6819,5.0458,43.797,18 75 | 76 | Sicilia=11.85,36.6,16.87,38.92,16 77 | Palermo=13.3462,38.10518,13.38156,38.12477,18 78 | Catania=15.07444,37.49202,15.09779,37.51344,18 79 | Messina=15.51909,38.17907,15.57617,38.20072,18 80 | Milazzo=15.2202,38.211,15.2561,38.2753,18 81 | Siracusa=15.26943,37.05161,15.30368,37.08075,18 82 | Ragusa=14.72217,36.91944,14.75166,36.92994,18 83 | Corleone=13.28669,37.80319,13.31484,37.82478,18 84 | Cerfalù=14.01734,38.03225,14.04049,38.0419,18 85 | Taormina=15.2739510926,37.8421563975,15.3078971299,37.8600822464,18 86 | 87 | Island=-25,62.8,-12.5,67,16 88 | 89 | Pyrinees-Toulouse-Marseille-Monaco=-2.2,41.45,7.98,44.33,12 90 | 91 | Auvergne=2.372,45.337,3.352,45.9,16 92 | 93 | Avignon=4.7814,43.9356,4.826,43.9579,18 94 | 95 | GrandGrenoble=5.392,45.02,6.168,45.335,16 96 | 97 | Toulouse=1.3932,43.5681,1.4705,43.627,18 98 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # osm-tile-tools 2 | 3 | This repo contains several tools related to tile rendering and serving. 4 | 5 | ## generate_tiles.py 6 | 7 | This tool is a fork of the original script in [OSM's mapnik-style](https://github.com/openstreetmap/mapnik-stylesheets/blob/master/generate_tiles.py). 8 | 9 | It has been expanded to handle more situations, including: 10 | 11 | * Threaded vs Forking (the latter works better doe to a bug in mapnik). 12 | * Store tiles in several formats and directory schemas: 13 | * slippy map tiles 14 | * mod_tile 15 | * mbtiles 16 | * Render tiles older than a certain date. 17 | * Not storing empty sea tiles. 18 | * Render by bbox, LonLat coords, or bboxes stored in a config file. 19 | * And more! 20 | 21 | ``` 22 | usage: generate_tiles.py [-h] 23 | [-b W,S,E,N | -B BBOX_NAME | -T METATILE [METATILE ...] 24 | | -c COORDS | -L LONG LAT] [-n MIN_ZOOM] 25 | [-x MAX_ZOOM] [-i MAPFILE] 26 | [-f {tiles,mbtiles,mod_tile,test}] [-o TILE_DIR] 27 | [-m METATILE_SIZE] [-t THREADS] 28 | [-p {threads,fork,single}] [--store-thread] [-X] 29 | [-N DAYS] [--missing-as-new] [-E {skip,link,write}] 30 | [--debug] [-l LOG_FILE] [--dry-run] [--strict] 31 | 32 | optional arguments: 33 | -h, --help show this help message and exit 34 | -b W,S,E,N, --bbox W,S,E,N 35 | -B BBOX_NAME, --bbox-name BBOX_NAME 36 | -T METATILE [METATILE ...], --tiles METATILE [METATILE ...] 37 | METATILE can be in the form Z,X,Y or Z/X/Y. 38 | -c COORDS, --coords COORDS 39 | COORDS can be in form 'Lat,Lon', ´Lat/Lon'. 40 | -L LONG LAT, --longlat LONG LAT 41 | -n MIN_ZOOM, --min-zoom MIN_ZOOM 42 | -x MAX_ZOOM, --max-zoom MAX_ZOOM 43 | -i MAPFILE, --input-file MAPFILE 44 | MapnikXML format. 45 | -f {tiles,mbtiles,mod_tile,test}, --format {tiles,mbtiles,mod_tile,test} 46 | -o TILE_DIR, --output-dir TILE_DIR 47 | -m METATILE_SIZE, --metatile-size METATILE_SIZE 48 | Must be a power of two. 49 | -t THREADS, --threads THREADS 50 | -p {threads,fork,single}, --parallel-method {threads,fork,single} 51 | --store-thread Have a separate process/thread for storing the tiles. 52 | -X, --skip-existing 53 | -N DAYS, --skip-newer DAYS 54 | --missing-as-new missing tiles in a meta tile count as newer, so we 55 | don't re-render metatiles with empty tiles. 56 | -E {skip,link,write}, --empty {skip,link,write} 57 | --debug 58 | -l LOG_FILE, --log-file LOG_FILE 59 | --dry-run 60 | --strict Use Mapnik's strict mode. 61 | ``` 62 | 63 | ## tile_list.py 64 | 65 | Returns the list of slippy map tiles for a bbox and max ZL 66 | 67 | Example: 68 | 69 | ./tile_list.py 7.419318,46.92836,7.487983,46.997832,10 70 | 0/0/0.png 71 | 1/1/0.png 72 | 2/2/1.png 73 | 3/4/2.png 74 | 4/8/5.png 75 | 5/16/11.png 76 | 6/33/22.png 77 | 7/66/45.png 78 | 8/133/90.png 79 | 9/266/180.png 80 | 10/533/360.png 81 | 82 | ## tile_server.py 83 | 84 | It serves slippy map tiles over http, using several sources than can b either in 85 | slippy map format or MBTiles. Each tile is search on each source until found, so 86 | you can stack several sources like that. 87 | 88 | I developed this so I could serve a country size map down to ZL11 plus several 89 | city-and-about maps down to ZL18. 90 | 91 | Example: 92 | 93 | ./tile_server.py Argentina.sqlitedb Cordoba.sqlitedb # OsmAnd MBTiles format 94 | 95 | # TODO 96 | 97 | The rest. 98 | -------------------------------------------------------------------------------- /tiles.py: -------------------------------------------------------------------------------- 1 | from math import pi, cos, sin, log, exp, atan 2 | from typing import List, Tuple, Dict, Optional, Any, Union 3 | 4 | from shapely.geometry import Polygon 5 | 6 | 7 | # DEG_TO_RAD:float = pi / 180 8 | # RAD_TO_DEG:float = 180 / pi 9 | DEG_TO_RAD = pi / 180 10 | RAD_TO_DEG = 180 / pi 11 | 12 | 13 | def constrain(lower_limit:float, x:float, upper_limit:float) -> float: 14 | """Constrains x to the [lower_limit, upper_limit] segment.""" 15 | ans = max(lower_limit, x) 16 | ans = min(ans, upper_limit) 17 | 18 | return ans 19 | 20 | 21 | class GoogleProjection: 22 | """ 23 | This class converts from LonLat to pixel and vice versa. For that, it pre 24 | calculates some values for each zoom level, which are store in 3 arrays. 25 | 26 | For information about the formulas in lon_lat2pixel() and pixel2lon_lat(), see 27 | https://en.wikipedia.org/wiki/Mercator_projection#Mathematics_of_the_Mercator_projection 28 | """ 29 | 30 | # see also https://alastaira.wordpress.com/2011/01/23/the-google-maps-bing-maps-spherical-mercator-projection/ 31 | 32 | def __init__(self, levels:int=18) -> None: 33 | self.pixels_per_degree:List[float] = [] 34 | self.pixels_per_radian:List[float] = [] 35 | self.center_pixel:List[Tuple[int, int]] = [] # pixel for (0, 0) 36 | # self.world_size:List[int] = [] # world size in pixels 37 | 38 | # NOTE: no support for high res tiles 39 | world_size:int = 256 # size in pixels of the image representing the whole world 40 | for d in range(levels + 1): # type: int 41 | center:int = world_size // 2 42 | self.pixels_per_degree.append(world_size / 360.0) 43 | self.pixels_per_radian.append(world_size / (2 * pi)) 44 | self.center_pixel.append( (center, center) ) 45 | # the world doubles in size on each zoom level 46 | world_size *= 2 47 | 48 | 49 | # it's LonLat! (lon, lat) 50 | def lon_lat2pixel(self, lon_lat:Tuple[float, float], zoom:int) -> Tuple[int, int]: 51 | lon, lat = lon_lat 52 | center_x, center_y = self.center_pixel[zoom] 53 | 54 | # x is easy because it's linear to the longitude 55 | x = center_x + round(lon * self.pixels_per_degree[zoom]) 56 | 57 | # y is... what? 58 | f = constrain(-0.9999, sin(DEG_TO_RAD * lat), 0.9999) 59 | y = center_y + round(0.5 * log((1 + f) / (1 - f)) * -self.pixels_per_radian[zoom]) 60 | 61 | return (x, y) 62 | 63 | 64 | def pixel2lon_lat(self, px:Tuple[int, int], zoom:int) -> Tuple[float,float]: 65 | x, y = px 66 | center_x, center_y = self.center_pixel[zoom] 67 | 68 | # longitude is linear to x 69 | lon = (x - center_x) / self.pixels_per_degree[zoom] 70 | 71 | angle = (y - center_y) / -self.pixels_per_radian[zoom] # angle in radians 72 | lat = RAD_TO_DEG * (2 * atan(exp(angle)) - 0.5 * pi) 73 | 74 | return (lon, lat) 75 | 76 | # enough ZLs for a lifetime 77 | tileproj = GoogleProjection(30) 78 | 79 | 80 | class Tile: 81 | # def __init__(self, z:int, x:int, y:int, metatile:Optional[MetaTile]=None) -> None: 82 | def __init__(self, z:int, x:int, y:int, metatile=None) -> None: 83 | # NOTE: there are 3 sets of coordinates and their vertical component grow in different directions 84 | 85 | # (z,),x,y are tile coords, relative to the upper left corner, so y grows downwards 86 | self.z = z 87 | self.x = x 88 | self.y = y 89 | 90 | # self.meta_index:Optional[Tuple[int, int]] = None 91 | self.meta_index = None 92 | self.meta_pixel_coords = None 93 | if metatile is not None: 94 | self.meta_index = (x - metatile.x, y - metatile.y) 95 | self.meta_pixel_coords = () 96 | self.size = metatile.tile_size 97 | else: 98 | # TODO: no support for hi res tiles 99 | self.size = 256 100 | 101 | # pixel_pos is based on (z,),x,y; it's relative to the world at this ZL and also grows downwards 102 | self.pixel_pos = (self.x * self.size, self.y * self.size) 103 | 104 | # corners are another way to express pixel_pos; same direction 105 | # ((x0, y0), (x1, y1)) 106 | self.corners = ( self.pixel_pos, 107 | (self.pixel_pos[0] + self.size, 108 | self.pixel_pos[1] + self.size) ) 109 | 110 | # but coords are LongLat, and Lat grows upwards, so wehn calling these functions, 111 | # we have to swap the lat's 112 | long0, lat1 = tileproj.pixel2lon_lat(self.corners[0], self.z) 113 | long1, lat0 = tileproj.pixel2lon_lat(self.corners[1], self.z) 114 | self.coords = ( (long0, lat0), (long1, lat1) ) 115 | 116 | polygon_points = [ (self.coords[i][0], self.coords[j][1]) 117 | for i, j in ((0, 0), (1, 0), (1, 1), (0, 1), (0, 0)) ] 118 | 119 | self.polygon = Polygon(polygon_points) 120 | 121 | self.image_size = (self.size, self.size) 122 | self.data: Optional[bytes] = None 123 | self.is_empty = None # Optional[bool] 124 | 125 | 126 | def __eq__(self, other): 127 | return ( self.z == other.z and self.x == other.x and self.y == other.y ) 128 | 129 | 130 | def __repr__(self): 131 | return "Tile(%d, %d, %d, %r)" % (self.z, self.x, self.y, self.meta_index) 132 | 133 | 134 | def __iter__(self): 135 | return self.iter() 136 | 137 | 138 | def iter(self): 139 | """Returns a generator over the 'coords'.""" 140 | yield self.z 141 | yield self.x 142 | yield self.y 143 | 144 | 145 | class PixelTile: 146 | """It's a (meta) tile with arbitrary pixel bounds.""" 147 | def __init__(self, z, center_x, center_y, size): 148 | self.z = z 149 | # keep the original coords, mostly for printing; meta_index is used for cutting 150 | self.x = center_x 151 | self.y = center_y 152 | 153 | # make sure it's rendered 154 | self.is_empty = False 155 | self.render = True 156 | 157 | half_size = size // 2 158 | # (x, y) 159 | self.pixel_pos = (center_x - half_size, center_y - half_size) 160 | # debug(self.pixel_pos) 161 | # (w, h) 162 | self.image_size = (size, size) 163 | # debug(self.image_size) 164 | 165 | self.tiles = [ self ] 166 | 167 | # ((x0, y0), (x1, y1)) 168 | self.corners = ( self.pixel_pos, 169 | (self.pixel_pos[0] + self.image_size[0], 170 | self.pixel_pos[1] + self.image_size[1]) ) 171 | 172 | # we have to swap the lat's 173 | long0, lat1 = tileproj.pixel2lon_lat(self.corners[0], self.z) 174 | long1, lat0 = tileproj.pixel2lon_lat(self.corners[1], self.z) 175 | self.coords = ( (long0, lat0), (long1, lat1) ) 176 | 177 | polygon_points = [ (self.coords[i][0], self.coords[j][1]) 178 | for i, j in ((0, 0), (1, 0), (1, 1), (0, 1), (0, 0)) ] 179 | 180 | self.polygon = Polygon(polygon_points) 181 | 182 | # times 183 | self.render_time = 0 184 | self.serializing_time = 0 185 | self.deserializing_time = 0 186 | self.saving_time = 0 187 | 188 | # Tile emulation 189 | self.meta_index = (0, 0) 190 | 191 | 192 | def __repr__(self) -> str: 193 | return "PixelTile(%d,%d,%d)" % (self.z, self.x, self.y) 194 | 195 | 196 | def child(self, tile:Tile): 197 | return None 198 | 199 | 200 | def children(self): 201 | return [] 202 | 203 | 204 | # TODO: move to BaseTile 205 | def __iter__(self): 206 | return self.iter() 207 | 208 | 209 | def iter(self): 210 | """Returns a generator over the 'coords'.""" 211 | yield self.z 212 | yield self.x 213 | yield self.y 214 | 215 | 216 | def times(self): 217 | return (self.render_time, self.serializing_time, self.deserializing_time, 218 | self.saving_time) 219 | 220 | # TODO: MetaTile factory 221 | 222 | # Children = List[MetaTile] 223 | class MetaTile: 224 | def __init__(self, z:int, x:int, y:int, wanted_size:int, tile_size:int) -> None: 225 | self.z = z 226 | self.x = x 227 | self.y = y 228 | 229 | self.wanted_size = wanted_size # in tiles 230 | self.size = min(2**z, wanted_size) 231 | self.tile_size = tile_size 232 | 233 | self.is_empty = True # to simplify code in store_metatile() 234 | self.render = True # this is going to be reset by store_metatile() 235 | 236 | # NOTE: children are not precomputed because it's recursive with no bounds 237 | # see children() 238 | # self._children:Optional[Children] = None 239 | self._children = None 240 | 241 | self.tiles = [ Tile(self.z, self.x + i, self.y + j, self) 242 | for i in range(self.size) for j in range(self.size) ] 243 | 244 | self.im: Optional[bytes] = None 245 | 246 | # (x, y) 247 | self.pixel_pos = (self.x * self.tile_size, self.y * self.tile_size) 248 | # (w, h) 249 | self.image_size = (self.size * self.tile_size, self.size * self.tile_size) 250 | 251 | # ((x0, y0), (x1, y1)) 252 | self.corners = ( self.pixel_pos, 253 | (self.pixel_pos[0] + self.image_size[0], 254 | self.pixel_pos[1] + self.image_size[1]) ) 255 | 256 | # we have to swap the lat's 257 | long0, lat1 = tileproj.pixel2lon_lat(self.corners[0], self.z) 258 | long1, lat0 = tileproj.pixel2lon_lat(self.corners[1], self.z) 259 | self.coords = ( (long0, lat0), (long1, lat1) ) 260 | 261 | polygon_points = [ (self.coords[i][0], self.coords[j][1]) 262 | for i, j in ((0, 0), (1, 0), (1, 1), (0, 1), (0, 0)) ] 263 | 264 | self.polygon = Polygon(polygon_points) 265 | 266 | # times 267 | self.render_time:float = 0 268 | self.serializing_time:float = 0 269 | self.deserializing_time = 0 270 | self.saving_time = 0 271 | 272 | @staticmethod 273 | def from_tile(tile: Tile, wanted_size): 274 | size = min(2**tile.z, wanted_size) 275 | x = tile.x // size * size 276 | y = tile.y // size * size 277 | 278 | return MetaTile(tile.z, x, y, wanted_size, tile.size) 279 | 280 | 281 | # see https://github.com/python/mypy/issues/2783#issuecomment-276596902 282 | # def __eq__(self, other:MetaTile) -> bool: # type: ignore 283 | def __eq__(self, other) -> bool: # type: ignore 284 | return ( self.z == other.z and self.x == other.x and self.y == other.y 285 | and self.size == other.size ) 286 | 287 | 288 | def __repr__(self) -> str: 289 | return "MetaTile(%d,%d,%d)" % (self.z, self.x, self.y) 290 | 291 | 292 | # def children(self) -> Children: 293 | def children(self): 294 | if self._children is None: 295 | if self.size == self.wanted_size: 296 | self._children = [ MetaTile(self.z + 1, 297 | 2 * self.x + i * self.size, 298 | 2 * self.y + j * self.size, 299 | self.wanted_size, self.tile_size) 300 | for i in range(2) for j in range(2) ] 301 | else: 302 | self._children = [ MetaTile(self.z + 1, 2 * self.x, 2 * self.y, 303 | self.wanted_size, self.tile_size) ] 304 | debug((self, self._children)) 305 | 306 | return self._children 307 | 308 | 309 | def __contains__(self, other:Tile) -> bool: 310 | if isinstance(other, Tile): 311 | if other.z == self.z-1: 312 | return ( self.x <= 2*other.x < self.x+self.size and 313 | self.y <= 2*other.y < self.y+self.size ) 314 | else: 315 | return ( self.x <= other.x < self.x+self.size and 316 | self.y <= other.y < self.y+self.size ) 317 | else: 318 | # TODO: more? 319 | return False 320 | 321 | 322 | # def child(self, tile:Tile) -> MetaTile: 323 | def child(self, tile:Tile): 324 | """Returns the child MetaTile were tile fits.""" 325 | if tile in self: 326 | # there's only one 327 | return [ child for child in self.children() if tile in child ][0] 328 | else: 329 | raise ValueError("%r not in %r" % (tile, self)) 330 | 331 | 332 | def __hash__(self): 333 | return hash((self.z, self.x, self.y, self.size)) 334 | 335 | 336 | def times(self): 337 | return (self.render_time, self.serializing_time, self.deserializing_time, 338 | self.saving_time) 339 | 340 | 341 | def tile_spec2zxy(tile_spec): # str -> Tuple[int, int, int] 342 | if ',' in tile_spec: 343 | data = tile_spec.split(',') 344 | elif '/' in tile_spec: 345 | data = tile_spec.split('/') 346 | else: 347 | raise ValueError("METATILE not in form Z,X,Y or Z/X/Y.") 348 | 349 | z, x, y = map(int, data) 350 | 351 | return (z, x, y) 352 | -------------------------------------------------------------------------------- /map_utils.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | 3 | from math import pi, cos, sin, log, exp, atan 4 | from configparser import ConfigParser 5 | import os.path 6 | from os.path import dirname, basename, join as path_join 7 | from os import listdir, unlink, mkdir, walk, makedirs 8 | import os 9 | from errno import ENOENT, EEXIST 10 | from shutil import copy, rmtree 11 | import datetime 12 | import errno 13 | import hashlib 14 | import sqlite3 15 | import stat 16 | 17 | from shapely.geometry import Polygon 18 | from shapely import wkt 19 | 20 | from tiles import GoogleProjection, Tile, MetaTile, PixelTile 21 | 22 | from logging import debug 23 | from typing import List, Tuple, Dict, Optional, Any, Union 24 | 25 | 26 | # helper types 27 | TileOrTuple = Union[Tile, Tuple[int, int, int]] 28 | 29 | class DiskBackend: 30 | fs_based = True 31 | 32 | def __init__(self, base:str, *more, **even_more): 33 | self.base_dir = base 34 | self.filename_pattern = even_more.get('filename_pattern', 35 | '{base_dir}/{z}/{x}/{y}.png') 36 | 37 | 38 | def tile_uri(self, tile: TileOrTuple) -> str: 39 | # this works because I made Tile iterable 40 | z, x, y = ( str(i) for i in tile ) 41 | base_dir = self.base_dir 42 | 43 | return self.filename_pattern.format(**locals()) 44 | 45 | 46 | def store(self, tile: Tile): 47 | assert tile.data is not None 48 | 49 | tile_uri = self.tile_uri(tile) 50 | makedirs(os.path.dirname(tile_uri), exist_ok=True) 51 | f = open(tile_uri, 'wb+') 52 | f.write(tile.data) 53 | f.close() 54 | 55 | 56 | def exists(self, tile: TileOrTuple): 57 | tile_uri = self.tile_uri(tile) 58 | return os.path.isfile(tile_uri) 59 | 60 | 61 | def fetch(self, tile: Tile): 62 | tile_uri = self.tile_uri(tile) 63 | try: 64 | print(tile_uri) 65 | f = open(tile_uri, 'br') 66 | except OSError as e: 67 | print(e) 68 | return False 69 | else: 70 | tile.data = f.read() 71 | f.close() 72 | 73 | return True 74 | 75 | 76 | def newer_than(self, tile: TileOrTuple, date, missing_as_new): 77 | tile_uri = self.tile_uri(tile) 78 | try: 79 | file_date = datetime.datetime.fromtimestamp(os.stat(tile_uri).st_mtime) 80 | # debug("%s: %s <-> %s", tile_uri, file_date.isoformat(), 81 | # date.isoformat()) 82 | return file_date > date 83 | except OSError as e: 84 | if e.errno == errno.ENOENT: 85 | # debug("%s: %s", tile_uri, missing_as_new) 86 | return missing_as_new 87 | else: 88 | raise 89 | 90 | 91 | def commit(self): 92 | # TODO: flush? 93 | pass 94 | 95 | 96 | def __contains__(self, tile: TileOrTuple): 97 | return self.exists(tile) 98 | 99 | 100 | class ModTileBackend(DiskBackend): 101 | def tile_uri(self, tile: TileOrTuple): 102 | # The metatiles are then stored 103 | # in the following directory structure: 104 | # /[base_dir]/[TileSetName]/[Z]/[xxxxyyyy]/[xxxxyyyy]/[xxxxyyyy]/[xxxxyyyy]/[xxxxyyyy].png 105 | # Where base_dir is a configurable base path for all tiles. TileSetName 106 | # is the name of the style sheet rendered. Z is the zoom level. 107 | # [xxxxyyyy] is an 8 bit number, with the first 4 bits taken from the x 108 | # coordinate and the second 4 bits taken from the y coordinate. This 109 | # attempts to cluster 16x16 square of tiles together into a single sub 110 | # directory for more efficient access patterns. 111 | z, x, y = tile 112 | 113 | crumbs: List[str] = [] 114 | for crumb_index in range(5): 115 | x, x_bits = divmod(x, 16) 116 | y, y_bits = divmod(y, 16) 117 | debug((x, x_bits, y, y_bits)) 118 | 119 | crumb = (x_bits << 4) + y_bits 120 | crumbs.insert(0, str(crumb)) 121 | 122 | return os.path.join(self.base_dir, str(z), *crumbs[:-1], crumbs[-1] + '.png') 123 | 124 | 125 | class TestBackend(DiskBackend): 126 | def __init__(self, base:str, *more, **even_more): 127 | self.base_dir = base 128 | self.filename_pattern = even_more.get('filename_pattern', 129 | '{base_dir}/{z}-{x}-{y}.png') 130 | 131 | 132 | # https://github.com/mapbox/node-mbtiles/blob/master/lib/schema.sql 133 | # https://github.com/mapbox/mbutil/blob/master/mbutil/util.py 134 | 135 | # according to the spec https://github.com/mapbox/mbtiles-spec/blob/master/1.2/spec.md 136 | # the schemas outlined are meant to be followed as interfaces. 137 | # SQLite views that produce compatible results are equally valid. 138 | # For convenience, this specification refers to tables and virtual tables (views) as tables. 139 | 140 | # so what happens with the tiles tables is exactly that: 141 | # it's implemented as a (read only) view on top of map an images 142 | # but internally we fill them separately 143 | 144 | class MBTilesBackend: 145 | fs_based = False 146 | 147 | # .sqlitedb 'cause I'll use it primarily for OsmAnd 148 | def __init__(self, path, bounds, min_zoom=0, max_zoom=18, center=None, ro=False): 149 | self.path = path 150 | 151 | if ro: 152 | spec = 'file:' + self.path + '?mode=ro' 153 | # print(spec) 154 | self.session = sqlite3.connect(spec, uri=True) 155 | else: 156 | self.session = sqlite3.connect(self.path) 157 | self.session.set_trace_callback(print) 158 | 159 | if not stat.S_ISREG(os.stat(self.path).st_mode): 160 | # create the db 161 | self.init() 162 | 163 | 164 | def init(self): 165 | cursor = self.session.cursor() 166 | # mbtiles 167 | cursor.execute('''CREATE TABLE IF NOT EXISTS metadata( 168 | name VARCHAR NOT NULL, 169 | value VARCHAR, 170 | PRIMARY KEY (name) 171 | );''') 172 | 173 | # OsmAnd 174 | # see https://github.com/osmandapp/Osmand/blob/master/OsmAnd/src/net/osmand/plus/SQLiteTileSource.java#L179 175 | # and https://osmand.net/help-online/technical-articles#OsmAnd_SQLite_Spec 176 | cursor.execute('''CREATE TABLE IF NOT EXISTS info( 177 | -- all these are optional 178 | -- rule VARCHAR, 179 | -- referer VARCHAR, 180 | -- timecolumn VARCHAR, 181 | -- expireminutes INTEGER, 182 | -- ellipsoid INTEGER, 183 | url VARCHAR, 184 | -- this one is important so we don't get constrained by BigPlanet, 185 | -- which has max_z == 17 186 | tilenumbering VARCHAR, 187 | minzoom INTEGER NOT NULL, 188 | maxzoom INTEGER NOT NULL 189 | );''') 190 | 191 | # common 192 | cursor.execute('''CREATE TABLE IF NOT EXISTS map( 193 | zoom_level INTEGER NOT NULL, 194 | tile_column INTEGER NOT NULL, 195 | tile_row INTEGER NOT NULL, 196 | tile_id VARCHAR(32), 197 | CONSTRAINT map_index PRIMARY KEY (zoom_level, tile_column, tile_row) 198 | );''') 199 | 200 | cursor.execute('''CREATE TABLE IF NOT EXISTS images( 201 | tile_id VARCHAR(32) NOT NULL, 202 | tile_data BLOB, 203 | PRIMARY KEY (tile_id) 204 | );''') 205 | 206 | # see https://gist.github.com/rzymek/034ef469fa01fdd592a6aadde76e95fa 207 | # just ignore the info about inverted y/tile_row 208 | cursor.execute('''CREATE VIEW IF NOT EXISTS tiles AS 209 | SELECT 210 | -- mbtiles 211 | map.zoom_level AS zoom_level, 212 | map.tile_column AS tile_column, 213 | map.tile_row AS tile_row, 214 | images.tile_data AS tile_data, 215 | 216 | -- OsmAnd 217 | map.zoom_level AS z, 218 | map.tile_column AS x, 219 | map.tile_row AS y, 220 | 0 AS s, -- TODO: check where does this 's' come from 221 | images.tile_data AS image 222 | FROM 223 | map JOIN images 224 | ON images.tile_id = map.tile_id;''') 225 | 226 | self.session.commit() 227 | 228 | #version|1.0.0 229 | #center|-18.7,65,7 230 | #attribution|Map data © OpenStreetMap CC-BY-SA; NASA SRTM; GLISM Glacier Database; ETOPO1 Bathymetry 231 | #description|Inspired by the 1975 map of Iceland by the Danish Geodetisk Institut. 232 | 233 | # generate metadata 234 | metadata = dict( 235 | name ='Elevation', 236 | type ='baselayer', 237 | version ='2.39.0-04f6d1b', # I wonder why git uses only 7 chars by default 238 | description ="StyXman's simple map", 239 | format ='png', 240 | bounds =','.join([ str(i) for i in bounds ]), 241 | attribution ='Map data © OpenStreetMap CC-BY-SA; NASA SRTM', 242 | ) 243 | 244 | for k, v in metadata.items(): 245 | try: 246 | cursor.execute('''INSERT INTO metadata(name, value) VALUES (?, ?);''', 247 | (k, v)) 248 | except sqlite3.IntegrityError: 249 | cursor.execute('''UPDATE metadata SET value = ? WHERE name = ?;''', 250 | (v, k)) 251 | 252 | # info for OsmAnd 253 | cursor.execute('''INSERT INTO info(url, tilenumbering, minzoom, maxzoom) VALUES (?, ?, ?, ?);''', 254 | ("http://dionecanali.hd.free.fr/~mdione/Elevation/$1/$2/$3.png", "normal", 255 | min_zoom, max_zoom)) 256 | 257 | # TODO: replace by (1 << 8) -1 ? 258 | for z in range(max_zoom + 1): 259 | cursor.execute('''INSERT INTO max_y(z, y) VALUES (?, ?);''', (z, 2**z - 1)) 260 | 261 | self.session.commit() 262 | self.session.set_trace_callback(None) 263 | 264 | cursor.close() 265 | 266 | 267 | def store (self, tile: Tile): 268 | assert tile.data is not None 269 | 270 | self.store_raw(tile.z, tile.x, tile.y, tile.data) 271 | 272 | 273 | def store_raw (self, z: int, x: int, y: int, image: bytes): 274 | # create one of these each time because there's no way to reset them 275 | # and barely takes any time 276 | hasher = hashlib.md5() 277 | 278 | # md5 gives 340282366920938463463374607431768211456 possible values 279 | # and is *fast* 280 | hasher.update(image) 281 | 282 | # thanks Pablo Carranza for pointing out possible collisions 283 | # further deduplicate with file length 284 | hasher.update(str(len(image)).encode('ascii')) 285 | img_id = hasher.hexdigest() 286 | 287 | debug((tile, img_id)) 288 | 289 | cursor = self.session.cursor () 290 | try: 291 | cursor.execute ('''INSERT INTO images (tile_id, tile_data) VALUES (?, ?);''', 292 | (img_id, image)) 293 | except sqlite3.IntegrityError: 294 | # it already exists and there's no reason to try to update anything 295 | pass 296 | 297 | try: 298 | cursor.execute ('''INSERT INTO map (zoom_level, tile_column, tile_row, tile_id) 299 | VALUES (?, ?, ?, ?);''', 300 | (z, x, y, img_id)) 301 | except sqlite3.IntegrityError: 302 | cursor.execute ('''UPDATE map 303 | SET tile_id = ? 304 | WHERE zoom_level = ? 305 | AND tile_column = ? 306 | AND tile_row = ?;''', 307 | (img_id, z, x, y)) 308 | cursor.close () 309 | 310 | 311 | def commit (self): 312 | if self.session.in_transaction: 313 | self.session.commit () 314 | 315 | 316 | def exists (self, tile: TileOrTuple): 317 | cursor= self.session.cursor () 318 | data= cursor.execute('''SELECT count(map.zoom_level) 319 | FROM map 320 | WHERE map.zoom_level = ? 321 | AND map.tile_column = ? 322 | AND map.tile_row = ?;''', 323 | tuple(tile)).fetchall() 324 | 325 | return data[0][0] == 1 326 | 327 | 328 | def fetch(self, tile: Tile): 329 | print(tile) 330 | cursor = self.session.cursor() 331 | data = cursor.execute('''SELECT tile_data 332 | FROM tiles 333 | WHERE tiles.z = ? 334 | AND tiles.x = ? 335 | AND tiles.y = ?;''', 336 | tuple(tile)).fetchall() 337 | 338 | if len(data) == 0: 339 | return False 340 | 341 | tile.data = data[0][0] 342 | return True 343 | 344 | 345 | # TODO: newer_than() 346 | 347 | 348 | def close (self): 349 | self.session.close () 350 | 351 | 352 | def __contains__(self, tile: TileOrTuple): 353 | return self.exists(tile) 354 | 355 | 356 | def coord_range(mn, mx, zoom): 357 | return ( coord for coord in range(mn, mx + 1) 358 | if coord >= 0 and coord < 2**zoom ) 359 | 360 | 361 | def bbox(value): 362 | data = value.split(',') 363 | for index, deg in enumerate(data): 364 | try: 365 | deg = float(deg) 366 | except ValueError: 367 | # let's try with x:y[:z] 368 | d = deg.split(':') 369 | if len(d) == 2: 370 | d.append('0') 371 | 372 | deg, mn, sec = [ int(x) for x in d ] 373 | deg = deg + 1/60.0*mn + 1/3600.0*sec 374 | 375 | data[index] = deg 376 | 377 | return data 378 | 379 | 380 | class BBox: 381 | def __init__(self, bbox, max_z): 382 | self.w, self.s, self.e, self.n = bbox 383 | self.max_z = max_z 384 | self.proj = GoogleProjection(self.max_z+1) # +1 385 | 386 | # it's LonLat! (x, y) 387 | self.upper_left = (self.w, self.n) 388 | self.lower_left = (self.w, self.s) 389 | self.upper_right = (self.e, self.n) 390 | self.lower_right = (self.e, self.s) 391 | 392 | # in degrees 393 | self.boundary = Polygon([ self.lower_left, self.lower_right, 394 | self.upper_right, self.upper_left, 395 | self.lower_left ]) 396 | 397 | 398 | def __contains__(self, tile): 399 | other = tile.polygon 400 | 401 | debug((other.wkt, self.boundary.wkt)) 402 | return other.intersects(self.boundary) 403 | 404 | 405 | def __repr__(self): 406 | return "BBox(%f, %f, %f, %f, %d)" % (self.w, self.s, self.e, self.n, 407 | self.max_z) 408 | 409 | 410 | # TODO: see if it doesn't make more sense to work everything at pixel level 411 | 412 | 413 | class Map: 414 | def __init__ (self, bbox, max_z): 415 | self.bbox = bbox 416 | self.max_z = max_z 417 | # TODO: 418 | self.tile_size = 256 419 | 420 | ll0 = (bbox[0],bbox[3]) 421 | ll1 = (bbox[2],bbox[1]) 422 | gprj = GoogleProjection(max_z+1) 423 | 424 | self.levels = [] 425 | for z in range (0, max_z+1): 426 | px0 = gprj.lon_lat2pixel(ll0,z) 427 | px1 = gprj.lon_lat2pixel(ll1,z) 428 | # print px0, px1 429 | self.levels.append (( (int (px0[0]/self.tile_size), int (px0[1]/self.tile_size)), 430 | (int (px1[0]/self.tile_size), int (px1[1]/self.tile_size)) )) 431 | 432 | 433 | def __contains__ (self, t): 434 | if len (t) == 3: 435 | z, x, y = t 436 | px0, px1= self.levels[z] 437 | # print (z, px0[0], x, px1[0], px0[1], y, px1[1]) 438 | ans = px0[0] <= x and x <= px1[0] 439 | # print ans 440 | ans = ans and px0[1] <= y and y <= px1[1] 441 | # print ans 442 | elif len (t) == 2: 443 | z, x = t 444 | px0, px1 = self.levels[z] 445 | # print (z, px0[0], x, px1[0]) 446 | ans = px0[0] <= x and x <= px1[0] 447 | else: 448 | raise ValueError 449 | 450 | return ans 451 | 452 | 453 | def iterate_x (self, z): 454 | px0, px1 = self.levels[z] 455 | return coord_range(px0[0], px1[0], z) # NOTE 456 | 457 | 458 | def iterate_y (self, z): 459 | px0, px1 = self.levels[z] 460 | return coord_range(px0[1], px1[1], z) # NOTE 461 | 462 | 463 | class Atlas: 464 | def __init__(self, maps): 465 | self.maps = {} 466 | atlas_config = ConfigParser() 467 | atlas_config.read('atlas.ini') 468 | self.minZoom = 0 469 | self.maxZoom = 0 470 | 471 | for map in maps: 472 | bb = bbox(atlas_config.get('maps', map)) 473 | # #4 is the max_z 474 | if bb[4] > self.maxZoom: 475 | self.maxZoom = int(bb[4]) 476 | 477 | for map in maps: 478 | bb= bbox (atlas_config.get ('maps', map)) 479 | self.maps[map]= Map (bb[:4], self.maxZoom) 480 | 481 | 482 | def __contains__ (self, t): 483 | w= False 484 | for m in self.maps.values (): 485 | w= w or t in m 486 | 487 | return w 488 | 489 | 490 | def iterate_x (self, z): 491 | for m in self.maps.values (): 492 | for x in m.iterate_x (z): 493 | yield x 494 | 495 | 496 | def iterate_y (self, z, x): 497 | for m in self.maps.values (): 498 | if (z, x) in m: 499 | for y in m.iterate_y (z): 500 | yield y 501 | 502 | 503 | def test_all(): 504 | # import logging 505 | # logging.basicConfig(level=logging.DEBUG) 506 | 507 | # Europe 508 | b = BBox((-10, 35, 30, 60), 18) 509 | 510 | def test(bbox, z, expected): 511 | # it's a square 512 | tiles = len(expected) 513 | for x in range(tiles): 514 | for y in range(tiles): 515 | t = Tile(z, x, y) 516 | result = t in bbox 517 | # NOTE: if you look at how expected is defined, this is the 518 | # right order of 'coords' 519 | if result != expected[y][x]: 520 | if expected[x][y]: 521 | raise AssertionError("%r not in %r" % (t, bbox)) 522 | else: 523 | raise AssertionError("%r in %r" % (t, bbox)) 524 | 525 | # ZL0 526 | expected = [ [ True ] ] 527 | test(b, 0, expected) 528 | 529 | # ZL1 530 | expected = [ [ True, True ], 531 | [ False, False ] ] 532 | test(b, 1, expected) 533 | 534 | # ZL2 535 | expected = [ [ False, False, False, False ], 536 | [ False, True, True , False ], 537 | [ False, False, False, False ], 538 | [ False, False, False, False ] ] 539 | test(b, 2, expected) 540 | 541 | # ZL3 542 | expected = [ [ False, False, False, False, False, False, False, False ], 543 | [ False, False, False, False, False, False, False, False ], 544 | [ False, False, False, True, True , False, False, False ], 545 | [ False, False, False, True, True , False, False, False ], 546 | [ False, False, False, False, False, False, False, False ], 547 | [ False, False, False, False, False, False, False, False ], 548 | [ False, False, False, False, False, False, False, False ], 549 | [ False, False, False, False, False, False, False, False ] ] 550 | test(b, 3, expected) 551 | 552 | g = GoogleProjection(18) 553 | for z in range(19): 554 | side = 256 * 2**z 555 | middle = side // 2 556 | 557 | x, y = g.lon_lat2pixel((0, 0), z) 558 | 559 | assert x == middle 560 | assert y == middle 561 | 562 | lon, lat = g.pixel2lon_lat((x, y), z) 563 | 564 | assert lon == 0 565 | assert lat == 0 566 | 567 | 568 | tile = Tile(0, 0, 0) 569 | assert MetaTile.from_tile(tile, 8) == MetaTile(0, 0, 0, 8, 256) 570 | 571 | tile = Tile(1, 1, 0) 572 | assert MetaTile.from_tile(tile, 8) == MetaTile(1, 0, 0, 8, 256) 573 | 574 | tile = Tile(2, 2, 1) 575 | assert MetaTile.from_tile(tile, 8) == MetaTile(2, 0, 0, 8, 256) 576 | 577 | tile = Tile(3, 4, 2) 578 | assert MetaTile.from_tile(tile, 8) == MetaTile(3, 0, 0, 8, 256) 579 | 580 | tile = Tile(4, 7, 8) 581 | assert MetaTile.from_tile(tile, 8) == MetaTile(4, 0, 8, 8, 256) 582 | 583 | print('A-OK') 584 | 585 | 586 | if __name__ == '__main__': 587 | test_all() 588 | -------------------------------------------------------------------------------- /rendering_tile_server-sockets.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | from collections import defaultdict, deque 4 | import multiprocessing 5 | import os 6 | import os.path 7 | import random 8 | import re 9 | from selectors import DefaultSelector as Selector, EVENT_READ, EVENT_WRITE 10 | import socket 11 | import sys 12 | import time 13 | import traceback 14 | 15 | from generate_tiles import RenderThread, StormBringer 16 | import map_utils 17 | from tiles import Tile, MetaTile 18 | import utils 19 | 20 | import logging 21 | from logging import debug, info, exception, warning 22 | long_format = "%(asctime)s %(name)16s:%(lineno)-4d (%(funcName)-21s) %(levelname)-8s %(message)s" 23 | short_format = "%(asctime)s %(message)s" 24 | 25 | logging.basicConfig(level=logging.DEBUG, format=long_format) 26 | 27 | 28 | # fake multiprocessing for testing 29 | class FakeRenderThread: 30 | def __init__(self, opts, input, output): 31 | self.input = input 32 | self.output = output 33 | 34 | def render(self, work): 35 | seconds = random.randint(3, 75) 36 | debug(f"[{self.name}] {work.metatile}: sleeping for {seconds}...") 37 | time.sleep(seconds) 38 | debug(f"[{self.name}] {work.metatile}: ... {seconds} seconds!") 39 | self.output.put(work) 40 | 41 | return True 42 | 43 | def load_map(self): 44 | pass 45 | 46 | def loop(self): 47 | debug(f"[{self.name}] loop") 48 | while self.single_step(): 49 | pass 50 | 51 | debug(f"[{self.name}] done") 52 | 53 | def single_step(self): 54 | work = self.input.get() 55 | debug(f"[{self.name}] step") 56 | if work is None: 57 | debug(f"[{self.name}] bye!") 58 | # it's the end; send the storage thread a message and finish 59 | self.output.put(None) 60 | return False 61 | 62 | return self.render(work) 63 | 64 | 65 | class Master: 66 | def __init__(self, opts): 67 | self.renderers = {} 68 | 69 | # we have several data structures here 70 | # clients_for_metatile maps MetaTiles to Clients, so we can: 71 | # * know if we have it either queued or rendering 72 | # * find it again so we can add more clients 73 | self.clients_for_metatile = defaultdict(set) 74 | # metatile_for_client maps Clients to MetaTiles, so we can remove the Client from the MT's client list 75 | self.metatile_for_client = {} 76 | # the MetaTile queue 77 | self.work_stack = deque(maxlen=4096) 78 | # the MetaTile being rendered 79 | self.in_flight = set() 80 | 81 | self.new_work = multiprocessing.Queue(1) 82 | self.store_queue = utils.SimpleQueue(5*opts.threads) 83 | self.info = multiprocessing.Queue(5*opts.threads) 84 | 85 | self.backend = map_utils.DiskBackend(opts.tile_dir) 86 | self.store_thread = StormBringer(opts, self.backend, self.store_queue, self.info) 87 | self.store_thread.name = 'store-embedded' 88 | 89 | for i in range(opts.threads): 90 | renderer = RenderThread(opts, self.new_work, self.store_queue) 91 | render_thread = multiprocessing.Process(target=renderer.loop, name=f"Renderer-{i+1:03d}") 92 | renderer.name = render_thread.name 93 | renderer.store_thread = self.store_thread 94 | 95 | render_thread.start() 96 | self.renderers[i] = render_thread 97 | 98 | def append(self, metatile, client): 99 | # if I move clients = self.clients_for_metatile[metatile] here, the entry is created 100 | # and we never enter the 'then' branch 101 | if metatile not in self.clients_for_metatile: 102 | debug(f"[Master]: first Client for {metatile!r}: {client}") 103 | 104 | self.work_stack.append(metatile) 105 | self.clients_for_metatile[metatile].add(client) 106 | self.metatile_for_client[client] = metatile 107 | else: 108 | clients = self.clients_for_metatile[metatile] 109 | clients.add(client) 110 | self.metatile_for_client[client] = metatile 111 | debug(f"[Master]: new Client for {metatile!r}: {clients}") 112 | 113 | def remove(self, client_to_remove): 114 | metatile = self.metatile_for_client[client_to_remove] 115 | 116 | # now, this might seem dangerous, but all these structures are handled on the main thread 117 | # so there is no danger of race conditions 118 | clients = self.clients_for_metatile[metatile] 119 | clients.remove(client) 120 | 121 | # if this metatile ends without clients, remove it 122 | if len(clients) == 0: 123 | # no need to remove it from in_flight; all will be handled when the MetaTile has finished being rendered 124 | # TODO: explain why 125 | if metatile not in self.in_flight: 126 | debug(f"clients for {metatile!r} empty, removing before it's sent for rendering") 127 | self.work_stack.remove(metatile) 128 | del self.clients_for_metatile[metatile] 129 | del self.metatile_for_client[client_to_remove] 130 | 131 | def render_tiles(self): 132 | try: 133 | self.loop([]) 134 | except KeyboardInterrupt: 135 | print('C-c detected, exiting') 136 | except Exception as e: 137 | print(f"Unknown exception {e}") 138 | finally: 139 | print('finishing!') 140 | self.finish() 141 | 142 | def loop(self, initial_metatiles): 143 | while True: 144 | self.single_step() 145 | 146 | def single_step(self): 147 | # TODO: similar to generate_tiles' 148 | 149 | # I could get to the pipes used for the Queues, but it's useless, as they're constantly ready 150 | # they're really controlled by the semaphores guarding those pipes 151 | # so select()ing on them leads to a tight loop 152 | # keep the probing version 153 | 154 | tight_loop = True 155 | 156 | # we have two Queues to manage, new_work and info 157 | # neither new_work.push() nor info.pop() should block, but luckily we're the only thread 158 | # writing on the former and reading from the latter 159 | 160 | # so we simply test-and-write and test-and-read 161 | 162 | # the doc says this is unreliable, but we don't care 163 | # full() can be inconsistent only if when we test is false 164 | # and when we put() is true, but only the master is writing 165 | # so no other thread can fill the queue 166 | while not self.new_work.full() and len(self.work_stack) > 0: 167 | tight_loop = False 168 | 169 | metatile = self.work_stack.popleft() # tiles.MetaTile 170 | 171 | if metatile is not None: 172 | # because we're the only writer, and it's not full, this can't block 173 | # debug('[Master] new_work.put...') 174 | self.new_work.put(metatile) 175 | # debug('[Master] ... new_work.put!') 176 | # debug(f"[Master] --> {metatile!r}") 177 | # debug(f"[Master] --> {clients}") 178 | 179 | # moving from work_stack to in_flight 180 | self.in_flight.add(metatile) 181 | else: 182 | # no more work to do 183 | tight_loop = True 184 | break 185 | 186 | result = [] 187 | while not self.info.empty(): 188 | tight_loop = False 189 | 190 | # debug('info.get...') 191 | metatile = self.info.get() 192 | clients = self.clients_for_metatile[metatile] 193 | 194 | # debug('... info.got!') 195 | # debug(f"[Master] <-- {metatile!r}") 196 | debug(f"[Master] <-- {clients=}") 197 | 198 | # bookkeeping 199 | self.in_flight.remove(metatile) 200 | del self.clients_for_metatile[metatile] 201 | for client in clients: 202 | del self.metatile_for_client[client] 203 | 204 | result.append(metatile) 205 | 206 | return tight_loop, result 207 | 208 | def finish(self): 209 | for i in range(opts.threads): 210 | self.new_work.put(None) 211 | 212 | while not self.info.empty(): 213 | data = self.info.get() 214 | # debug(f"[Master] <-- {data=}") 215 | 216 | self.new_work.join() 217 | for i in range(opts.threads): 218 | self.renderers[i].join() 219 | debug('finished') 220 | 221 | 222 | class DoubleDict: 223 | def __init__(self): 224 | self.forward = {} 225 | self.backward = {} 226 | 227 | def __getitem__(self, key): 228 | if key in self.forward: 229 | return self.forward[key] 230 | 231 | return self.backward[key] 232 | 233 | def __setitem__(self, key, value): 234 | self.forward[key] = value 235 | self.backward[value] = key 236 | 237 | def __delitem__(self, key): 238 | if key not in self.forward and key not in self.backward: 239 | raise KeyError 240 | 241 | if key in self.forward: 242 | value = self.forward.pop(key) 243 | else: 244 | value = self.backward.pop(key) 245 | 246 | if value in self.forward: 247 | del self.forward[value] 248 | else: 249 | del self.backward[value] 250 | 251 | def __contains__(self, key): 252 | return key in self.forward or key in self.backward 253 | 254 | def get(self, key, default=None): 255 | if key in self.forward: 256 | value = self.forward.pop(key) 257 | elif key in self.backward: 258 | value = self.backward.pop(key) 259 | else: 260 | value = default 261 | 262 | return value 263 | 264 | 265 | class Client: 266 | def __init__(self, socket): 267 | self.socket = socket 268 | 269 | # to support short reads we will be using a buffer and an offset that will point to the first free byte 270 | self.read_buffer = memoryview(bytearray(4096)) # we really don't need much, since requests are quite small 271 | self.read_buffer_offset = 0 272 | self.request_read = False 273 | 274 | self.write_data = [] 275 | self.write_file = None 276 | 277 | self.tile_path = None 278 | 279 | def recv(self): 280 | # if the last time we finished reading the request, we have to start from 0 281 | if self.request_read: 282 | self.request_read = False 283 | self.read_buffer_offset = 0 284 | 285 | read = self.socket.recv_into(self.read_buffer[self.read_buffer_offset:]) # the size is automatic 286 | if read > 0: 287 | self.read_buffer_offset += read 288 | 289 | # ugh that bytes(), I hope it's cheap 290 | if b'\r\n\r\n' in bytes(self.read_buffer[:self.read_buffer_offset]): 291 | self.request_read = True 292 | 293 | return self.read_buffer[:self.read_buffer_offset] 294 | 295 | def send(self, data): 296 | if isinstance(data, bytes): 297 | # textual data 298 | self.write_data.append(data) 299 | else: 300 | self.write_file = data 301 | 302 | def flush(self): 303 | try: 304 | for data in self.write_data: 305 | sent = self.socket.send(data) 306 | # TODO implement handling of short writes 307 | assert sent == len(data) 308 | 309 | if self.write_file is not None: 310 | self.socket.sendfile(open(self.write_file, 'br')) 311 | except BrokenPipeError: 312 | # the client died, close it. 313 | warning(f"{self.getpeername()} died, closing") 314 | self.close() 315 | 316 | def close(self): 317 | self.socket.close() 318 | 319 | def fileno(self): 320 | return self.socket.fileno() 321 | 322 | def getpeername(self): 323 | if self.client_name is None: 324 | self.client_name = self.socket.getpeername() 325 | 326 | return self.client_name 327 | 328 | def __hash__(self): 329 | return hash(self.socket) 330 | 331 | 332 | class Server: 333 | content_type = { 334 | '.png': 'image/png', 335 | '.html': 'text/html', 336 | '.htm': 'text/html', 337 | '.js': 'text/javascript', 338 | } 339 | 340 | 341 | def __init__(self, opts): 342 | self.opts = opts 343 | 344 | self.listener = socket.socket() 345 | # before bind 346 | self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) 347 | self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, True) 348 | 349 | self.listener.bind( ('', 8080) ) 350 | info(f"Listening on port 8080, serving from {self.opts.tile_dir}.") 351 | self.listener.listen(32) 352 | self.listener.setblocking(False) 353 | 354 | self.selector = Selector() 355 | self.selector.register(self.listener, EVENT_READ) 356 | 357 | self.clients = set() 358 | self.queries_clients = DoubleDict() 359 | self.client_for_peer = {} 360 | 361 | # this looks like dupe'd from Master, but they have slightly different life cycles 362 | self.clients_for_metatile = defaultdict(set) 363 | 364 | # canonicalize 365 | # root = os.path.abspath(root) 366 | 367 | # b'GET /12/2111/1500.png HTTP/1.1\r\nHost: ioniq:8080\r\nConnection: Keep-Alive\r\nAccept-Encoding: gzip\r\nUser-Agent: okhttp/3.12.2\r\n\r\n' 368 | # but we only care about the first line, so 369 | # GET /12/2111/1500.png HTTP/1.1 370 | self.request_re = re.compile(r'(?P[A-Z]+) (?P.*) (?P.*)') 371 | 372 | self.master = Master(self.opts) 373 | 374 | def accept(self): 375 | # new client 376 | # debug('accept..') 377 | client_socket, addr = self.listener.accept() 378 | # debug('...ed!') 379 | debug(f"connection from {addr}") 380 | 381 | client = Client(client_socket) 382 | 383 | self.clients.add(client) 384 | self.client_for_peer[client.getpeername()] = client 385 | self.selector.register(client, EVENT_READ) 386 | 387 | def client_read(self, client): 388 | data = bytes(client.recv()) 389 | debug(f"read from {client.getpeername()}: {data}") 390 | 391 | if len(data) == 0: 392 | debug(f"client {client.getpeername()} disconnected") 393 | 394 | # clean up trailing queries from the work_stack 395 | query = self.queries_clients.get(client, None) 396 | if query is not None: 397 | try: 398 | self.master.work_stack.remove(query) 399 | except ValueError: 400 | # already being rendered, ignore 401 | pass 402 | 403 | # debug(f"client {client.getpeername()} disconnected, was waiting for {query}") 404 | else: 405 | # debug(f"client {client.getpeername()} disconnected, didn't made any query yet.") 406 | pass 407 | 408 | # TODO: 409 | # self.responses[client] = [] 410 | 411 | # now we need to wait for client to be ready to write 412 | self.selector.unregister(client) 413 | self.selector.register(client, EVENT_WRITE) 414 | elif client.request_read: 415 | # we finish reading from this one for now 416 | self.selector.unregister(client) 417 | 418 | # splitlines() already handles any type of separators 419 | lines = data.decode().splitlines() 420 | request_line = lines[0] 421 | match = self.request_re.match(request_line) 422 | 423 | debug(match) 424 | if match is None: 425 | client.send(b'HTTP/1.1 400 KO\r\n\r\n') 426 | else: 427 | if match['method'] != 'GET': 428 | debug(f"404: bad method {match['method']}") 429 | client.send(b'HTTP/1.1 405 only GETs\r\n\r\n') 430 | else: 431 | path = match['url'] 432 | 433 | # TODO: similar code is in tile_server. try to refactor 434 | try: 435 | # _ gets '' because path is absolute 436 | _, z, x, y_ext = path.split('/') 437 | except ValueError: 438 | # not a tile, try to serve the file 439 | # but remove the leading / so o.p.join() does not use it as an abs path 440 | self.answer(client, os.path.join(self.opts.tile_dir, path[1:])) 441 | else: 442 | # TODO: make sure ext matches the file type we return 443 | y, ext = os.path.splitext(y_ext) 444 | 445 | # o.p.join() considers path to be absolute, so it ignores root 446 | tile_path = os.path.join(self.opts.tile_dir, z, x, y_ext) 447 | 448 | # try to send the tile first, but do not send 404s 449 | if not self.answer(client, tile_path, send_404=False): 450 | tile = Tile(*[ int(coord) for coord in (z, x, y) ]) 451 | metatile = MetaTile.from_tile(tile, self.opts.metatile_size) 452 | debug(f"{client.getpeername()}: {metatile!r}") 453 | 454 | client.metatile = metatile 455 | client.tile_path = tile_path 456 | 457 | self.clients_for_metatile[metatile].add(client) 458 | self.master.append(metatile, client.getpeername()) 459 | self.queries_clients[client] = tile_path 460 | else: 461 | debug(f"short read from {client.getpeername()}") 462 | 463 | def client_write(self, client): 464 | client.flush() 465 | # BUG: no keep alive support 466 | debug(f"closing {client.getpeername()}") 467 | client.close() 468 | 469 | # bookkeeping 470 | self.selector.unregister(client) 471 | self.clients.remove(client) 472 | if client in self.queries_clients: 473 | del self.queries_clients[client] 474 | 475 | def loop(self): 476 | while True: 477 | try: 478 | # debug(f"select... [{len(self.master.work_stack)=}; {self.master.new_work.qsize()=}; {self.master.store_queue.qsize()=}; {self.master.info.qsize()=}]") 479 | # debug(self.selector.get_map()) 480 | for key, events in self.selector.select(1): 481 | # debug('...ed!') 482 | ready_socket = key.fileobj 483 | 484 | if ready_socket == self.listener: 485 | self.accept() 486 | elif ready_socket in self.clients: 487 | client = ready_socket 488 | 489 | if events & EVENT_READ: 490 | self.client_read(client) 491 | 492 | if events & EVENT_WRITE: 493 | self.client_write(client) 494 | 495 | # advance the queues 496 | _, jobs = self.master.single_step() 497 | 498 | for metatile in jobs: 499 | debug(f"{metatile=}") 500 | # BUG: ugh, shouldn't be touching master's internals like this 501 | clients = self.clients_for_metatile[metatile] 502 | debug(f"{clients=}") 503 | 504 | for client in clients: 505 | self.answer(client, client.tile_path) 506 | 507 | # bookkeeping 508 | del self.clients_for_metatile[metatile] 509 | except Exception as e: 510 | if isinstance(e, KeyboardInterrupt): 511 | raise 512 | else: 513 | traceback.print_exc() 514 | 515 | def answer(self, client, file_path, send_404=True): 516 | # debug(f"answering {client.getpeername()} for {file_path} ") 517 | try: 518 | # this could be considered 'blocking', but if the fs is slow, we have other problems 519 | # debug(f"find me {file_path}...") 520 | file_attrs = os.stat(file_path) 521 | # debug('... stat!') 522 | except FileNotFoundError: 523 | if send_404: 524 | info(f"404: not found {file_path}...") 525 | client.send(f"HTTP/1.1 404 not here {file_path}\r\n\r\n".encode()) 526 | 527 | return False 528 | else: 529 | info(f"200: found {file_path} for {client.getpeername()}") 530 | ext = os.path.splitext(file_path)[1] 531 | client.send(b'HTTP/1.1 200 OK\r\n') 532 | client.send(f"Content-Type: {self.content_type[ext]}\r\n".encode()) 533 | client.send(f"Content-Length: {file_attrs.st_size}\r\n\r\n".encode()) 534 | client.send(file_path) 535 | 536 | # now we need to wait for client to be ready to write 537 | self.selector.register(client, EVENT_WRITE) 538 | 539 | return True 540 | 541 | 542 | class Options: 543 | pass 544 | 545 | 546 | def main(root): 547 | opts = Options() 548 | 549 | # alphabetical order 550 | opts.bbox = None 551 | opts.coords = None 552 | opts.dry_run = False 553 | opts.empty = 'skip' 554 | opts.empty_color = '#aad3df' 555 | opts.empty_size = 103 556 | opts.format = 'tiles' # TODO? 557 | opts.mapfile = 'Elevation.xml' 558 | opts.mapnik_strict = False 559 | opts.max_zoom = 21 # deep enough 560 | opts.metatile_size = 2 561 | opts.more_opts = {} 562 | opts.parallel = 'fork' 563 | opts.parallel_factory = None # TODO 564 | opts.single_tiles = False 565 | opts.store_thread = False 566 | opts.threads = 8 # TODO 567 | # TODO: 568 | # opts.tile_dir = app.root_dir 569 | opts.tile_dir = root 570 | opts.tile_file_format = 'png' 571 | opts.tile_file_format_options = '' 572 | # TODO: no support for hi-res tiles (512) 573 | opts.tile_size = 256 574 | 575 | server = Server(opts) 576 | server.loop() 577 | 578 | 579 | if __name__ == '__main__': 580 | main(sys.argv[1]) 581 | -------------------------------------------------------------------------------- /generate_tiles.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | # https://github.com/openstreetmap/mapnik-stylesheets/blob/master/generate_tiles.py with *LOTS* of enhancements 4 | 5 | from subprocess import call 6 | import sys, os, os.path 7 | import queue 8 | from argparse import ArgumentParser 9 | import time 10 | import errno 11 | import threading 12 | import datetime 13 | import errno 14 | import multiprocessing 15 | from random import randint, random 16 | from os import getpid 17 | import math 18 | from signal import signal, SIGINT, SIG_IGN 19 | 20 | 21 | try: 22 | import mapnik2 as mapnik 23 | except: 24 | import mapnik 25 | 26 | import pyproj 27 | 28 | import map_utils 29 | import utils 30 | import tiles 31 | 32 | 33 | import logging 34 | from logging import debug, info, exception, warning 35 | long_format = "%(asctime)s %(name)16s:%(lineno)-4d (%(funcName)-21s) %(levelname)-8s %(message)s" 36 | short_format = "%(asctime)s %(message)s" 37 | 38 | # EPSGs 39 | WebMerc = 3857 40 | LonLat = 4326 41 | 42 | from typing import Optional, List, Set, Dict, Any 43 | 44 | 45 | # TODO: this gives very bad numbers because if we start from a low enough ZL many tiles will be discarded 46 | # replace it with an algorithm that precounts all the tiles that are really going to be rendered 47 | # down there there m,ust be a part of the algo that gets all the tiles for ZL x that match the original bbox 48 | # TODO: check whether if it's a generator 49 | # TODO: or replace with something that simultaes going down the pyramid; in fact, it could be a dry run version 50 | # of the actual rendering algo 51 | 52 | 53 | class RenderStack: 54 | ''' 55 | A render stack implemented with a list... and more. 56 | 57 | Although this is implemented with a list, I prefer the semantic of these 58 | methods and the str() representation given by the list being pop from/push 59 | into the left. 60 | 61 | The stack has a first element, which is the one ready to be pop()'ed. 62 | Because this element might need to be returned into the stack, there's the confirm() 63 | method which actually pops it, leaving the next one ready. 64 | 65 | The stack also autofills with children when we pop an element. Because 66 | these children might not be needed to be rendered, they're stored in 67 | another list, to_validate. 68 | 69 | stack.push(e) 70 | e = stack.pop() 71 | stack.confirm() 72 | ''' 73 | def __init__(self, max_zoom:int) -> None: 74 | # I don't need order here, it's (probably) better if I validate tiles 75 | # as soon as possible 76 | self.first:Optional[tiles.MetaTile] = None 77 | self.ready:List[tiles.MetaTile] = [] 78 | self.max_zoom = max_zoom 79 | 80 | 81 | def push(self, metatile:tiles.MetaTile) -> None: 82 | # debug("%s, %s, %s", self.first, self.ready, self.to_validate) 83 | if self.first is not None: 84 | self.ready.insert(0, self.first) 85 | 86 | self.first = metatile 87 | 88 | 89 | def pop(self) -> Optional[tiles.MetaTile]: 90 | return self.first 91 | 92 | 93 | def confirm(self) -> None: 94 | '''Mark the top of the stack as sent to render, factually pop()'ing it.''' 95 | if self.first is not None: 96 | # metatile:tiles.MetaTile = self.first 97 | metatile = self.first 98 | 99 | # t:Optional[tiles.Tile] = None 100 | t = None 101 | if len(self.ready) > 0: 102 | t = self.ready.pop(0) 103 | 104 | self.first = t 105 | # debug("%s, %s, %s", self.first, self.ready, self.to_validate) 106 | 107 | 108 | def size(self) -> int: 109 | # debug("%s, %s, %s", self.first, self.ready, self.to_validate) 110 | # HACK: int(bool) ∈ (0, 1) 111 | # ans:int = int(self.first is not None) + len(self.ready) 112 | ans = int(self.first is not None) + len(self.ready) 113 | # debug(ans) 114 | return ans 115 | 116 | 117 | RenderChildren = Dict[tiles.Tile, bool] 118 | class RenderThread: 119 | def __init__(self, opts, input, output) -> None: 120 | self.opts = opts 121 | self.input = input 122 | self.output = output 123 | 124 | # self.metatile_size:int = opts.metatile_size 125 | # self.image_size:int = self.opts.tile_size * self.metatile_size 126 | self.metatile_size = opts.metatile_size 127 | 128 | self.store_thread = None 129 | 130 | 131 | def render(self, metatile:tiles.MetaTile) -> Dict[tiles.Tile, bool]: 132 | # get LatLong (EPSG:4326) 133 | w, n = metatile.coords[0] 134 | e, s = metatile.coords[1] 135 | # debug("lonlat: %r %r %r %r", w, s, e, n) 136 | 137 | w, n = ( int(x) for x in self.transformer.transform(n, w) ) 138 | e, s = ( int(x) for x in self.transformer.transform(s, e) ) 139 | # debug("webmerc: %r %r %r %r", w, s, e, n) 140 | 141 | # Bounding box for the tile 142 | if hasattr(mapnik, 'mapnik_version') and mapnik.mapnik_version() >= 800: 143 | # lower left and upper right corner points. 144 | bbox = mapnik.Box2d(w, s, e, n) 145 | else: 146 | bbox = mapnik.Envelope(w, s, e, n) 147 | 148 | # debug("bbox: %r", bbox) 149 | 150 | image_size = self.opts.tile_size * min(self.metatile_size, 2**metatile.z) 151 | self.m.resize(image_size, image_size) 152 | self.m.zoom_to_box(bbox) 153 | if self.m.buffer_size < 128: 154 | self.m.buffer_size = 128 155 | 156 | start = time.perf_counter() 157 | if not self.opts.dry_run: 158 | if self.opts.format == 'svg': 159 | im = cairo.SVGSurface(f"{self.opts.tile_dir}/{self.opts.coords[0][0]}-{self.opts.coords[0][1]}.svg", 160 | self.opts.tile_size, self.opts.tile_size) 161 | elif self.opts.format == 'pdf': 162 | im = cairo.PDFSurface(f"{self.opts.tile_dir}/{self.opts.coords[0][0]}-{self.opts.coords[0][1]}.pdf", 163 | self.opts.tile_size, self.opts.tile_size) 164 | else: 165 | im = mapnik.Image(image_size, image_size) 166 | 167 | # Render image with default Agg renderer 168 | debug('[%s] rende...', self.name) 169 | # TODO: handle exception, send back into queue 170 | try: 171 | debug(mapnik.render(self.m, im)) 172 | except Exception as e: 173 | debug(e) 174 | 175 | mid = time.perf_counter() 176 | metatile.render_time = mid - start 177 | metatile.serializing_time = 0 178 | 179 | # render errors are quite bad, bailout 180 | return False 181 | 182 | debug('[%s] ...ring!', self.name) 183 | mid = time.perf_counter() 184 | 185 | # TODO: all this is on a single tile, not a metatile 186 | 187 | # converting to png256 is the fastest I have found so far: 188 | # python3.6 -m timeit -s 'import mapnik; im = mapnik.Image.fromstring(open("Attic/tmp/369.png", "br").read())' 'data = im.tostring("png256")' 189 | # 100 loops, best of 3: 7.72 msec per loop 190 | # tostring() looks nice, but I can't rebuild a mapnik.Image from it :( 191 | # python3.6 -m timeit -s 'import mapnik; im = mapnik.Image.fromstring(open("Attic/tmp/369.png", "br").read())' 'data = im.tostring()' 192 | # 100000 loops, best of 3: 13.8 usec per loop 193 | # python3.6 -m timeit -s 'import mapnik, bz2; im = mapnik.Image.fromstring(open("Attic/tmp/369.png", "br").read())' 'c = bz2.BZ2Compressor(); c.compress(im.tostring()); data = c.flush()' 194 | # 10 loops, best of 3: 20.3 msec per loop 195 | # python3.6 -m timeit -s 'import mapnik, gzip; im = mapnik.Image.fromstring(open("Attic/tmp/369.png", "br").read())' 'data = gzip.compress(im.tostring())' 196 | # 10 loops, best of 3: 27.7 msec per loop 197 | # python3.6 -s -m timeit -s 'import mapnik, lzma; im = mapnik.Image.fromstring(open("Attic/tmp/369.png", "br").read())' "c = lzma.LZMACompressor(); c.compress(im.tostring()); data = c.flush()" 198 | # 10 loops, best of 3: 92 msec per loop 199 | 200 | # TODO: 201 | # but bz2 compresses the best, 52714 png vs 49876 bzip vs 70828 gzip vs 53032 lzma 202 | 203 | if not self.opts.store_thread: 204 | # if there is not a separate store thread, the metatile will go in a non-marshaling queue, 205 | # so no need tostring() it 206 | metatile.im = im 207 | else: 208 | metatile.im = im.tostring('png256') # here it's converted only for serialization reasons 209 | 210 | end = time.perf_counter() 211 | else: 212 | debug('[%s] thumbtumbling', self.name) 213 | time.sleep(randint(0, 30) / 10) 214 | mid = time.perf_counter() 215 | end = time.perf_counter() 216 | 217 | metatile.render_time = mid - start 218 | metatile.serializing_time = end - mid 219 | 220 | debug("[%s] putting %r", self.name, metatile) 221 | self.output.put(metatile) 222 | debug("[%s] put! (%d)", self.name, self.output.qsize()) 223 | 224 | if not self.opts.store_thread and self.output.qsize() > 0: 225 | # NOTE: mypy complains here that Item "Process" of "Union[Process, StormBringer]" has no attribute "single_step" 226 | # the solutions are ugly, so I'm leaving it as that 227 | self.store_thread.single_step() 228 | 229 | return True 230 | 231 | 232 | def load_map(self): 233 | start = time.perf_counter() 234 | 235 | # don't worry about the size, we're going to change it later 236 | self.m = mapnik.Map(0, 0) 237 | # Load style XML 238 | if not self.opts.dry_run: 239 | try: 240 | mapnik.load_map(self.m, self.opts.mapfile, self.opts.mapnik_strict) 241 | except RuntimeError as e: 242 | print(f"Error loading map: {e.args[0]}") 243 | return False 244 | 245 | end = time.perf_counter() 246 | info('[%s] Map loading took %.6fs', self.name, end - start) 247 | 248 | self.transformer = pyproj.Transformer.from_crs(f'EPSG:{LonLat}', f'EPSG:{WebMerc}') 249 | 250 | # Projects between tile pixel co-ordinates and LatLong (EPSG:4326) 251 | # this is *not* the same as EPSG:3857 because every zoom level has its own pixel space 252 | self.tileproj = tiles.GoogleProjection(self.opts.max_zoom + 1) 253 | 254 | return True 255 | 256 | 257 | def loop(self): 258 | # disable SIGINT so C-c/KeyboardInterrupt is handled by Master 259 | # even in the case of multiprocessing 260 | sig = signal(SIGINT, SIG_IGN) 261 | 262 | if not self.load_map(): 263 | # TODO: send a ready/failed message to the Master 264 | return 265 | 266 | info("[%s]: starting...", self.name) 267 | debug('[%s] looping the loop', self.name) 268 | 269 | while self.single_step(): 270 | pass 271 | 272 | info("[%s] finished", self.name) 273 | 274 | 275 | def single_step(self): 276 | # Fetch a tile from the queue and render it 277 | debug("[%s] get..", self.name) 278 | # metatile:Optional[tiles.MetaTile] = self.input.get() 279 | metatile = self.input.get() 280 | debug("[%s] got! %r", self.name, metatile) 281 | 282 | if metatile is None: 283 | # it's the end; send the storage thread a message and finish 284 | debug("[%s] putting %r", self.name, None) 285 | self.output.put(None) 286 | debug("[%s] put! (%d)", self.name, self.output.qsize()) 287 | 288 | return False 289 | 290 | return self.render(metatile) 291 | 292 | 293 | # backends:Dict[str,Any] = dict( 294 | backends = dict( 295 | tiles= map_utils.DiskBackend, 296 | mbtiles= map_utils.MBTilesBackend, 297 | mod_tile=map_utils.ModTileBackend, 298 | test= map_utils.TestBackend, 299 | # SVGs and PDFs are saved by the renderer itself 300 | # but we still need to provide a callable 301 | svg= lambda *more, **even_more: None, 302 | pdf= lambda *more, **even_more: None, 303 | ) 304 | 305 | default_options = dict( 306 | png256='t=0', 307 | jpeg='quality=50', 308 | ) 309 | 310 | 311 | class StormBringer: 312 | def __init__(self, opts, backend, input, output): 313 | self.opts = opts 314 | self.backend = backend 315 | self.input = input 316 | self.output = output 317 | # the amount of threads writing on input 318 | # this is needed so we can stop only after all the writers sent their last jobs 319 | self.writers = self.opts.threads 320 | self.done_writers = 0 321 | 322 | if self.opts.tile_file_format == 'png': 323 | self.tile_file_format = 'png256' 324 | elif self.opts.tile_file_format == 'jpg': 325 | self.tile_file_format = 'jpeg' 326 | 327 | if self.opts.tile_file_format_options == '': 328 | self.opts.tile_file_format_options = default_options[self.tile_file_format] 329 | 330 | self.tile_file_format += f":{self.opts.tile_file_format_options}" 331 | 332 | 333 | def loop(self): 334 | # disable SIGINT so C-c/KeyboardInterrupt is handled by Master 335 | # even in the case of multiprocessing 336 | sig = signal(SIGINT, SIG_IGN) 337 | 338 | debug('[%s] curling the curl', self.name) 339 | 340 | while self.single_step(): 341 | pass 342 | 343 | debug('[%s] done', self.name) 344 | 345 | 346 | def single_step(self): 347 | debug('[%s] >... (%d)', self.name, self.input.qsize()) 348 | metatile = self.input.get() 349 | debug('[%s] ...> %s', self.name, metatile) 350 | 351 | if metatile is not None: 352 | debug('[%s] sto...', self.name) 353 | self.store_metatile(metatile) 354 | debug('[%s] ...red!', self.name) 355 | # we don't need it anymore and *.Queue complains that 356 | # mapnik._mapnik.Image is not pickle()'able 357 | metatile.im = None 358 | self.output.put(metatile) 359 | else: 360 | # this writer finished 361 | self.done_writers += 1 362 | debug('[%s] %d <-> %d', self.name, self.writers, self.done_writers) 363 | 364 | return self.done_writers != self.writers 365 | 366 | 367 | def store_metatile(self, metatile): 368 | # save the image, splitting it in the right amount of tiles 369 | if not self.opts.dry_run: 370 | start = time.perf_counter() 371 | if not self.opts.store_thread: 372 | image = metatile.im 373 | else: 374 | image = mapnik.Image.frombuffer(metatile.im) 375 | mid = time.perf_counter() 376 | 377 | for tile in metatile.tiles: 378 | self.store_tile(tile, image) 379 | 380 | # do not try to compute this for tiles outside the asked range 381 | if metatile.z < self.opts.max_zoom: 382 | child = metatile.child(tile) 383 | # PixelTile does not have 384 | if child is not None: 385 | child.is_empty = child.is_empty and tile.is_empty 386 | 387 | end = time.perf_counter() 388 | 389 | for child in metatile.children(): 390 | # don't render child if: empty; or single tile mode; or too deep 391 | debug((child.is_empty, self.opts.single_tiles, tile.z, self.opts.max_zoom)) 392 | if child.is_empty or self.opts.single_tiles or metatile.z == self.opts.max_zoom: 393 | child.render = False 394 | 395 | metatile.deserializing_time = mid - start 396 | metatile.saving_time = end - mid 397 | else: 398 | # do not try to compute this for tiles outside the asked range 399 | if metatile.x < self.opts.max_zoom: 400 | for child in metatile.children(): 401 | rand = random() 402 | # 5% are fake empties 403 | child.is_empty = (rand <= 0.05 and 2**metatile.z >= self.opts.metatile_size) 404 | child.render = not (child.is_empty or self.opts.single_tiles or metatile.z == self.opts.max_zoom) 405 | 406 | 407 | def store_tile(self, tile, image): 408 | # SVG and PDF are stored by the renderer 409 | if self.opts.format not in ('svg', 'pdf'): 410 | i, j = tile.meta_index 411 | 412 | # TODO: Tile.meta_pixel_coords 413 | # TODO: pass tile_size to MetaTile and Tile 414 | img = image.view(i*self.opts.tile_size, j*self.opts.tile_size, 415 | self.opts.tile_size, self.opts.tile_size) 416 | 417 | # this seems like duplicated work, but we need one looseless format for serializing 418 | # and another format for the final tile 419 | tile.data = img.tostring(self.tile_file_format) 420 | 421 | # debug((len(tile.data), tile.data[41:44])) 422 | tile.is_empty = len(tile.data) == self.opts.empty_size and tile.data[41:44] == self.opts.empty_color 423 | 424 | if tile.is_empty: 425 | debug('Skipping empty tile') 426 | 427 | if not tile.is_empty or self.opts.empty == 'write': 428 | self.backend.store(tile) 429 | elif tile.is_empty and self.opts.empty == 'link': 430 | # TODO 431 | pass 432 | 433 | self.backend.commit() 434 | 435 | 436 | class Master: 437 | def __init__(self, opts) -> None: 438 | self.opts = opts 439 | self.renderers:Dict[int, Union[multiprocessing.Process, threading.Thread]] = {} 440 | self.store_thread:Union[multiprocessing.Process, StormBringer] 441 | # we need at least space for the initial batch 442 | # but do not auto push children in tiles mode 443 | self.work_stack = RenderStack(opts.max_zoom) 444 | 445 | # counters 446 | self.went_out = self.came_back = 0 447 | self.tiles_to_render = self.tiles_rendered = self.tiles_skipped = 0 448 | 449 | # statistics 450 | self.median = utils.MedianTracker() 451 | 452 | self.create_infra() 453 | 454 | 455 | def create_infra(self): 456 | # this creates a queue that's not used for on-demand usage, but it's not much, so we don't care 457 | if self.opts.parallel == 'fork': 458 | debug('forks, using mp.Queue()') 459 | # work_out queue is size 1, so initial metatiles don't clog it at the beginning 460 | # because we need to be able to replace the top with its children to drill down 461 | # and 'reuse' cache before moving to another area. 462 | self.new_work = multiprocessing.Queue(1) 463 | 464 | # store_queue and info need enough space to hold the four children of the metatiles 465 | # returned by all threads and more just in case, otherwise the queue blocks and we get a deadlock 466 | if not self.opts.store_thread: 467 | # BUG: why do we still use a queue when we're storing the tiles 468 | # in the same thread as the one rendering them? 469 | debug('utils.SimpleQueue') 470 | self.store_queue = utils.SimpleQueue(5*self.opts.threads) 471 | else: 472 | self.store_queue = multiprocessing.Queue(5*self.opts.threads) 473 | self.info = multiprocessing.Queue(5*self.opts.threads) 474 | elif self.opts.parallel == 'threads': 475 | debug('threads, using queue.Queue()') 476 | # TODO: warning about mapnik and multithreads 477 | self.new_work = queue.Queue(32) 478 | if not self.opts.store_thread: 479 | debug('utils.SimpleQueue') 480 | self.store_queue = utils.SimpleQueue(32) 481 | else: 482 | self.store_queue = queue.Queue(32) 483 | self.info = queue.Queue(32) 484 | else: # 'single' 485 | debug('single mode, using queue.Queue()') 486 | self.new_work = queue.Queue(1) 487 | if not self.opts.store_thread: 488 | self.store_queue = utils.SimpleQueue(1) 489 | else: 490 | self.store_queue = queue.Queue(1) 491 | self.info = queue.Queue(1) 492 | 493 | self.backend = backends[self.opts.format](self.opts.tile_dir, self.opts.bbox, 494 | **self.opts.more_opts) 495 | 496 | # Launch rendering threads 497 | if self.opts.parallel != 'single': 498 | sb = StormBringer(self.opts, self.backend, self.store_queue, self.info) 499 | if self.opts.store_thread: 500 | self.store_thread = self.opts.parallel_factory(target=sb.loop) 501 | sb.name = self.store_thread.name 502 | self.store_thread.start() 503 | debug("Started store thread %s", self.store_thread.name) 504 | else: 505 | sb.name = 'store-embedded' 506 | self.store_thread = sb 507 | 508 | for i in range(self.opts.threads): 509 | renderer = RenderThread(self.opts, self.new_work, self.store_queue) 510 | 511 | render_thread = self.opts.parallel_factory(target=renderer.loop, name=f"Renderer-{i + 1:03d}") 512 | renderer.name = render_thread.name 513 | 514 | if not self.opts.store_thread: 515 | debug("Store object created, attached to thread") 516 | renderer.store_thread = self.store_thread 517 | 518 | render_thread.start() 519 | debug("Started render thread %s", render_thread.name) 520 | 521 | self.renderers[i] = render_thread 522 | else: 523 | # in this case we create the 'thread', but in fact we only use its single_step() 524 | self.store_thread = StormBringer(self.opts, self.backend, self.store_queue, 525 | self.info) 526 | self.store_thread.name = 'single-store' 527 | debug("Store object created, not threaded") 528 | self.renderer = RenderThread(self.opts, self.new_work, self.store_queue) 529 | self.renderer.name = 'single-render' 530 | self.renderer.load_map() 531 | self.renderer.store_thread = self.store_thread 532 | debug("Renderer object created, not threaded") 533 | 534 | if not os.path.isdir(self.opts.tile_dir) and not self.opts.format == 'mbtiles': 535 | debug("creating dir %s", self.opts.tile_dir) 536 | os.makedirs(self.opts.tile_dir, exist_ok=True) 537 | 538 | 539 | def progress(self, metatile, *args, format='%s'): 540 | percentage = ( (self.tiles_rendered + self.tiles_skipped) / 541 | self.tiles_to_render * 100 ) 542 | 543 | now = time.perf_counter() 544 | time_elapsed = now - self.start 545 | metatile_render_time = sum(metatile.times()) 546 | 547 | if metatile_render_time > 0: 548 | self.median.add(metatile_render_time / len(metatile.tiles)) 549 | debug(self.median.items) 550 | 551 | if self.tiles_rendered > 0: 552 | time_per_tile_median = self.median.median() 553 | time_per_tile_all_tiles = time_elapsed / (self.tiles_rendered + self.tiles_skipped) 554 | time_per_tile_rendered_tiles = time_elapsed / self.tiles_rendered 555 | time_per_tile = (time_per_tile_all_tiles + time_per_tile_rendered_tiles) / 2 556 | debug("times: median: %10.6f; all: %10.6f; rendered: %10.6f; calculated: %10.6f", 557 | time_per_tile_median, time_per_tile_all_tiles, time_per_tile_rendered_tiles, time_per_tile) 558 | 559 | eta = ( (self.tiles_to_render - self.tiles_rendered - self.tiles_skipped) * 560 | time_per_tile ) / self.opts.threads 561 | debug((self.start, now, time_elapsed, metatile_render_time, time_per_tile, eta)) 562 | 563 | format = "[%d+%d/%d: %7.4f%%] %r: " + format + " [Elapsed: %dh%02dm%06.3fs, ETA: %dh%02dm%06.3fs, Total: %dh%02dm%06.3fs]" 564 | info(format, self.tiles_rendered, self.tiles_skipped, self.tiles_to_render, 565 | percentage, metatile, *args, *utils.time2hms(time_elapsed), 566 | *utils.time2hms(eta), *utils.time2hms(time_elapsed + eta)) 567 | else: 568 | format = "[%d+%d/%d: %7.4f%%] %r: " + format + " [Elapsed: %dh%02dm%06.3fs, ETA: ∞, Total: ∞]" 569 | info(format, self.tiles_rendered, self.tiles_skipped, self.tiles_to_render, 570 | percentage, metatile, *args, *utils.time2hms(time_elapsed)) 571 | 572 | 573 | def metatiles_for_bbox(self): 574 | result = [] 575 | 576 | # attributes used a lot, so hold them in local vars 577 | bbox = self.opts.bbox 578 | min_zoom = self.opts.min_zoom 579 | tile_size = self.opts.tile_size 580 | # if the ZL is too low, shrink the metatile_size 581 | metatile_size = min(self.opts.metatile_size, 2**min_zoom) 582 | metatile_pixel_size = metatile_size * tile_size 583 | 584 | debug('rendering bbox %s: %s', self.opts.bbox_name, bbox) 585 | # debug(bbox.lower_left) 586 | # debug(bbox.upper_right) 587 | w, s = tiles.tileproj.lon_lat2pixel(bbox.lower_left, min_zoom) 588 | e, n = tiles.tileproj.lon_lat2pixel(bbox.upper_right, min_zoom) 589 | # debug("pixel ZL%r: %r, %r, %r, %r", min_zoom, w, s, e, n) 590 | # debug("%d", 2**min_zoom) 591 | 592 | w = w // metatile_pixel_size * metatile_size 593 | s = (s // metatile_pixel_size + 1) * metatile_size 594 | e = (e // metatile_pixel_size + 1) * metatile_size 595 | n = n // metatile_pixel_size * metatile_size 596 | # debug("tiles: %r, %r, %r, %r", w, s, e, n) 597 | # debug("%sx%s", list(range(w, e, metatile_size)), list(range(n, s, metatile_size))) 598 | 599 | count = 0 600 | info('Creating initial metatiles...') 601 | for x in range(w, e, metatile_size): 602 | for y in range(n, s, metatile_size): 603 | metatile = tiles.MetaTile(min_zoom, x, y, self.opts.metatile_size, 604 | tile_size) 605 | # TODO: convert this into a generator 606 | # for that we will need to modify the RenderStack so we have 2 sections, one the generator, 607 | # another for the chidren stacked on top 608 | result.append(metatile) 609 | count += 1 610 | if count % 1000 == 0: 611 | info('%d...' % count) 612 | 613 | info("%d initial metatiles created." % count) 614 | 615 | return result 616 | 617 | 618 | def render_tiles(self) -> None: 619 | debug("render_tiles(%s)", self.opts) 620 | 621 | initial_metatiles = [] 622 | if not self.opts.single_tiles: 623 | initial_metatiles = self.metatiles_for_bbox() 624 | else: 625 | # TODO: if possible, order them in depth first/proximity? fashion. 626 | debug('rendering individual tiles') 627 | initial_metatiles = self.opts.tiles 628 | 629 | try: 630 | self.loop(initial_metatiles) 631 | except KeyboardInterrupt as e: 632 | info("Ctrl-c detected, exiting...") 633 | except Exception as e: 634 | info('unknown exception caught!') 635 | exception(str(e)) 636 | finally: 637 | debug('render_tiles() out!') 638 | self.finish() 639 | 640 | 641 | def push_all_children(self, metatile): 642 | if metatile.z < self.opts.max_zoom and self.opts.push_children: 643 | for child in metatile.children(): 644 | # we have no other info about whether they should be 645 | # rendered or not, so render them just in case. at worst, 646 | # they could either be empty tiles or too new too 647 | self.work_stack.push(child) 648 | 649 | 650 | def should_render(self, metatile): 651 | # TODO: move all these checks to another thread/process. 652 | # skip:bool 653 | if metatile in self.opts.bbox: 654 | if self.opts.skip_existing or self.opts.skip_newer is not None: 655 | debug('skip test existing: %s; newer: %s', self.opts.skip_existing, 656 | self.opts.skip_newer) 657 | skip = True 658 | 659 | for tile in metatile.tiles: # type: tiles.Tile 660 | if self.opts.skip_existing: 661 | # TODO: missing as present? 662 | # NOTE: it's called missing_as_new 663 | skip = skip and self.backend.exists(tile) 664 | # debug('skip: %s', skip) 665 | message = "present, skipping" 666 | else: 667 | skip = ( skip and 668 | self.backend.newer_than(tile, self.opts.skip_newer, 669 | self.opts.missing_as_new) ) 670 | # debug('skip: %s', skip) 671 | message = "too new, skipping" 672 | 673 | if skip: 674 | self.work_stack.confirm() 675 | self.tiles_skipped += len(metatile.tiles) 676 | self.progress(metatile, message) 677 | 678 | # notify the children, so they get a chance to be rendered 679 | self.push_all_children(metatile) 680 | else: 681 | skip = False 682 | # debug('skip: %s', skip) 683 | else: 684 | # do not render tiles out of the bbox 685 | skip = True 686 | self.work_stack.confirm() 687 | 688 | # we count this one and all it descendents as skipped 689 | self.tiles_skipped += ( len(metatile.tiles) * 690 | utils.pyramid_count(metatile.z, opts.max_zoom) ) 691 | self.progress(metatile, "out of bbox") 692 | 693 | return not skip 694 | 695 | 696 | def loop(self, initial_metatiles) -> None: 697 | self.start = time.perf_counter() 698 | 699 | for metatile in initial_metatiles: 700 | debug("... %r", (metatile, )) 701 | self.work_stack.push(metatile) 702 | 703 | first_tiles = len(initial_metatiles) 704 | if self.opts.single_tiles: 705 | self.tiles_to_render = first_tiles * len(metatile.tiles) 706 | else: 707 | # all initial_metatiles are from the same zoom level 708 | self.tiles_to_render = ( first_tiles * len(metatile.tiles) * 709 | utils.pyramid_count(opts.min_zoom, opts.max_zoom) ) 710 | 711 | while ( self.work_stack.size() > 0 or 712 | self.went_out > self.came_back or 713 | self.tiles_to_render > self.tiles_rendered + self.tiles_skipped ): 714 | 715 | tight_loop = self.single_step() 716 | 717 | if tight_loop: 718 | # if there's nothing to do, we take a 1/10th second nap 719 | time.sleep(0.1) 720 | 721 | total_time = time.perf_counter() - self.start 722 | h, m, s = utils.time2hms(total_time) 723 | 724 | info("total time: %3dh%02dm%02ds", h, m, s) 725 | metatiles_rendered = self.tiles_rendered / self.opts.metatile_size**2 726 | 727 | if metatiles_rendered != 0: 728 | info("%8.3f s/metatile", total_time / metatiles_rendered) 729 | info("%8.3f metatile/s", metatiles_rendered / total_time * opts.threads) 730 | info("%8.3f s/tile", total_time / self.tiles_rendered) 731 | info("%8.3f tile/s", self.tiles_rendered / total_time * opts.threads) 732 | 733 | debug('loop() out!') 734 | 735 | 736 | 737 | def single_step(self): 738 | # I could get to the pipes used for the Queues, but it's useless, as they're constantly ready 739 | # they're really controlled by the semaphores guarding those pipes 740 | # so select()ing on them leads to a tight loop 741 | # keep the probing version 742 | 743 | tight_loop = True 744 | 745 | # we have two Queues to manage, new_work and info 746 | # neither new_work.push() nor info.pop() should block, but luckily we're the only thread 747 | # writing on the former and reading from the latter 748 | 749 | # so we simply test-and-write and test-and-read 750 | 751 | # the doc says this is unreliable, but we don't care 752 | # full() can be inconsistent only if when we test is false 753 | # and when we put() is true, but only the master is writing 754 | # so no other thread can fill the queue 755 | while not self.new_work.full(): 756 | tight_loop = False 757 | 758 | metatile = self.work_stack.pop() # tiles.MetaTile 759 | if metatile is not None: 760 | # TODO: move to another thread 761 | if not self.should_render(metatile): 762 | continue 763 | 764 | # self.log_grafana(str(metatile)) 765 | 766 | # because we're the only writer, and it's not full, this can't block 767 | self.new_work.put(metatile) 768 | self.work_stack.confirm() 769 | self.went_out += 1 770 | debug("--> %r", (metatile, )) 771 | 772 | if self.opts.parallel == 'single': 773 | self.renderer.single_step() 774 | # also, get out of here, so we can clean up 775 | # in the next loop 776 | break 777 | else: 778 | # no more work to do 779 | tight_loop = True 780 | break 781 | 782 | while not self.info.empty(): 783 | tight_loop = False 784 | 785 | data = self.info.get() # type: Tuple[str, Any] 786 | debug("<-- %s", data) 787 | 788 | self.handle_new_work(data) 789 | 790 | return tight_loop 791 | 792 | 793 | def handle_new_work(self, metatile): 794 | # an empty metatile will be accounted as rendered, 795 | # but the children can be pruned 796 | if self.opts.push_children and metatile.z < self.opts.max_zoom: 797 | # TODO: why reversed? 798 | for child in reversed(metatile.children()): 799 | debug("%r: %s, %s", child, child.render, child.is_empty) 800 | if child.render: 801 | self.work_stack.push(child) 802 | elif child.is_empty: 803 | self.tiles_skipped += ( len(child.tiles) * 804 | utils.pyramid_count(child.z, opts.max_zoom) ) 805 | self.progress(child, format="empty") 806 | 807 | self.tiles_rendered += len(metatile.tiles) 808 | self.came_back += 1 809 | 810 | self.progress(metatile, *metatile.times(), format="%8.3f, %8.3f, %8.3f, %8.3f") 811 | 812 | 813 | def finish(self): 814 | if self.opts.parallel != 'single': 815 | info('stopping threads/procs') 816 | # signal render threads to exit by sending empty request to queue 817 | for i in range(self.opts.threads): 818 | info("%d...", (i + 1)) 819 | self.new_work.put(None) 820 | 821 | while self.went_out > self.came_back: 822 | debug("sent: %d; returned: %d", self.went_out, self.came_back) 823 | data = self.info.get() # type: str, Any 824 | debug("<-- %s", data) 825 | 826 | self.handle_new_work(data) 827 | 828 | # wait for pending rendering jobs to complete 829 | if not self.opts.parallel == 'fork': 830 | self.new_work.join() 831 | self.store_thread.join() 832 | else: 833 | self.new_work.close() 834 | self.new_work.join_thread() 835 | 836 | for i in range(self.opts.threads): 837 | self.renderers[i].join() 838 | 839 | 840 | def parse_args(): 841 | parser = ArgumentParser() 842 | 843 | group = parser.add_mutually_exclusive_group() 844 | group.add_argument('-b', '--bbox', dest='bbox', default='-180,-85,180,85', metavar='W,S,E,N') 845 | group.add_argument('-B', '--bbox-name', dest='bbox_name', default=None) 846 | group.add_argument('-T', '--tiles', dest='tiles', default= None, nargs='+', metavar='[Z,X,Y|Z/X/Y]', 847 | help="Render this list of [meta]tiles.") 848 | group.add_argument('-c', '--coords', dest='coords', default=None, nargs='+', 849 | metavar='[Lat,Long|Lat/Long]', 850 | help="Render this exact coords as the center of the [meta]tile. Order is LatLong, like in 31˚S, 64°W") 851 | group.add_argument('-L', '--longlat', dest='longlat', default=None, nargs=2, 852 | metavar=('LONG', 'LAT'), 853 | help='Render this exact coords as the center of the [meta]tile. Order is LongLat as in X,Y') 854 | 855 | parser.add_argument('-n', '--min-zoom', dest='min_zoom', default=0, type=int) 856 | parser.add_argument('-x', '--max-zoom', dest='max_zoom', default=18, type=int) 857 | 858 | parser.add_argument('-i', '--input-file', dest='mapfile', default='osm.xml', 859 | help="MapnikXML format.") 860 | parser.add_argument('-f', '--format', dest='format', default='tiles', 861 | choices=('tiles', 'mbtiles', 'mod_tile', 'test', 'svg', 'pdf')) 862 | parser.add_argument('-F', '--tile-file-format', dest='tile_file_format', default='png', 863 | choices=('png', 'jpeg', 'svg', 'pdf')) 864 | parser.add_argument('-O', '--tile-file-format-options', dest='tile_file_format_options', default='', 865 | help='Check https://github.com/mapnik/mapnik/wiki/') 866 | parser.add_argument('-o', '--output-dir', dest='tile_dir', default='tiles/') 867 | parser.add_argument( '--filename-pattern', dest='filename_pattern', default=None, 868 | help="Pattern may include {base_dir}, {x}, {y} and {z}.") 869 | 870 | # TODO: check it's a power of 2 871 | parser.add_argument('-m', '--metatile-size', dest='metatile_size', default=1, type=int, 872 | help='Must be a power of two.') 873 | 874 | parser.add_argument('-t', '--threads', dest='threads', default=utils.NUM_CPUS, 875 | type=int) 876 | parser.add_argument('-p', '--parallel-method', dest='parallel', default='fork', 877 | choices=('threads', 'fork', 'single')) 878 | parser.add_argument( '--store-thread', dest='store_thread', default=False, 879 | action='store_true', help="Have a separate process/thread for storing the tiles.") 880 | 881 | parser.add_argument('-X', '--skip-existing', dest='skip_existing', default=False, 882 | action='store_true') 883 | # TODO: newer than input_file 884 | parser.add_argument('-N', '--skip-newer', dest='skip_newer', default=None, 885 | type=float, metavar='DAYS') 886 | parser.add_argument('-M', '--missing-as-new', dest='missing_as_new', default=False, 887 | action='store_true', help="missing tiles in a meta tile count as newer, so we don't re-render metatiles with empty tiles.") 888 | parser.add_argument('-e', '--empty-color', dest='empty_color', metavar='[#]RRGGBB', required=True, 889 | help='Define the color of empty space (usually sea/ocean color) for empty tile detection.') 890 | parser.add_argument('-s', '--empty-size', dest='empty_size', type=int, default=103, 891 | help='The byte size of empty tiles.') 892 | parser.add_argument('-E', '--empty', dest='empty', default='skip', 893 | choices=('skip', 'link', 'write')) 894 | 895 | parser.add_argument( '--debug', dest='debug', default=False, 896 | action='store_true') 897 | parser.add_argument('-l', '--log-file', dest='log_file', default=None) 898 | parser.add_argument( '--dry-run', dest='dry_run', default=False, 899 | action='store_true') 900 | 901 | parser.add_argument( '--mapnik-debug', dest='mapnik_debug', default=False, 902 | action='store_true', help='''Turn on mapnik's debug logs.''') 903 | parser.add_argument( '--mapnik-strict', dest='mapnik_strict', default=False, 904 | action='store_true', help='''Use Mapnik's strict mode.''') 905 | 906 | # TODO: buffer size (256?) 907 | opts = parser.parse_args() 908 | 909 | # verboseness 910 | if opts.debug: 911 | logging.basicConfig(level=logging.DEBUG, format=long_format) 912 | else: 913 | logging.basicConfig(level=logging.INFO, format=short_format) 914 | 915 | if opts.mapnik_debug: 916 | mapnik.logger.set_severity(mapnik.severity_type.Debug) 917 | 918 | debug(opts) 919 | 920 | ## log outputs 921 | if opts.log_file is not None: 922 | root = logging.getLogger() 923 | file_handler = logging.FileHandler(opts.log_file) 924 | 925 | if opts.debug: 926 | file_handler.setFormatter(logging.Formatter(long_format)) 927 | file_handler.setLevel(logging.DEBUG) 928 | # the root logger will be pre-filtering by level 929 | # so we need to set its level to the lowest possible 930 | root.setLevel(logging.DEBUG) 931 | else: 932 | file_handler.setFormatter(logging.Formatter(short_format)) 933 | file_handler.setLevel(logging.INFO) 934 | root.setLevel(logging.INFO) 935 | 936 | root.addHandler(file_handler) 937 | 938 | ## tile_dir 939 | if opts.format == 'tiles' and opts.tile_dir[-1] != '/': 940 | # we need the trailing /, it's actually a series of BUG s in render_tiles() 941 | opts.tile_dir += '/' 942 | 943 | opts.tile_dir = os.path.abspath(opts.tile_dir) 944 | if opts.skip_newer is not None: 945 | opts.skip_newer = ( datetime.datetime.now() - 946 | datetime.timedelta(days=opts.skip_newer) ) 947 | 948 | ## bbox 949 | if opts.bbox_name is not None: 950 | # pick bbox from bboxes.ini 951 | atlas = map_utils.Atlas([ opts.bbox_name ]) 952 | opts.bbox = map_utils.BBox(atlas.maps[opts.bbox_name].bbox, opts.max_zoom) 953 | else: 954 | opts.bbox = map_utils.BBox([ float(s) for s in opts.bbox.split(',') ], opts.max_zoom) 955 | 956 | ## tile_file_format 957 | if opts.format == 'mod_tile' and opts.tile_file_format != 'png': 958 | warning(f"mod_tile format doesnt support '{opts.tile_file_format}'. Forcing 'png'.") 959 | opts.tile_file_format = 'png' 960 | 961 | ## misc stuff for certain formats 962 | if opts.format in ('mod_tile', 'test'): 963 | # mod_tile's tiles are really metatiles, 8x8 964 | # so we make the tile size 8 times a tile, and the metatile size is divided by 8 965 | opts.tile_size = 8 * 256 966 | 967 | # normalize values 968 | if opts.metatile_size < 8: 969 | opts.metatile_size = 8 970 | opts.metatile_size //= 8 971 | 972 | if opts.tiles is not None: 973 | metatiles = [] 974 | 975 | for tile_spec in opts.tiles: 976 | z, x, y = tiles.tile_spec2zxy(tile_spec) 977 | 978 | # normalize 979 | x //= 8 980 | y //= 8 981 | 982 | metatile = tiles.MetaTile(z, x, y, opts.metatile_size, opts.tile_size) 983 | metatiles.append(metatile) 984 | 985 | opts.tiles = metatiles 986 | else: 987 | # NOTE: no high res support 988 | opts.tile_size = 256 989 | if opts.tiles is not None: 990 | metatiles = [] 991 | 992 | for tile_spec in opts.tiles: 993 | z, x, y = tiles.tile_spec2zxy(tile_spec) 994 | metatile = tiles.MetaTile(z, x, y, opts.metatile_size, opts.tile_size) 995 | metatiles.append(metatile) 996 | 997 | opts.tiles = metatiles 998 | 999 | # TODO: svg is too special? 1000 | if opts.format in ('svg', 'pdf'): 1001 | if opts.coords is None and opts.longlat is None: 1002 | error('SVG/PDF formats only work with --coords or --longlat.') 1003 | sys.exit(1) 1004 | 1005 | if opts.parallel != 'single': 1006 | warning('SVG/PDF formats. Forcing single thread.') 1007 | opts.parallel = 'single' 1008 | 1009 | if opts.store_thread: 1010 | warning('SVG/PDF formats. Forcing no store thread.') 1011 | opts.store_thread = False 1012 | 1013 | if opts.tile_file_format != opts.format: 1014 | warning('SVG/PDF formats. Forcing file format.') 1015 | opts.tile_file_format = opts.format 1016 | 1017 | # lazy import 1018 | debug('loading cairo for SVG/PDF') 1019 | global cairo 1020 | import cairo 1021 | 1022 | ## convert coordinates 1023 | if opts.coords is not None or opts.longlat is not None: 1024 | opts.tile_size = 1024 1025 | 1026 | if opts.coords is not None: 1027 | # input is Lat,Long, but tileproj works with Lon,Lat; convert 1028 | new_coords = [] 1029 | 1030 | for coord in opts.coords: 1031 | data = coord.split('/') 1032 | 1033 | if len(data) == 3: 1034 | # it has a zoom level spec too, use that as min_zoom, max_zoom 1035 | zoom, lat, long = data 1036 | opts.min_zoom = int(zoom) 1037 | opts.max_zoom = opts.min_zoom 1038 | else: 1039 | lat, long = data 1040 | 1041 | new_coords.append( (float(long), float(lat)) ) 1042 | 1043 | opts.coords = new_coords 1044 | 1045 | elif opts.longlat is not None: 1046 | # input is Long Lat already 1047 | long, lat = opts.longlat 1048 | opts.coords = [ (float(long), float(lat)) ] 1049 | 1050 | debug(opts.coords) 1051 | 1052 | metatiles = [] 1053 | 1054 | for coord in opts.coords: 1055 | for z in range(opts.min_zoom, opts.max_zoom + 1): 1056 | # TODO: maybe move this conversion to PixelTile 1057 | x, y = tiles.tileproj.lon_lat2pixel(coord, z) 1058 | tile = tiles.PixelTile(z, x, y, opts.tile_size) 1059 | metatiles.append(tile) 1060 | 1061 | opts.tiles = metatiles 1062 | 1063 | ## parallel + threads 1064 | if opts.threads == 1: 1065 | opts.parallel == 'single' 1066 | 1067 | # I need this for ... what? 1068 | if opts.parallel == 'single': 1069 | opts.threads = 1 1070 | opts.store_thread = False 1071 | elif opts.parallel == 'fork': 1072 | debug('mp.Process()') 1073 | opts.parallel_factory = multiprocessing.Process 1074 | elif opts.parallel == 'threads': 1075 | debug('th.Thread()') 1076 | opts.parallel_factory = threading.Thread 1077 | 1078 | ## empty_color 1079 | # TODO: accept other formats 1080 | try: 1081 | # cut the leading # 1082 | if len(opts.empty_color) == 7 and opts.empty_color[0] == '#': 1083 | opts.empty_color = opts.empty_color[1:] 1084 | 1085 | opts.empty_color = bytes([ int(opts.empty_color[index:index + 2], 16) 1086 | for index in (0, 2, 4) ]) 1087 | except ValueError: 1088 | parser.print_help() 1089 | sys.exit(1) 1090 | 1091 | ## more_opts, for tile backends 1092 | opts.more_opts = {} 1093 | if opts.filename_pattern is not None: 1094 | opts.more_opts['filename_pattern'] = opts.filename_pattern 1095 | 1096 | # semantic opts 1097 | opts.single_tiles = opts.tiles is not None 1098 | opts.push_children = not opts.single_tiles 1099 | 1100 | # debug(opts) 1101 | info(opts) 1102 | 1103 | return opts 1104 | 1105 | 1106 | if __name__ == "__main__": 1107 | opts = parse_args() 1108 | 1109 | master = Master(opts) 1110 | 1111 | # fixes for locally installed mapnik 1112 | mapnik.register_fonts ('/usr/share/fonts/') 1113 | mapnik.register_plugins ('/home/mdione/local/lib/mapnik/input/') 1114 | info(mapnik.__file__) 1115 | 1116 | master.render_tiles() 1117 | --------------------------------------------------------------------------------