├── .gitignore ├── LICENSE ├── apply-osmchange.py ├── globalmaptiles.py ├── insert_osm_data.py ├── insert_tiled_osm_data.py ├── map_server.py ├── readme.md └── tile_server.py /.gitignore: -------------------------------------------------------------------------------- 1 | \#*\# 2 | .* 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Ian Dees 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /apply-osmchange.py: -------------------------------------------------------------------------------- 1 | """Applies an OSMChange file to the database""" 2 | 3 | import sys 4 | import os 5 | import time 6 | import urllib2 7 | import StringIO 8 | import gzip 9 | from datetime import datetime 10 | from xml.sax import make_parser 11 | from xml.sax.handler import ContentHandler 12 | from pymongo import Connection 13 | 14 | def convert_time(isotime): 15 | "Returns the time string as a time tuple" 16 | t = datetime.strptime(isotime, "%Y-%m-%dT%H:%M:%SZ") 17 | return time.mktime(t.timetuple()) 18 | 19 | class OsmChangeHandler(ContentHandler): 20 | """This ContentHandler works with the OSMChange XML file""" 21 | def __init__(self, client): 22 | """Initializes the OsmChange object""" 23 | self.action = "" 24 | self.record = {} 25 | self.nodes = [] 26 | self.ways = [] 27 | self.relations = [] 28 | self.client = client 29 | 30 | def fillDefault(self, attrs): 31 | """Fills in default attributes for new records""" 32 | self.record['id'] = long(attrs['id']) 33 | self.record['timestamp'] = convert_time(attrs['timestamp']) 34 | self.record['tags'] = {} 35 | if attrs.has_key('user'): 36 | self.record['user'] = attrs['user'] 37 | if attrs.has_key('uid'): 38 | self.record['uid'] = long(attrs['uid']) 39 | if attrs.has_key('version'): 40 | self.record['version'] = int(attrs['version']) 41 | if attrs.has_key('changeset'): 42 | self.record['changeset'] = long(attrs['changeset']) 43 | 44 | def startElement(self, name, attrs): 45 | """Parse the XML element at the start""" 46 | if name in ['create', 'modify', 'delete']: 47 | self.action = name 48 | elif name == 'node': 49 | self.record = {} 50 | self.fillDefault(attrs) 51 | self.record['loc'] = {'lat': float(attrs['lat']), 'lon': float(attrs['lon'])} 52 | elif name == 'tag': 53 | # MongoDB doesn't let us have dots in the key names. 54 | k = attrs['k'] 55 | k = k.replace('.', ',,') 56 | self.record['tags'][k] = attrs['v'] 57 | elif name == 'way': 58 | self.fillDefault(attrs) 59 | self.record['nodes'] = [] 60 | elif name == 'relation': 61 | self.fillDefault(attrs) 62 | self.record['members'] = [] 63 | elif name == 'nd': 64 | ref = long(attrs['ref']) 65 | self.record['nodes'].append(ref) 66 | 67 | nodes2ways = self.client.osm.nodes.find_one({ 'id' : ref }) 68 | if nodes2ways: 69 | if 'ways' not in nodes2ways: 70 | nodes2ways['ways'] = [] 71 | nodes2ways['ways'].append(self.record['id']) 72 | self.client.osm.nodes.save(nodes2ways) 73 | else: 74 | print "Node %d ref'd by way %d not in file." % \ 75 | (ref, self.record['id']) 76 | elif name == 'member': 77 | ref = long(attrs['ref']) 78 | member = {'type': attrs['type'], 79 | 'ref': ref, 80 | 'role': attrs['role']} 81 | self.record['members'].append(member) 82 | 83 | if attrs['type'] == 'way': 84 | ways2relations = self.client.osm.ways.find_one({ 'id' : ref}) 85 | if ways2relations: 86 | if 'relations' not in ways2relations: 87 | ways2relations['relations'] = [] 88 | ways2relations['relations'].append(self.record['id']) 89 | self.client.osm.ways.save(ways2relations) 90 | elif attrs['type'] == 'node': 91 | nodes2relations = self.client.osm.nodes.find_one({ 'id' : ref}) 92 | if nodes2relations: 93 | if 'relations' not in nodes2relations: 94 | nodes2relations['relations'] = [] 95 | nodes2relations['relations'].append(self.record['id']) 96 | self.client.osm.nodes.save(nodes2relations) 97 | elif name == 'node': 98 | self.record['loc'] = {'lat': float(attrs['lat']), 99 | 'lon': float(attrs['lon'])} 100 | self.fillDefault(attrs) 101 | 102 | elif name == 'tag': 103 | # MongoDB doesn't let us have dots in the key names. 104 | k = attrs['k'] 105 | k = k.replace('.', ',,') 106 | self.record['tags'][k] = attrs['v'] 107 | elif name == 'way': 108 | self.fillDefault(attrs) 109 | self.record['nodes'] = [] 110 | elif name == 'relation': 111 | self.fillDefault(attrs) 112 | self.record['members'] = [] 113 | elif name == 'nd': 114 | ref = long(attrs['ref']) 115 | self.record['nodes'].append(ref) 116 | elif name == 'member': 117 | ref = long(attrs['ref']) 118 | member = {'type': attrs['type'], 119 | 'ref': ref, 120 | 'role': attrs['role']} 121 | self.record['members'].append(member) 122 | 123 | def endElement(self, name): 124 | """Finish parsing osm objects or actions""" 125 | if name in ('node', 'way', 'relation'): 126 | self.type = name 127 | if self.action == 'delete': 128 | self.record['visible'] = False 129 | if self.type == 'way': 130 | nodes = self.client.osm.nodes.find({ 'id': { '$in': self.record['nodes'] } }, 131 | { 'loc': 1, '_id': 0 }) 132 | self.record['loc'] = [] 133 | for node in nodes: 134 | self.record['loc'].append(node['loc']) 135 | getattr(self, name + 's').append(self.record) 136 | elif name in ('create', 'modify', 'delete'): 137 | if name == 'create': 138 | for coll in ('nodes', 'ways', 'relations'): 139 | if getattr(self, coll): 140 | getattr(self.client.osm, coll).insert(getattr(self, coll)) 141 | elif name == 'modify': 142 | for coll in ('nodes', 'ways', 'relations'): 143 | if getattr(self, coll): 144 | primitive_list = getattr(self, coll) 145 | for prim in primitive_list: 146 | getattr(self.client.osm, coll).update({'id': prim['id']}, 147 | prim) 148 | elif name == 'delete': 149 | for coll in ('nodes', 'ways', 'relations'): 150 | if getattr(self, coll): 151 | primitive_list = getattr(self, coll) 152 | for prim in primitive_list: 153 | getattr(self.client.osm, coll).remove({'id': prim['id']}) 154 | self.action = None 155 | 156 | if __name__ == "__main__": 157 | client = Connection() 158 | parser = make_parser() 159 | 160 | keepGoing = True 161 | 162 | while keepGoing: 163 | # Read the state.txt 164 | sf = open('state.txt', 'r') 165 | 166 | state = {} 167 | for line in sf: 168 | if line[0] == '#': 169 | continue 170 | (k, v) = line.split('=') 171 | state[k] = v.strip().replace("\\:", ":") 172 | 173 | # Grab the sequence number and build a URL out of it 174 | sqnStr = state['sequenceNumber'].zfill(9) 175 | url = "http://planet.openstreetmap.org/minute-replicate/%s/%s/%s.osc.gz" % (sqnStr[0:3], sqnStr[3:6], sqnStr[6:9]) 176 | 177 | print "Downloading change file (%s)." % (url) 178 | content = urllib2.urlopen(url) 179 | content = StringIO.StringIO(content.read()) 180 | gzipper = gzip.GzipFile(fileobj=content) 181 | 182 | print "Parsing change file." 183 | handler = OsmChangeHandler(client) 184 | parser.setContentHandler(handler) 185 | parser.parse(gzipper) 186 | 187 | # Download the next state file 188 | nextSqn = int(state['sequenceNumber']) + 1 189 | sqnStr = str(nextSqn).zfill(9) 190 | url = "http://planet.openstreetmap.org/minute-replicate/%s/%s/%s.state.txt" % (sqnStr[0:3], sqnStr[3:6], sqnStr[6:9]) 191 | try: 192 | u = urllib2.urlopen(url) 193 | statefile = open('state.txt', 'w') 194 | statefile.write(u.read()) 195 | statefile.close() 196 | except Exception, e: 197 | keepGoing = False 198 | print e 199 | 200 | client.disconnect() 201 | -------------------------------------------------------------------------------- /globalmaptiles.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ############################################################################### 3 | # $Id$ 4 | # 5 | # Project: GDAL2Tiles, Google Summer of Code 2007 & 2008 6 | # Global Map Tiles Classes 7 | # Purpose: Convert a raster into TMS tiles, create KML SuperOverlay EPSG:4326, 8 | # generate a simple HTML viewers based on Google Maps and OpenLayers 9 | # Author: Klokan Petr Pridal, klokan at klokan dot cz 10 | # Web: http://www.klokan.cz/projects/gdal2tiles/ 11 | # 12 | ############################################################################### 13 | # Copyright (c) 2008 Klokan Petr Pridal. All rights reserved. 14 | # 15 | # Permission is hereby granted, free of charge, to any person obtaining a 16 | # copy of this software and associated documentation files (the "Software"), 17 | # to deal in the Software without restriction, including without limitation 18 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, 19 | # and/or sell copies of the Software, and to permit persons to whom the 20 | # Software is furnished to do so, subject to the following conditions: 21 | # 22 | # The above copyright notice and this permission notice shall be included 23 | # in all copies or substantial portions of the Software. 24 | # 25 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 26 | # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 27 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 28 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 29 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 30 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 31 | # DEALINGS IN THE SOFTWARE. 32 | ############################################################################### 33 | 34 | """ 35 | globalmaptiles.py 36 | 37 | Global Map Tiles as defined in Tile Map Service (TMS) Profiles 38 | ============================================================== 39 | 40 | Functions necessary for generation of global tiles used on the web. 41 | It contains classes implementing coordinate conversions for: 42 | 43 | - GlobalMercator (based on EPSG:900913 = EPSG:3785) 44 | for Google Maps, Yahoo Maps, Microsoft Maps compatible tiles 45 | - GlobalGeodetic (based on EPSG:4326) 46 | for OpenLayers Base Map and Google Earth compatible tiles 47 | 48 | More info at: 49 | 50 | http://wiki.osgeo.org/wiki/Tile_Map_Service_Specification 51 | http://wiki.osgeo.org/wiki/WMS_Tiling_Client_Recommendation 52 | http://msdn.microsoft.com/en-us/library/bb259689.aspx 53 | http://code.google.com/apis/maps/documentation/overlays.html#Google_Maps_Coordinates 54 | 55 | Created by Klokan Petr Pridal on 2008-07-03. 56 | Google Summer of Code 2008, project GDAL2Tiles for OSGEO. 57 | 58 | In case you use this class in your product, translate it to another language 59 | or find it usefull for your project please let me know. 60 | My email: klokan at klokan dot cz. 61 | I would like to know where it was used. 62 | 63 | Class is available under the open-source GDAL license (www.gdal.org). 64 | """ 65 | 66 | import math 67 | 68 | class GlobalMercator(object): 69 | """ 70 | TMS Global Mercator Profile 71 | --------------------------- 72 | 73 | Functions necessary for generation of tiles in Spherical Mercator projection, 74 | EPSG:900913 (EPSG:gOOglE, Google Maps Global Mercator), EPSG:3785, OSGEO:41001. 75 | 76 | Such tiles are compatible with Google Maps, Microsoft Virtual Earth, Yahoo Maps, 77 | UK Ordnance Survey OpenSpace API, ... 78 | and you can overlay them on top of base maps of those web mapping applications. 79 | 80 | Pixel and tile coordinates are in TMS notation (origin [0,0] in bottom-left). 81 | 82 | What coordinate conversions do we need for TMS Global Mercator tiles:: 83 | 84 | LatLon <-> Meters <-> Pixels <-> Tile 85 | 86 | WGS84 coordinates Spherical Mercator Pixels in pyramid Tiles in pyramid 87 | lat/lon XY in metres XY pixels Z zoom XYZ from TMS 88 | EPSG:4326 EPSG:900913 89 | .----. --------- -- TMS 90 | / \ <-> | | <-> /----/ <-> Google 91 | \ / | | /--------/ QuadTree 92 | ----- --------- /------------/ 93 | KML, public WebMapService Web Clients TileMapService 94 | 95 | What is the coordinate extent of Earth in EPSG:900913? 96 | 97 | [-20037508.342789244, -20037508.342789244, 20037508.342789244, 20037508.342789244] 98 | Constant 20037508.342789244 comes from the circumference of the Earth in meters, 99 | which is 40 thousand kilometers, the coordinate origin is in the middle of extent. 100 | In fact you can calculate the constant as: 2 * math.pi * 6378137 / 2.0 101 | $ echo 180 85 | gdaltransform -s_srs EPSG:4326 -t_srs EPSG:900913 102 | Polar areas with abs(latitude) bigger then 85.05112878 are clipped off. 103 | 104 | What are zoom level constants (pixels/meter) for pyramid with EPSG:900913? 105 | 106 | whole region is on top of pyramid (zoom=0) covered by 256x256 pixels tile, 107 | every lower zoom level resolution is always divided by two 108 | initialResolution = 20037508.342789244 * 2 / 256 = 156543.03392804062 109 | 110 | What is the difference between TMS and Google Maps/QuadTree tile name convention? 111 | 112 | The tile raster itself is the same (equal extent, projection, pixel size), 113 | there is just different identification of the same raster tile. 114 | Tiles in TMS are counted from [0,0] in the bottom-left corner, id is XYZ. 115 | Google placed the origin [0,0] to the top-left corner, reference is XYZ. 116 | Microsoft is referencing tiles by a QuadTree name, defined on the website: 117 | http://msdn2.microsoft.com/en-us/library/bb259689.aspx 118 | 119 | The lat/lon coordinates are using WGS84 datum, yeh? 120 | 121 | Yes, all lat/lon we are mentioning should use WGS84 Geodetic Datum. 122 | Well, the web clients like Google Maps are projecting those coordinates by 123 | Spherical Mercator, so in fact lat/lon coordinates on sphere are treated as if 124 | the were on the WGS84 ellipsoid. 125 | 126 | From MSDN documentation: 127 | To simplify the calculations, we use the spherical form of projection, not 128 | the ellipsoidal form. Since the projection is used only for map display, 129 | and not for displaying numeric coordinates, we don't need the extra precision 130 | of an ellipsoidal projection. The spherical projection causes approximately 131 | 0.33 percent scale distortion in the Y direction, which is not visually noticable. 132 | 133 | How do I create a raster in EPSG:900913 and convert coordinates with PROJ.4? 134 | 135 | You can use standard GIS tools like gdalwarp, cs2cs or gdaltransform. 136 | All of the tools supports -t_srs 'epsg:900913'. 137 | 138 | For other GIS programs check the exact definition of the projection: 139 | More info at http://spatialreference.org/ref/user/google-projection/ 140 | The same projection is degined as EPSG:3785. WKT definition is in the official 141 | EPSG database. 142 | 143 | Proj4 Text: 144 | +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 145 | +k=1.0 +units=m +nadgrids=@null +no_defs 146 | 147 | Human readable WKT format of EPGS:900913: 148 | PROJCS["Google Maps Global Mercator", 149 | GEOGCS["WGS 84", 150 | DATUM["WGS_1984", 151 | SPHEROID["WGS 84",6378137,298.2572235630016, 152 | AUTHORITY["EPSG","7030"]], 153 | AUTHORITY["EPSG","6326"]], 154 | PRIMEM["Greenwich",0], 155 | UNIT["degree",0.0174532925199433], 156 | AUTHORITY["EPSG","4326"]], 157 | PROJECTION["Mercator_1SP"], 158 | PARAMETER["central_meridian",0], 159 | PARAMETER["scale_factor",1], 160 | PARAMETER["false_easting",0], 161 | PARAMETER["false_northing",0], 162 | UNIT["metre",1, 163 | AUTHORITY["EPSG","9001"]]] 164 | """ 165 | 166 | def __init__(self, tileSize=256): 167 | "Initialize the TMS Global Mercator pyramid" 168 | self.tileSize = tileSize 169 | self.initialResolution = 2 * math.pi * 6378137 / self.tileSize 170 | # 156543.03392804062 for tileSize 256 pixels 171 | self.originShift = 2 * math.pi * 6378137 / 2.0 172 | # 20037508.342789244 173 | 174 | def LatLonToMeters(self, lat, lon ): 175 | "Converts given lat/lon in WGS84 Datum to XY in Spherical Mercator EPSG:900913" 176 | 177 | mx = lon * self.originShift / 180.0 178 | my = math.log( math.tan((90 + lat) * math.pi / 360.0 )) / (math.pi / 180.0) 179 | 180 | my = my * self.originShift / 180.0 181 | return mx, my 182 | 183 | def MetersToLatLon(self, mx, my ): 184 | "Converts XY point from Spherical Mercator EPSG:900913 to lat/lon in WGS84 Datum" 185 | 186 | lon = (mx / self.originShift) * 180.0 187 | lat = (my / self.originShift) * 180.0 188 | 189 | lat = 180 / math.pi * (2 * math.atan( math.exp( lat * math.pi / 180.0)) - math.pi / 2.0) 190 | return lat, lon 191 | 192 | def PixelsToMeters(self, px, py, zoom): 193 | "Converts pixel coordinates in given zoom level of pyramid to EPSG:900913" 194 | 195 | res = self.Resolution( zoom ) 196 | mx = px * res - self.originShift 197 | my = py * res - self.originShift 198 | return mx, my 199 | 200 | def MetersToPixels(self, mx, my, zoom): 201 | "Converts EPSG:900913 to pyramid pixel coordinates in given zoom level" 202 | 203 | res = self.Resolution( zoom ) 204 | px = (mx + self.originShift) / res 205 | py = (my + self.originShift) / res 206 | return px, py 207 | 208 | def PixelsToTile(self, px, py): 209 | "Returns a tile covering region in given pixel coordinates" 210 | 211 | tx = int( math.ceil( px / float(self.tileSize) ) - 1 ) 212 | ty = int( math.ceil( py / float(self.tileSize) ) - 1 ) 213 | return tx, ty 214 | 215 | def PixelsToRaster(self, px, py, zoom): 216 | "Move the origin of pixel coordinates to top-left corner" 217 | 218 | mapSize = self.tileSize << zoom 219 | return px, mapSize - py 220 | 221 | def MetersToTile(self, mx, my, zoom): 222 | "Returns tile for given mercator coordinates" 223 | 224 | px, py = self.MetersToPixels( mx, my, zoom) 225 | return self.PixelsToTile( px, py) 226 | 227 | def TileBounds(self, tx, ty, zoom): 228 | "Returns bounds of the given tile in EPSG:900913 coordinates" 229 | 230 | minx, miny = self.PixelsToMeters( tx*self.tileSize, ty*self.tileSize, zoom ) 231 | maxx, maxy = self.PixelsToMeters( (tx+1)*self.tileSize, (ty+1)*self.tileSize, zoom ) 232 | return ( minx, miny, maxx, maxy ) 233 | 234 | def TileLatLonBounds(self, tx, ty, zoom ): 235 | "Returns bounds of the given tile in latutude/longitude using WGS84 datum" 236 | 237 | bounds = self.TileBounds( tx, ty, zoom) 238 | minLat, minLon = self.MetersToLatLon(bounds[0], bounds[1]) 239 | maxLat, maxLon = self.MetersToLatLon(bounds[2], bounds[3]) 240 | 241 | return ( minLat, minLon, maxLat, maxLon ) 242 | 243 | def Resolution(self, zoom ): 244 | "Resolution (meters/pixel) for given zoom level (measured at Equator)" 245 | 246 | # return (2 * math.pi * 6378137) / (self.tileSize * 2**zoom) 247 | return self.initialResolution / (2**zoom) 248 | 249 | def ZoomForPixelSize(self, pixelSize ): 250 | "Maximal scaledown zoom of the pyramid closest to the pixelSize." 251 | 252 | for i in range(30): 253 | if pixelSize > self.Resolution(i): 254 | return i-1 if i!=0 else 0 # We don't want to scale up 255 | 256 | def GoogleTile(self, tx, ty, zoom): 257 | "Converts TMS tile coordinates to Google Tile coordinates" 258 | 259 | # coordinate origin is moved from bottom-left to top-left corner of the extent 260 | return tx, (2**zoom - 1) - ty 261 | 262 | def QuadTree(self, tx, ty, zoom ): 263 | "Converts TMS tile coordinates to Microsoft QuadTree" 264 | 265 | quadKey = "" 266 | ty = (2**zoom - 1) - ty 267 | for i in range(zoom, 0, -1): 268 | digit = 0 269 | mask = 1 << (i-1) 270 | if (tx & mask) != 0: 271 | digit += 1 272 | if (ty & mask) != 0: 273 | digit += 2 274 | quadKey += str(digit) 275 | 276 | return quadKey 277 | 278 | #--------------------- 279 | 280 | class GlobalGeodetic(object): 281 | """ 282 | TMS Global Geodetic Profile 283 | --------------------------- 284 | 285 | Functions necessary for generation of global tiles in Plate Carre projection, 286 | EPSG:4326, "unprojected profile". 287 | 288 | Such tiles are compatible with Google Earth (as any other EPSG:4326 rasters) 289 | and you can overlay the tiles on top of OpenLayers base map. 290 | 291 | Pixel and tile coordinates are in TMS notation (origin [0,0] in bottom-left). 292 | 293 | What coordinate conversions do we need for TMS Global Geodetic tiles? 294 | 295 | Global Geodetic tiles are using geodetic coordinates (latitude,longitude) 296 | directly as planar coordinates XY (it is also called Unprojected or Plate 297 | Carre). We need only scaling to pixel pyramid and cutting to tiles. 298 | Pyramid has on top level two tiles, so it is not square but rectangle. 299 | Area [-180,-90,180,90] is scaled to 512x256 pixels. 300 | TMS has coordinate origin (for pixels and tiles) in bottom-left corner. 301 | Rasters are in EPSG:4326 and therefore are compatible with Google Earth. 302 | 303 | LatLon <-> Pixels <-> Tiles 304 | 305 | WGS84 coordinates Pixels in pyramid Tiles in pyramid 306 | lat/lon XY pixels Z zoom XYZ from TMS 307 | EPSG:4326 308 | .----. ---- 309 | / \ <-> /--------/ <-> TMS 310 | \ / /--------------/ 311 | ----- /--------------------/ 312 | WMS, KML Web Clients, Google Earth TileMapService 313 | """ 314 | 315 | def __init__(self, tileSize = 256): 316 | self.tileSize = tileSize 317 | 318 | def LatLonToPixels(self, lat, lon, zoom): 319 | "Converts lat/lon to pixel coordinates in given zoom of the EPSG:4326 pyramid" 320 | 321 | res = 180 / 256.0 / 2**zoom 322 | px = (180 + lat) / res 323 | py = (90 + lon) / res 324 | return px, py 325 | 326 | def PixelsToTile(self, px, py): 327 | "Returns coordinates of the tile covering region in pixel coordinates" 328 | 329 | tx = int( math.ceil( px / float(self.tileSize) ) - 1 ) 330 | ty = int( math.ceil( py / float(self.tileSize) ) - 1 ) 331 | return tx, ty 332 | 333 | def Resolution(self, zoom ): 334 | "Resolution (arc/pixel) for given zoom level (measured at Equator)" 335 | 336 | return 180 / 256.0 / 2**zoom 337 | #return 180 / float( 1 << (8+zoom) ) 338 | 339 | def TileBounds(tx, ty, zoom): 340 | "Returns bounds of the given tile" 341 | res = 180 / 256.0 / 2**zoom 342 | return ( 343 | tx*256*res - 180, 344 | ty*256*res - 90, 345 | (tx+1)*256*res - 180, 346 | (ty+1)*256*res - 90 347 | ) 348 | 349 | if __name__ == "__main__": 350 | import sys, os 351 | 352 | def Usage(s = ""): 353 | print "Usage: globalmaptiles.py [-profile 'mercator'|'geodetic'] zoomlevel lat lon [latmax lonmax]" 354 | print 355 | if s: 356 | print s 357 | print 358 | print "This utility prints for given WGS84 lat/lon coordinates (or bounding box) the list of tiles" 359 | print "covering specified area. Tiles are in the given 'profile' (default is Google Maps 'mercator')" 360 | print "and in the given pyramid 'zoomlevel'." 361 | print "For each tile several information is printed including bonding box in EPSG:900913 and WGS84." 362 | sys.exit(1) 363 | 364 | profile = 'mercator' 365 | zoomlevel = None 366 | lat, lon, latmax, lonmax = None, None, None, None 367 | boundingbox = False 368 | 369 | argv = sys.argv 370 | i = 1 371 | while i < len(argv): 372 | arg = argv[i] 373 | 374 | if arg == '-profile': 375 | i = i + 1 376 | profile = argv[i] 377 | 378 | if zoomlevel is None: 379 | zoomlevel = int(argv[i]) 380 | elif lat is None: 381 | lat = float(argv[i]) 382 | elif lon is None: 383 | lon = float(argv[i]) 384 | elif latmax is None: 385 | latmax = float(argv[i]) 386 | elif lonmax is None: 387 | lonmax = float(argv[i]) 388 | else: 389 | Usage("ERROR: Too many parameters") 390 | 391 | i = i + 1 392 | 393 | if profile != 'mercator': 394 | Usage("ERROR: Sorry, given profile is not implemented yet.") 395 | 396 | if zoomlevel == None or lat == None or lon == None: 397 | Usage("ERROR: Specify at least 'zoomlevel', 'lat' and 'lon'.") 398 | if latmax is not None and lonmax is None: 399 | Usage("ERROR: Both 'latmax' and 'lonmax' must be given.") 400 | 401 | if latmax != None and lonmax != None: 402 | if latmax < lat: 403 | Usage("ERROR: 'latmax' must be bigger then 'lat'") 404 | if lonmax < lon: 405 | Usage("ERROR: 'lonmax' must be bigger then 'lon'") 406 | boundingbox = (lon, lat, lonmax, latmax) 407 | 408 | tz = zoomlevel 409 | mercator = GlobalMercator() 410 | 411 | mx, my = mercator.LatLonToMeters( lat, lon ) 412 | print "Spherical Mercator (ESPG:900913) coordinates for lat/lon: " 413 | print (mx, my) 414 | tminx, tminy = mercator.MetersToTile( mx, my, tz ) 415 | 416 | if boundingbox: 417 | mx, my = mercator.LatLonToMeters( latmax, lonmax ) 418 | print "Spherical Mercator (ESPG:900913) cooridnate for maxlat/maxlon: " 419 | print (mx, my) 420 | tmaxx, tmaxy = mercator.MetersToTile( mx, my, tz ) 421 | else: 422 | tmaxx, tmaxy = tminx, tminy 423 | 424 | for ty in range(tminy, tmaxy+1): 425 | for tx in range(tminx, tmaxx+1): 426 | tilefilename = "%s/%s/%s" % (tz, tx, ty) 427 | print tilefilename, "( TileMapService: z / x / y )" 428 | 429 | gx, gy = mercator.GoogleTile(tx, ty, tz) 430 | print "\tGoogle:", gx, gy 431 | quadkey = mercator.QuadTree(tx, ty, tz) 432 | print "\tQuadkey:", quadkey, '(',int(quadkey, 4),')' 433 | bounds = mercator.TileBounds( tx, ty, tz) 434 | print 435 | print "\tEPSG:900913 Extent: ", bounds 436 | wgsbounds = mercator.TileLatLonBounds( tx, ty, tz) 437 | print "\tWGS84 Extent:", wgsbounds 438 | print "\tgdalwarp -ts 256 256 -te %s %s %s %s %s %s_%s_%s.tif" % ( 439 | bounds[0], bounds[1], bounds[2], bounds[3], "", tz, tx, ty) 440 | print 441 | -------------------------------------------------------------------------------- /insert_osm_data.py: -------------------------------------------------------------------------------- 1 | """This program parses an OSM XML file and inserts the data in a 2 | MongoDB database""" 3 | 4 | import sys 5 | import os 6 | import time 7 | import pymongo 8 | from datetime import datetime 9 | #from xml.sax import make_parser 10 | #from xml.sax.handler import ContentHandler 11 | from pymongo import MongoClient 12 | from xml.etree.cElementTree import iterparse 13 | 14 | class OsmHandler(object): 15 | """Base class for parsing OSM XML data""" 16 | def __init__(self, client): 17 | self.client = client 18 | 19 | self.client.osm.nodes.ensure_index([('loc', pymongo.GEO2D)]) 20 | self.client.osm.nodes.ensure_index([('id', pymongo.ASCENDING), 21 | ('version', pymongo.DESCENDING)]) 22 | 23 | self.client.osm.ways.ensure_index([('loc', pymongo.GEO2D)]) 24 | self.client.osm.ways.ensure_index([('id', pymongo.ASCENDING), 25 | ('version', pymongo.DESCENDING)]) 26 | 27 | self.client.osm.relations.ensure_index([('id', pymongo.ASCENDING), 28 | ('version', pymongo.DESCENDING)]) 29 | self.stat_nodes = 0 30 | self.stat_ways = 0 31 | self.stat_relations = 0 32 | self.lastStatString = "" 33 | self.statsCount = 0 34 | 35 | def writeStatsToScreen(self): 36 | for char in self.lastStatString: 37 | sys.stdout.write('\b') 38 | self.lastStatString = "%dk nodes, %dk ways, %d relations" % (self.stat_nodes / 1000, 39 | self.stat_ways / 1000, 40 | self.stat_relations) 41 | sys.stdout.write(self.lastStatString) 42 | 43 | def fillDefault(self, attrs): 44 | ts = None 45 | """Fill in default record values""" 46 | record = dict(_id=long(attrs['id']), 47 | #ts=self.isoToTimestamp(attrs['timestamp']), 48 | ts=attrs['timestamp'] if 'timestamp' in attrs else None, 49 | tg=[], 50 | ky=[]) 51 | #record['_id'] = long(attrs['id']) 52 | #record['timestamp'] = self.isoToTimestamp(attrs['timestamp']) 53 | #record['tags'] = [] 54 | #record['keys'] = [] 55 | if attrs.has_key('user'): 56 | record['un'] = attrs['user'] 57 | if attrs.has_key('uid'): 58 | record['ui'] = long(attrs['uid']) 59 | if attrs.has_key('version'): 60 | record['v'] = int(attrs['version']) 61 | if attrs.has_key('changeset'): 62 | record['ch'] = long(attrs['changeset']) 63 | return record 64 | 65 | def isoToTimestamp(self, isotime): 66 | """Parse a date and return a time tuple""" 67 | t = datetime.strptime(isotime, "%Y-%m-%dT%H:%M:%SZ") 68 | return time.mktime(t.timetuple()) 69 | 70 | def parse(self, file_obj): 71 | nodes = [] 72 | ways = [] 73 | 74 | context = iter(iterparse(file_obj, events=('start', 'end'))) 75 | event, root = context.next() 76 | 77 | for (event, elem) in context: 78 | name = elem.tag 79 | attrs = elem.attrib 80 | 81 | if 'start' == event: 82 | """Parse the XML element at the start""" 83 | if name == 'node': 84 | record = self.fillDefault(attrs) 85 | loc = [float(attrs['lat']), 86 | float(attrs['lon'])] 87 | record['loc'] = loc 88 | elif name == 'tag': 89 | k = attrs['k'] 90 | v = attrs['v'] 91 | # MongoDB doesn't let us have dots in the key names. 92 | #k = k.replace('.', ',,') 93 | record['tg'].append((k, v)) 94 | record['ky'].append(k) 95 | elif name == 'way': 96 | # Insert remaining nodes 97 | if len(nodes) > 0: 98 | self.client.osm.nodes.insert(nodes) 99 | nodes = [] 100 | 101 | record = self.fillDefault(attrs) 102 | record['nd'] = [] 103 | elif name == 'relation': 104 | # Insert remaining ways 105 | if len(ways) > 0: 106 | self.client.osm.ways.insert(ways) 107 | ways = [] 108 | 109 | record = self.fillDefault(attrs) 110 | record['mm'] = [] 111 | elif name == 'nd': 112 | ref = long(attrs['ref']) 113 | record['nd'].append(ref) 114 | elif name == 'member': 115 | record['mm'].append(dict(type=attrs['type'], 116 | ref=long(attrs['ref']), 117 | role=attrs['role'])) 118 | 119 | if attrs['type'] == 'way': 120 | ways2relations = self.client.osm.ways.find_one({ '_id' : ref}) 121 | if ways2relations: 122 | if 'relations' not in ways2relations: 123 | ways2relations['relations'] = [] 124 | ways2relations['relations'].append(record['_id']) 125 | self.client.osm.ways.save(ways2relations) 126 | elif attrs['type'] == 'node': 127 | nodes2relations = self.client.osm.nodes.find_one({ '_id' : ref}) 128 | if nodes2relations: 129 | if 'relations' not in nodes2relations: 130 | nodes2relations['relations'] = [] 131 | nodes2relations['relations'].append(record['_id']) 132 | self.client.osm.nodes.save(nodes2relations) 133 | elif 'end' == event: 134 | """Finish parsing an element 135 | (only really used with nodes, ways and relations)""" 136 | if name == 'node': 137 | if len(record['tg']) == 0: 138 | del record['tg'] 139 | if len(record['ky']) == 0: 140 | del record['ky'] 141 | nodes.append(record) 142 | if len(nodes) > 2500: 143 | self.client.osm.nodes.insert(nodes) 144 | nodes = [] 145 | self.writeStatsToScreen() 146 | 147 | record = {} 148 | self.stat_nodes = self.stat_nodes + 1 149 | elif name == 'way': 150 | if len(record['tg']) == 0: 151 | del record['tg'] 152 | if len(record['ky']) == 0: 153 | del record['ky'] 154 | nds = dict((rec['_id'], rec) for rec in self.client.osm.nodes.find({ '_id': { '$in': record['nd'] } }, { 'loc': 1, '_id': 1 })) 155 | record['loc'] = [] 156 | for node in record['nd']: 157 | if node in nds: 158 | record['loc'].append(nds[node]['loc']) 159 | else: 160 | print 'node not found: '+ str(node) 161 | 162 | ways.append(record) 163 | if len(ways) > 2000: 164 | self.client.osm.ways.insert(ways) 165 | ways = [] 166 | 167 | record = {} 168 | self.statsCount = self.statsCount + 1 169 | if self.statsCount > 1000: 170 | self.writeStatsToScreen() 171 | self.statsCount = 0 172 | self.stat_ways = self.stat_ways + 1 173 | elif name == 'relation': 174 | if len(record['tg']) == 0: 175 | del record['tg'] 176 | if len(record['ky']) == 0: 177 | del record['ky'] 178 | self.client.osm.relations.save(record) 179 | record = {} 180 | self.statsCount = self.statsCount + 1 181 | if self.statsCount > 10: 182 | self.writeStatsToScreen() 183 | self.statsCount = 0 184 | self.stat_relations = self.stat_relations + 1 185 | elem.clear() 186 | root.clear() 187 | 188 | if __name__ == "__main__": 189 | if len(sys.argv) != 2: 190 | print "Usage: %s " % (sys.argv[0]) 191 | sys.exit(-1) 192 | 193 | filename = sys.argv[1] 194 | 195 | if not os.path.exists(filename): 196 | print "Path %s doesn't exist." % (filename) 197 | sys.exit(-1) 198 | 199 | client = MongoClient() 200 | #parser = make_parser() 201 | handler = OsmHandler(client) 202 | #parser.setContentHandler(handler) 203 | #parser.parse(open(filename)) 204 | handler.parse(open(filename)) 205 | client.disconnect() 206 | -------------------------------------------------------------------------------- /insert_tiled_osm_data.py: -------------------------------------------------------------------------------- 1 | """This program parses an OSM XML file and inserts the data in a 2 | MongoDB database""" 3 | 4 | import sys 5 | import os 6 | import time 7 | import pymongo 8 | from datetime import datetime 9 | from xml.sax import make_parser 10 | from xml.sax.handler import ContentHandler 11 | from pymongo import Connection 12 | 13 | from globalmaptiles import GlobalMercator 14 | 15 | class OsmHandler(ContentHandler): 16 | """Base class for parsing OSM XML data""" 17 | def __init__(self, client): 18 | self.proj = GlobalMercator() 19 | self.nodeRecords = [] 20 | self.wayRecords = [] 21 | self.relationRecords = [] 22 | self.record = {} 23 | self.nodeLocations = {} 24 | self.client = client 25 | 26 | self.stats = {'nodes': 0, 'ways': 0, 'relations': 0} 27 | self.lastStatString = "" 28 | self.statsCount = 0 29 | 30 | def writeStatsToScreen(self): 31 | for char in self.lastStatString: 32 | sys.stdout.write('\b') 33 | self.lastStatString = "%d nodes, %d ways, %d relations" % (self.stats['nodes'], 34 | self.stats['ways'], 35 | self.stats['relations']) 36 | sys.stdout.write(self.lastStatString) 37 | 38 | def fillDefault(self, attrs): 39 | """Fill in default record values""" 40 | self.record['_id'] = int(attrs['id']) 41 | self.record['ts'] = self.isoToTimestamp(attrs['timestamp']) 42 | self.record['tg'] = [] 43 | if attrs.has_key('user'): 44 | self.record['u'] = attrs['user'] 45 | if attrs.has_key('uid'): 46 | self.record['uid'] = int(attrs['uid']) 47 | if attrs.has_key('version'): 48 | self.record['v'] = int(attrs['version']) 49 | if attrs.has_key('changeset'): 50 | self.record['c'] = int(attrs['changeset']) 51 | 52 | def isoToTimestamp(self, isotime): 53 | """Parse a date and return a time tuple""" 54 | t = datetime.strptime(isotime, "%Y-%m-%dT%H:%M:%SZ") 55 | return time.mktime(t.timetuple()) 56 | 57 | def quadKey(self, lat, lon, zoom): 58 | (mx, my) = self.proj.LatLonToMeters(lat, lon) 59 | (tx, ty) = self.proj.MetersToTile(mx, my, zoom) 60 | return self.proj.QuadTree(tx, ty, zoom) 61 | 62 | def startElement(self, name, attrs): 63 | """Parse the XML element at the start""" 64 | if name == 'node': 65 | self.fillDefault(attrs) 66 | self.record['loc'] = {'lat': float(attrs['lat']), 67 | 'lon': float(attrs['lon'])} 68 | self.record['qk'] = self.quadKey(float(attrs['lat']), float(attrs['lon']), 17) 69 | self.nodeLocations[self.record['_id']] = self.record['qk'] 70 | elif name == 'changeset': 71 | self.fillDefault(attrs) 72 | elif name == 'tag': 73 | k = attrs['k'] 74 | v = attrs['v'] 75 | # MongoDB doesn't let us have dots in the key names. 76 | #k = k.replace('.', ',,') 77 | self.record['tg'].append((k, v)) 78 | elif name == 'way': 79 | self.fillDefault(attrs) 80 | self.record['n'] = [] 81 | self.record['loc'] = [] 82 | elif name == 'relation': 83 | self.fillDefault(attrs) 84 | self.record['m'] = [] 85 | elif name == 'nd': 86 | ref = int(attrs['ref']) 87 | self.record['n'].append(ref) 88 | refLoc = self.nodeLocations[ref] 89 | if refLoc not in self.record['loc']: 90 | self.record['loc'].append(refLoc) 91 | elif name == 'member': 92 | ref = int(attrs['ref']) 93 | member = {'type': attrs['type'], 94 | 'ref': ref, 95 | 'role': attrs['role']} 96 | self.record['m'].append(member) 97 | 98 | if attrs['type'] == 'way': 99 | ways2relations = self.client.osm.ways.find_one({ '_id' : ref}) 100 | if ways2relations: 101 | if 'relations' not in ways2relations: 102 | ways2relations['relations'] = [] 103 | ways2relations['relations'].append(self.record['_id']) 104 | self.client.osm.ways.save(ways2relations) 105 | elif attrs['type'] == 'node': 106 | nodes2relations = self.client.osm.nodes.find_one({ '_id' : ref}) 107 | if nodes2relations: 108 | if 'relations' not in nodes2relations: 109 | nodes2relations['relations'] = [] 110 | nodes2relations['relations'].append(self.record['_id']) 111 | self.client.osm.nodes.save(nodes2relations) 112 | 113 | def endElement(self, name): 114 | """Finish parsing an element 115 | (only really used with nodes, ways and relations)""" 116 | if name == 'node': 117 | self.nodeRecords.append(self.record) 118 | if len(self.nodeRecords) > 1500: 119 | self.client.osm.nodes.insert(self.nodeRecords) 120 | self.nodeRecords = [] 121 | self.writeStatsToScreen() 122 | self.record = {} 123 | self.stats['nodes'] = self.stats['nodes'] + 1 124 | elif name == 'way': 125 | # Clean up any existing nodes 126 | if len(self.nodeRecords) > 0: 127 | self.client.osm.nodes.insert(self.nodeRecords) 128 | self.nodeRecords = [] 129 | 130 | self.wayRecords.append(self.record) 131 | if len(self.wayRecords) > 100: 132 | self.client.osm.ways.insert(self.wayRecords) 133 | self.wayRecords = [] 134 | self.writeStatsToScreen() 135 | self.record = {} 136 | self.stats['ways'] = self.stats['ways'] + 1 137 | elif name == 'relation': 138 | self.client.osm.relations.save(self.record) 139 | self.record = {} 140 | self.statsCount = self.statsCount + 1 141 | if self.statsCount > 10: 142 | self.writeStatsToScreen() 143 | self.statsCount = 0 144 | self.stats['relations'] = self.stats['relations'] + 1 145 | 146 | if __name__ == "__main__": 147 | filename = sys.argv[1] 148 | 149 | if not os.path.exists(filename): 150 | print "Path %s doesn't exist." % (filename) 151 | sys.exit(-1) 152 | 153 | client = Connection() 154 | parser = make_parser() 155 | handler = OsmHandler(client) 156 | parser.setContentHandler(handler) 157 | parser.parse(open(filename)) 158 | client.disconnect() 159 | 160 | print 161 | -------------------------------------------------------------------------------- /map_server.py: -------------------------------------------------------------------------------- 1 | import pymongo 2 | from pymongo import Connection 3 | from xml.sax.saxutils import escape 4 | import re 5 | 6 | class OsmApi: 7 | def __init__(self): 8 | self.client = Connection() 9 | 10 | def getNodesQuery(self, query): 11 | cursor = self.client.osm.nodes.find(query) 12 | 13 | nodes = {} 14 | for row in cursor: 15 | nodes[row['_id']] = row 16 | 17 | return nodes 18 | 19 | def getNodes(self, query): 20 | nodes = self.getNodesQuery(query) 21 | 22 | return {'nodes': nodes.values()} 23 | 24 | def getWaysInBounds(self, box): 25 | return self.getWaysQuery([('bbox', box)]) 26 | 27 | def getWaysQuery(self, query): 28 | cursor = self.client.osm.ways.find(query) 29 | 30 | ways = {} 31 | for row in cursor: 32 | ways[row['_id']] = row 33 | 34 | return ways 35 | 36 | def getWays(self, query): 37 | ways = self.getWaysQuery(query) 38 | 39 | nodes = {} 40 | nodes = self.getNodesFromWays(ways, nodes) 41 | 42 | return {'nodes': nodes.values(), 'ways': ways.values()} 43 | 44 | def getNodesFromWays(self, ways, existingNodes): 45 | nodeIds = set() 46 | 47 | for way in ways.values(): 48 | for nodeId in way['nd']: 49 | nodeIds.add(nodeId) 50 | 51 | cursor = self.client.osm.nodes.find({'_id': {'$in': list(nodeIds)} }) 52 | 53 | for row in cursor: 54 | if row['_id'] not in existingNodes: 55 | existingNodes[row['_id']] = row 56 | 57 | return existingNodes 58 | 59 | 60 | def getWaysFromNodes(self, nodes): 61 | wayIds = set() 62 | 63 | for node in nodes: 64 | wayIdsFromNode = self.getWayIdsUsingNodeId(node['_id']) 65 | 66 | for wayId in wayIdsFromNode: 67 | wayIds.add(wayId) 68 | 69 | ways = [] 70 | 71 | for wayId in wayIds: 72 | way = self.client.osm.ways.find_one({'_id' : wayId}) 73 | if way: 74 | ways.append(way) 75 | else: 76 | print "Error. Couldn't find way id %d." % wayId 77 | 78 | return ways 79 | 80 | def getWayIdsUsingNodeId(self, id): 81 | cursor = self.client.osm.nodes.find_one({'_id' : id }, ['ways']) 82 | if cursor and 'ways' in cursor: 83 | return cursor['ways'] 84 | else: 85 | return [] 86 | 87 | def getRelationsFromWays(self, ways): 88 | relationIds = set() 89 | 90 | for (wid, way) in ways.items(): 91 | id = way['_id'] 92 | relationIdsFromWay = self.getRelationIdsUsingWayId(id) 93 | 94 | for relationId in relationIdsFromWay: 95 | relationIds.add(relationId) 96 | 97 | relations = [] 98 | 99 | for relationId in relationIds: 100 | relation = self.client.osm.relations.find_one({'_id' : relationId}) 101 | if relation: 102 | relations.append(relation) 103 | else: 104 | print "Error. Couldn't find relation id %d." % relationId 105 | 106 | return relations 107 | 108 | def getRelationIdsUsingWayId(self, id): 109 | cursor = self.client.osm.ways.find_one({'_id' : id }, ['relations']) 110 | if cursor and 'relations' in cursor: 111 | return cursor['relations'] 112 | else: 113 | return [] 114 | 115 | def getNodeById(self, id): 116 | cursor = self.client.osm.nodes.find_one({'_id' : id }) 117 | if cursor: 118 | return {'nodes': [cursor]} 119 | else: 120 | return {} 121 | 122 | def getWayById(self, id): 123 | print id 124 | cursor = self.client.osm.ways.find_one({'_id' : id }) 125 | if cursor: 126 | return {'ways': [cursor]} 127 | else: 128 | return {} 129 | 130 | def getRelationById(self, id): 131 | cursor = self.client.osm.relations.ways.find_one({'_id' : id }) 132 | if cursor: 133 | return {'relations': [cursor]} 134 | else: 135 | return {} 136 | 137 | def getPrimitives(self, xapi_query): 138 | nodes = self.getNodesQuery(xapi_query) 139 | 140 | ways = self.getWaysQuery(xapi_query) 141 | 142 | nodes = self.getNodesFromWays(ways, nodes) 143 | 144 | return {'nodes': nodes.values(), 'ways': ways.values()} 145 | 146 | def getBbox(self, bbox): 147 | import time, sys 148 | 149 | start = time.time() 150 | 151 | nodes = self.getNodesQuery(bbox) 152 | 153 | timeA = time.time() 154 | sys.stderr.write("\n" % (timeA - start)) 155 | 156 | ways = self.getWaysQuery(bbox) 157 | 158 | timeB = time.time() 159 | sys.stderr.write("\n" % (timeB - timeA)) 160 | 161 | wayNodes = self.getNodesFromWays(ways, nodes) 162 | 163 | timeC = time.time() 164 | sys.stderr.write("\n" % (timeC - timeB)) 165 | 166 | 167 | timeD = time.time() 168 | sys.stderr.write("\n" % (timeD - timeC)) 169 | 170 | relations = self.getRelationsFromWays(ways) 171 | 172 | timeE = time.time() 173 | sys.stderr.write("\n" % (timeE - timeD)) 174 | 175 | bboxArr = bbox['loc']['$within']['$polygon'] 176 | doc = {'bounds': {'minlat': bboxArr[0][0], 177 | 'minlon': bboxArr[0][1], 178 | 'maxlat': bboxArr[2][0], 179 | 'maxlon': bboxArr[2][1]}, 180 | 'nodes': nodes.values(), 181 | 'ways': ways.values(), 182 | 'relations': relations} 183 | 184 | return doc 185 | 186 | class OsmXmlOutput: 187 | def addNotNullAttr(self, mappable, mappableElement, name, outName=None): 188 | if not outName: 189 | outName = name 190 | if name in mappable: 191 | mappableElement.setAttribute(escape(outName), escape(unicode(mappable[name]))) 192 | 193 | def defaultAttrs(self, mappableElement, mappable): 194 | self.addNotNullAttr(mappable, mappableElement, "_id", "id") 195 | self.addNotNullAttr(mappable, mappableElement, "v", "version") 196 | self.addNotNullAttr(mappable, mappableElement, "un", "user") 197 | self.addNotNullAttr(mappable, mappableElement, "ts", "timestamp") 198 | 199 | def tagNodes(self, doc, mappableElement, mappable): 200 | if 'tg' in mappable: 201 | for mappable in mappable['tg']: 202 | tagElement = doc.createElement("tag") 203 | k,v = mappable 204 | tagElement.setAttribute("k", k) 205 | tagElement.setAttribute("v", v) 206 | mappableElement.appendChild(tagElement) 207 | 208 | def iter(self, data): 209 | from xml.dom.minidom import Document 210 | doc = Document() 211 | 212 | yield '\n' % ("mongosm 0.1", "0.6") 213 | 214 | if 'bounds' in data: 215 | yield '\n' % ( 216 | str(data['bounds']['minlat']), 217 | str(data['bounds']['minlon']), 218 | str(data['bounds']['maxlat']), 219 | str(data['bounds']['maxlon'])) 220 | 221 | if 'nodes' in data: 222 | for node in data['nodes']: 223 | nodeElem = doc.createElement("node") 224 | nodeElem.setAttribute("lat", str(node['loc'][0])) 225 | nodeElem.setAttribute("lon", str(node['loc'][1])) 226 | self.defaultAttrs(nodeElem, node) 227 | self.tagNodes(doc, nodeElem, node) 228 | yield "%s\n" % (nodeElem.toxml('UTF-8'),) 229 | 230 | if 'ways' in data: 231 | for way in data['ways']: 232 | wayElem = doc.createElement("way") 233 | self.defaultAttrs(wayElem, way) 234 | self.tagNodes(doc, wayElem, way) 235 | for ref in way['nd']: 236 | refElement = doc.createElement("nd") 237 | refElement.setAttribute("ref", str(ref)) 238 | wayElem.appendChild(refElement) 239 | yield "%s\n" % (wayElem.toxml('UTF-8'),) 240 | 241 | if 'relations' in data: 242 | for relation in data['relations']: 243 | relationElem = doc.createElement("relation") 244 | self.defaultAttrs(relationElem, relation) 245 | self.tagNodes(doc, relationElem, relation) 246 | for member in relation['mm']: 247 | memberElem = doc.createElement("member") 248 | memberElem.setAttribute("type", member['type']) 249 | memberElem.setAttribute("ref", str(member['ref'])) 250 | memberElem.setAttribute("role", member['role']) 251 | relationElem.appendChild(memberElem) 252 | yield "%s\n" % (relationElem.toxml('UTF-8'),) 253 | 254 | yield '\n' 255 | 256 | import time, sys 257 | import os 258 | import urlparse 259 | from werkzeug.wrappers import Request, Response 260 | from werkzeug.routing import Map, Rule 261 | from werkzeug.exceptions import HTTPException, NotFound 262 | 263 | class Mongosm(object): 264 | def decodePolyline(self, encoded): 265 | i = 0 266 | lat = 0.0 267 | lon = 0.0 268 | points = [] 269 | 270 | while i < len(encoded): 271 | b = 0 272 | shift = 0 273 | result = 0 274 | while True: 275 | b = ord(encoded[i]) - 63 276 | i = i + 1 277 | result |= (b & 0x1f) << shift 278 | shift = shift + 5 279 | if b < 0x20: 280 | break 281 | if (result & 1) > 0: 282 | dlat = ~(result >> 1) 283 | else: 284 | dlat = (result >> 1) 285 | lat = lat + dlat 286 | 287 | shift = 0 288 | result = 0 289 | while True: 290 | b = ord(encoded[i]) - 63 291 | i = i + 1 292 | result |= (b & 0x1f) << shift 293 | shift = shift + 5 294 | if b < 0x20: 295 | break 296 | if (result & 1) > 0: 297 | dlng = ~(result >> 1) 298 | else: 299 | dlng = (result >> 1) 300 | lon = lon + dlng 301 | 302 | points.append([lat * 1e-5, lon * 1e-5]) 303 | return points 304 | 305 | def buildMongoQuery(self, xapiQuery): 306 | q = {} 307 | groups = re.findall(r'(?:\[(.*?)\])', xapiQuery) 308 | for g in groups: 309 | (left, right) = g.split('=') 310 | if left == '@user': 311 | q['user'] = right 312 | elif left == '@uid': 313 | q['uid'] = long(right) 314 | elif left is '@changeset': 315 | q['changeset'] = long(right) 316 | elif left == 'bbox': 317 | (minlon, minlat, maxlon, maxlat) = right.split(',') 318 | bboxPolygon = [ [float(minlat),float(minlon)], 319 | [float(minlat),float(maxlon)], 320 | [float(maxlat),float(maxlon)], 321 | [float(maxlat),float(minlon)] ] 322 | q['loc'] = { '$within': { '$polygon': bboxPolygon } } 323 | elif left == 'poly': 324 | decodedPolygon = self.decodePolyline(right) 325 | q['loc'] = { '$within': { '$polygon': decodedPolygon } } 326 | elif right == u'*': 327 | q['tags.%s' % (left,)] = {'$exists': True} 328 | else: 329 | q['tags.%s' % (left,)] = right 330 | 331 | print "Built query: %s" % (q,) 332 | 333 | return q 334 | 335 | def mapRequest(self, request): 336 | #(minlon, minlat, maxlon, maxlat) = request.args['bbox'].split(',') 337 | #bbox = [[float(minlat), float(minlon)],[float(maxlat), float(maxlon)]] 338 | query = self.buildMongoQuery('[bbox=%s]' % (request.args['bbox'],)) 339 | 340 | api = OsmApi() 341 | data = api.getBbox(query) 342 | 343 | outputter = OsmXmlOutput() 344 | 345 | return Response(outputter.iter(data), content_type='text/xml', direct_passthrough=True) 346 | 347 | def changesetsRequest(self, request): 348 | return Response("%s" % (xapi_query,)) 349 | 350 | def getNode(self, request, id): 351 | api = OsmApi() 352 | data = api.getNodeById(long(id)) 353 | 354 | outputter = OsmXmlOutput() 355 | return Response(outputter.iter(data), content_type='text/xml') 356 | 357 | def getNodeQuery(self, request, xapi_query): 358 | query = self.buildMongoQuery(xapi_query) 359 | 360 | api = OsmApi() 361 | data = api.getNodes(query) 362 | 363 | outputter = OsmXmlOutput() 364 | return Response(outputter.iter(data), content_type='text/xml') 365 | 366 | def getWay(self, request, id): 367 | api = OsmApi() 368 | data = api.getWayById(long(id)) 369 | 370 | outputter = OsmXmlOutput() 371 | return Response(outputter.iter(data), content_type='text/xml') 372 | 373 | def getWayQuery(self, request, xapi_query): 374 | query = self.buildMongoQuery(xapi_query) 375 | 376 | api = OsmApi() 377 | data = api.getWays(query) 378 | 379 | outputter = OsmXmlOutput() 380 | return Response(outputter.iter(data), content_type='text/xml') 381 | 382 | def getRelation(self, request, id): 383 | api = OsmApi() 384 | data = api.getRelationById(long(id)) 385 | 386 | outputter = OsmXmlOutput() 387 | return Response(outputter.iter(data), content_type='text/xml') 388 | 389 | def getRelationQuery(self, request, xapi_query): 390 | query = self.buildMongoQuery(xapi_query) 391 | 392 | api = OsmApi() 393 | data = api.getRelations(query) 394 | 395 | outputter = OsmXmlOutput() 396 | return Response(outputter.iter(data), content_type='text/xml') 397 | 398 | def getPrimitiveQuery(self, request, xapi_query): 399 | query = self.buildMongoQuery(xapi_query) 400 | 401 | api = OsmApi() 402 | data = api.getPrimitives(query) 403 | 404 | outputter = OsmXmlOutput() 405 | return Response(outputter.iter(data), content_type='text/xml') 406 | 407 | def capabilitiesRequest(self, request): 408 | return Response(""" 409 | 410 | 411 | 412 | 413 | 414 | """) 415 | 416 | def __init__(self): 417 | self.url_map = Map([ 418 | Rule('/api/0.6/map', endpoint='mapRequest'), 419 | 420 | Rule('/api/0.6/changesets', endpoint='changesetsRequest'), 421 | 422 | Rule('/api/0.6/node/', endpoint='getNode'), 423 | Rule('/api/0.6/way/', endpoint='getWay'), 424 | Rule('/api/0.6/relation/', endpoint='getRelation'), 425 | 426 | Rule('/api/0.6/node', endpoint='getNodeQuery'), 427 | Rule('/api/0.6/way', endpoint='getWayQuery'), 428 | Rule('/api/0.6/relation', endpoint='getRelationQuery'), 429 | 430 | Rule('/api/0.6/*', endpoint='getPrimitiveQuery'), 431 | 432 | Rule('/api/capabilities', endpoint='capabilitiesRequest'), 433 | ]) 434 | 435 | def dispatch_request(self, request): 436 | adapter = self.url_map.bind_to_environ(request.environ) 437 | try: 438 | endpoint, values = adapter.match() 439 | return getattr(self, endpoint)(request, **values) 440 | except HTTPException, e: 441 | return e 442 | 443 | def wsgi_app(self, environ, start_response): 444 | request = Request(environ) 445 | response = self.dispatch_request(request) 446 | return response(environ, start_response) 447 | 448 | def __call__(self, environ, start_response): 449 | return self.wsgi_app(environ, start_response) 450 | 451 | if __name__ == '__main__': 452 | from werkzeug.serving import run_simple 453 | app = Mongosm() 454 | run_simple('0.0.0.0', 5000, app, use_debugger=True, use_reloader=True) 455 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | MongOSM 2 | ======= 3 | 4 | MongOSM is a set of Python utilities that manipulate OSM data in MongoDB: 5 | 6 | - `insert_osm_data.py` reads an OSM file and writes it to a MongoDB database. 7 | - `map_server.py` uses Werkzeug to start a WSGI server that responds to the 8 | read-only OSM APIs and most of the XAPI-style predicate queries. 9 | - `apply-osmchange.py` is currently not tested, but it is supposed to read 10 | minutely change files from planet.osm.org and keep the MongoDB database 11 | up to date. 12 | 13 | Installation 14 | ------------ 15 | 16 | 1. Grab MongoDB 1.9+ (2.0 is best) from http://mongodb.org/. 17 | 2. Unpack mongodb and run it. 18 | 3. Install pymongo. (http://api.mongodb.org/python/current/installation.html) 19 | 4. Grab some OSM XML data. 20 | 5. Run `python insert_osm_data.py ` and wait. 21 | 6. Install Werkzeug. 22 | 7. Run `python map_server.py` 23 | 8. Browse to http://localhost:5000/api/0.6/node/1 to verify a (probably empty) 24 | response. 25 | -------------------------------------------------------------------------------- /tile_server.py: -------------------------------------------------------------------------------- 1 | import pymongo 2 | from pymongo import Connection 3 | from xml.sax.saxutils import escape 4 | import re 5 | 6 | from globalmaptiles import GlobalMercator 7 | 8 | class OsmApi: 9 | def __init__(self): 10 | self.client = Connection() 11 | self.proj = GlobalMercator() 12 | 13 | def getTile(self, zoom, x, y): 14 | (x, y) = self.proj.GoogleTile(x,y,zoom) 15 | quadkey = self.proj.QuadTree(x,y,zoom) 16 | print "Querying for %s." % (quadkey,) 17 | (minlat, minlon, maxlat, maxlon) = self.proj.TileLatLonBounds(x,y,zoom) 18 | 19 | # Nodes in the tile 20 | nodes = {} 21 | cursor = self.client.osm.nodes.find({'qk': {'$regex': "^%s" % (quadkey,)} }) 22 | for row in cursor: 23 | nodes[row['_id']] = row 24 | 25 | # Ways with nodes in the tile 26 | ways = {} 27 | cursor = self.client.osm.ways.find({'loc': {'$regex': "^%s" % (quadkey,)} }) 28 | for row in cursor: 29 | ways[row['_id']] = row 30 | 31 | # Nodes on ways that extend beyond the bounding box 32 | otherNids = set() 33 | for way in ways.values(): 34 | for nid in way['nodes']: 35 | otherNids.add(nid) 36 | cursor = self.client.osm.nodes.find({'_id': {'$in': list(otherNids)} }) 37 | for row in cursor: 38 | nodes[row['_id']] = row 39 | 40 | # Relations that contain any of the above as members 41 | relations = {} 42 | 43 | # Sort the results by id 44 | nodes = sorted(nodes.iteritems()) 45 | ways = sorted(ways.iteritems()) 46 | relations = sorted(relations.iteritems()) 47 | 48 | doc = {'bounds': {'minlat': minlat, 49 | 'minlon': minlon, 50 | 'maxlat': maxlat, 51 | 'maxlon': maxlon}, 52 | 'nodes': nodes, 53 | 'ways': ways, 54 | 'relations': relations} 55 | 56 | return doc 57 | 58 | class OsmXmlOutput: 59 | def addNotNullAttr(self, mappable, mappableElement, name, outName=None): 60 | if not outName: 61 | outName = name 62 | if name in mappable: 63 | mappableElement.setAttribute(escape(outName), escape(unicode(mappable[name]))) 64 | 65 | def defaultAttrs(self, mappableElement, mappable): 66 | self.addNotNullAttr(mappable, mappableElement, "_id", "id") 67 | self.addNotNullAttr(mappable, mappableElement, "v", "version") 68 | self.addNotNullAttr(mappable, mappableElement, "user") 69 | 70 | def tagNodes(self, doc, mappableElement, mappable): 71 | for mappable in mappable['tags']: 72 | k,v = mappable 73 | tagElement = doc.createElement("tag") 74 | tagElement.setAttribute("k", k) 75 | tagElement.setAttribute("v", v) 76 | mappableElement.appendChild(tagElement) 77 | 78 | def iter(self, data): 79 | from xml.dom.minidom import Document 80 | doc = Document() 81 | 82 | yield '\n' % ("tiled mongosm 0.1", "0.6") 83 | 84 | if 'bounds' in data: 85 | yield '\n' % ( 86 | str(data['bounds']['minlat']), 87 | str(data['bounds']['minlon']), 88 | str(data['bounds']['maxlat']), 89 | str(data['bounds']['maxlon'])) 90 | 91 | if 'nodes' in data: 92 | for (id, node) in data['nodes']: 93 | nodeElem = doc.createElement("node") 94 | nodeElem.setAttribute("lat", str(node['loc']['lat'])) 95 | nodeElem.setAttribute("lon", str(node['loc']['lon'])) 96 | self.defaultAttrs(nodeElem, node) 97 | self.tagNodes(doc, nodeElem, node) 98 | yield "%s\n" % (nodeElem.toxml('UTF-8'),) 99 | 100 | if 'ways' in data: 101 | for (id, way) in data['ways']: 102 | wayElem = doc.createElement("way") 103 | self.defaultAttrs(wayElem, way) 104 | self.tagNodes(doc, wayElem, way) 105 | for ref in way['nodes']: 106 | refElement = doc.createElement("nd") 107 | refElement.setAttribute("ref", str(ref)) 108 | wayElem.appendChild(refElement) 109 | yield "%s\n" % (wayElem.toxml('UTF-8'),) 110 | 111 | if 'relations' in data: 112 | for (id, relation) in data['relations']: 113 | relationElem = doc.createElement("relation") 114 | self.defaultAttrs(relationElem, relation) 115 | self.tagNodes(doc, relationElem, relation) 116 | for member in relation['members']: 117 | memberElem = doc.createElement("member") 118 | memberElem.setAttribute("type", member['type']) 119 | memberElem.setAttribute("ref", str(member['ref'])) 120 | memberElem.setAttribute("role", member['role']) 121 | relationElem.appendChild(memberElem) 122 | yield "%s\n" % (relationElem.toxml('UTF-8'),) 123 | 124 | yield '\n' 125 | 126 | import time, sys 127 | import os 128 | import urlparse 129 | from werkzeug.wrappers import Request, Response 130 | from werkzeug.routing import Map, Rule 131 | from werkzeug.exceptions import HTTPException, NotFound 132 | 133 | class Mongosm(object): 134 | 135 | def tileRequest(self, request, zoom, x, y): 136 | #(minlon, minlat, maxlon, maxlat) = request.args['bbox'].split(',') 137 | #bbox = [[float(minlat), float(minlon)],[float(maxlat), float(maxlon)]] 138 | 139 | api = OsmApi() 140 | data = api.getTile(int(zoom), int(x), int(y)) 141 | 142 | outputter = OsmXmlOutput() 143 | 144 | return Response(outputter.iter(data), content_type='text/xml', direct_passthrough=True) 145 | 146 | def capabilitiesRequest(self, request): 147 | return Response(""" 148 | 149 | 150 | 151 | 152 | 153 | """) 154 | 155 | def __init__(self): 156 | self.url_map = Map([ 157 | Rule('/tiles/0.6///', endpoint='tileRequest'), 158 | Rule('/api/capabilities', endpoint='capabilitiesRequest'), 159 | ]) 160 | 161 | def dispatch_request(self, request): 162 | adapter = self.url_map.bind_to_environ(request.environ) 163 | try: 164 | endpoint, values = adapter.match() 165 | return getattr(self, endpoint)(request, **values) 166 | except HTTPException, e: 167 | return e 168 | 169 | def wsgi_app(self, environ, start_response): 170 | request = Request(environ) 171 | response = self.dispatch_request(request) 172 | return response(environ, start_response) 173 | 174 | def __call__(self, environ, start_response): 175 | return self.wsgi_app(environ, start_response) 176 | 177 | if __name__ == '__main__': 178 | from werkzeug.serving import run_simple 179 | app = Mongosm() 180 | run_simple('0.0.0.0', 5000, app, use_debugger=True, use_reloader=True) 181 | --------------------------------------------------------------------------------