├── .gitignore ├── Dockerfile ├── LICENSE.md ├── README.md ├── bin ├── docker_entrypoint.sh └── osm-export-tool ├── examples └── python_example.py ├── osm_export_tool ├── __init__.py ├── cmd.py ├── geometry.py ├── mapping.py ├── mappings │ ├── HDX.yml │ ├── HDX_v2.yml │ ├── InAWARE.yml │ ├── default.yml │ └── simple.yml ├── nontabular.py ├── package.py ├── sources.py ├── sql.py └── tabular.py ├── requirements.txt ├── setup.py └── test ├── __init__.py ├── test_mapping.py ├── test_sources.py └── test_sql.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.gpkg 2 | *.shp 3 | *.shx 4 | *.dbf 5 | *.pbf 6 | *.pyc 7 | *.egg-info 8 | venv 9 | build 10 | dist 11 | files 12 | examples/tmp 13 | examples/tools 14 | .env 15 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:buster-slim 2 | 3 | RUN apt-get update && apt-get install -yq \ 4 | python3-pip \ 5 | python3-gdal 6 | 7 | COPY . /source/osm-export-tool-python 8 | 9 | RUN pip3 install /source/osm-export-tool-python 10 | 11 | COPY bin/docker_entrypoint.sh /bin/docker_entrypoint.sh 12 | 13 | ENTRYPOINT [ "/bin/docker_entrypoint.sh" ] 14 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019, Humanitarian OpenStreetMap Team. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | Neither the name of copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OSM Export Tool 2 | 3 | This project is in a usable state on Linux and Mac. The current [Export Tool web service](https://export.hotosm.org) repository is at [hotosm/osm-export-tool](https://github.com/hotosm/osm-export-tool/tree/master/ops). 4 | 5 | ## Motivation 6 | 7 | This program filters and transforms OpenStreetMap data into thematic, tabular GIS formats. 8 | Filtering of features is specified via SQL embedded in a YAML mapping file, for example: 9 | ``` 10 | buildings_with_addresses: # creates a thematic layer called "buildings_with_addresses" 11 | types: 12 | - polygons 13 | select: 14 | - building 15 | - height 16 | - addr:housenumber 17 | where: 18 | - building = 'yes' and addr:housenumber IS NOT NULL 19 | ``` 20 | 21 | It can also create files in non-tabular formats such as those for Garmin GPS devices or the OSMAnd Android app. (coming soon) 22 | 23 | ## Installation 24 | 25 | * install via `pip install osm-export-tool`. Python 3 and a working GDAL installation are required. GDAL can be installed via Homebrew on Mac or the `ubuntugis` PPAs on Ubuntu. 26 | 27 | PyOsmium is used to read OSM files and GDAL/OGR is used to write GIS files, so this program should be reasonably fast and light on memory. There is a built-in OSM reader available for GDAL/OGR, but this program is much more flexible. 28 | 29 | This library will not automatically install GDAL because it needs to match the version on your system. You will need to separately run `pip install GDAL==2.3.2` (change 2.3.2 to match `gdalinfo --version`) 30 | 31 | ## Running with Docker 32 | 33 | If you want to avoid installing the right version of GDAL on your system you can run the program as a docker container instead. 34 | 35 | To build the docker image, use the following command. 36 | 37 | ``` 38 | docker build -t osm-export-tool . 39 | ``` 40 | 41 | To run the tool as a container, using your current directory as working directory, use the following command. 42 | 43 | ``` 44 | docker run -it --rm -v $(pwd):/work osm-export-tool INPUT_FILE.pbf OUTPUT_NAME 45 | ``` 46 | 47 | ## Example usage 48 | 49 | ``` 50 | osm-export-tool INPUT_FILE.pbf OUTPUT_NAME 51 | ``` 52 | will create OUTPUT_NAME.gpkg. 53 | 54 | All the below flags are optional. 55 | 56 | * -m, --mapping : specify a mapping YAML. Defaults to `osm_export_tool/mappings/defaults.yaml`, which is a very broad selection of OSM tags ported from the [imposm3 defaults](https://github.com/omniscale/imposm3/blob/master/example-mapping.yml). 57 | * `-f, --formats` : a comma-separated list of formats such as `gpkg,shp`. Defaults to just gpkg. 58 | * `--omit-osm-ids`: By default, every table will have an `osm_id` column. Relation IDs are negative. 59 | * `--clip `: either a .poly or GeoJSON file. 60 | * The GeoJSON must be either a Polygon or MultiPolygon geometry, or a FeatureCollection with one Polygon or MultiPolygon feature. 61 | * Clipping is performed by Shapely and can be slow. It is recommended to filter the input PBF with a tool like [osmium-tool](https://github.com/osmcode/osmium-tool). 62 | 63 | ## YAML Mapping 64 | 65 | * SQL statements must be comparisons of keys to constants with the key first. 66 | * Valid examples: 67 | * `height > 20` 68 | * `amenity='parking' OR (building = 'yes' and height > 5)` 69 | * Invalid examples: 70 | * `20 < height` 71 | * `building > height` 72 | * More examples can be found in the [mappings directory](osm_export_tool/mappings). 73 | * if the `types` key is omitted, it defaults to `points`, `lines` and `polygons`. 74 | * At least one tag is required as a child of the `select` key. 75 | * If the `where` key is omitted, it defaults to choosing all features where any of the `select`ed keys are present. 76 | * if `where` is a list of SQL, it is equivalent to joining each SQL in the list with `OR`. 77 | 78 | ## Output formats 79 | 80 | 1. OGC GeoPackage (gpkg) 81 | * This is the default export format, and the most flexible for modern GIS applications. 82 | * tables will be created with the wkbUnknown geometry type, which allows heterogeneous geometry types. 83 | 84 | 2. Shapefile (shp) 85 | * Each layer and geometry type is a separate .SHP file. This is because each .SHP file only supports a single geometry type and column schema. 86 | 87 | 3. KML (kml) 88 | * Each layer and geometry type is a separate .KML file. This is because the GDAL/OGR KML driver does not support interleaved writing of features with different geometry types. 89 | 90 | 4. Maps.ME (coming soon) 91 | 92 | 5. OsmAnd (coming soon) 93 | 94 | 6. Garmin (coming soon) 95 | -------------------------------------------------------------------------------- /bin/docker_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | mkdir -p /work 6 | cd /work 7 | osm-export-tool "$@" 8 | -------------------------------------------------------------------------------- /bin/osm-export-tool: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from osm_export_tool.cmd import main 4 | 5 | if __name__ == '__main__': 6 | main() -------------------------------------------------------------------------------- /examples/python_example.py: -------------------------------------------------------------------------------- 1 | import osm_export_tool 2 | import osm_export_tool.tabular as tabular 3 | import osm_export_tool.nontabular as nontabular 4 | from osm_export_tool.mapping import Mapping 5 | from osm_export_tool.geometry import load_geometry 6 | from osm_export_tool.sources import Overpass, Pbf, OsmExpress, OsmiumTool 7 | from osm_export_tool.package import create_package, create_posm_bundle 8 | from os.path import join 9 | 10 | GEOJSON = """{ 11 | "type": "MultiPolygon", 12 | "coordinates": [[ 13 | [ 14 | [-155.077815, 19.722514], 15 | [-155.087643, 19.722514], 16 | [-155.087643, 19.715929], 17 | [-155.077815, 19.715929], 18 | [-155.077815, 19.722514] 19 | ] 20 | ]] 21 | }""" 22 | 23 | geom = load_geometry(GEOJSON) 24 | tempdir = 'tmp' 25 | 26 | with open('../osm_export_tool/mappings/default.yml','r') as f: 27 | mapping_txt = f.read() 28 | mapping = Mapping(mapping_txt) 29 | 30 | source = Overpass('http://overpass.hotosm.org',geom,join(tempdir,'extract.osm.pbf'),tempdir=tempdir,mapping=mapping,use_existing=False) 31 | 32 | shp = tabular.Shapefile("tmp/example",mapping) 33 | gpkg = tabular.Geopackage("tmp/example",mapping) 34 | kml = tabular.Kml("tmp/example",mapping) 35 | tabular_outputs = [shp,gpkg,kml] 36 | 37 | h = tabular.Handler(tabular_outputs,mapping) 38 | 39 | h.apply_file(source.path(), locations=True, idx='sparse_file_array') 40 | 41 | for output in tabular_outputs: 42 | output.finalize() 43 | 44 | osmand_files = nontabular.osmand(source.path(),'tools/OsmAndMapCreator-main',tempdir=tempdir) 45 | garmin_files = nontabular.garmin(source.path(),'tools/splitter-r583/splitter.jar','tools/mkgmap-r3890/mkgmap.jar',tempdir=tempdir) 46 | #mwm_files = nontabular.mwm(source.path(),join(tempdir,'mwm'),'generate_mwm.sh','/usr/local/bin/generator_tool','/usr/bin/osmconvert') 47 | #mbtiles_files = nontabular.mbtiles(geom,join(tempdir,'output.mbtiles'),'http://tile.openstreetmap.org/{z}/{x}/{y}.png',14,14) 48 | 49 | files = [] 50 | files += shp.files 51 | files += gpkg.files 52 | files += kml.files 53 | files += osmand_files 54 | files += garmin_files 55 | #files += mbtiles_files 56 | files.append(osm_export_tool.File('osm_pbf',[source.path()],'')) 57 | create_package(join(tempdir,'shp.zip'),shp.files,boundary_geom=geom) 58 | create_posm_bundle(join(tempdir,'bundle.tgz'),files,"Title","name","description",geom) -------------------------------------------------------------------------------- /osm_export_tool/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | from enum import Enum 3 | # force loading of shapely before ogr, see https://github.com/Toblerity/Shapely/issues/490 4 | from shapely.geometry import shape, MultiPolygon, Polygon 5 | 6 | name = 'osm_export_tool' 7 | 8 | class GeomType(Enum): 9 | POINT = 1 10 | LINE = 2 11 | POLYGON = 3 12 | 13 | def GetHumanReadable(size,precision=2): 14 | suffixes=['B','KB','MB','GB','TB'] 15 | suffixIndex = 0 16 | while size > 1024 and suffixIndex < 4: 17 | suffixIndex += 1 #increment the index of the suffix 18 | size = size/1024.0 #apply the division 19 | return "%.*f%s"%(precision,size,suffixes[suffixIndex]) 20 | 21 | # can be more than one file (example: Shapefile w/ sidecars) 22 | class File: 23 | def __init__(self,output_name,parts,extra = {}): 24 | self.output_name = output_name 25 | self.parts = parts 26 | self.extra = extra 27 | 28 | @classmethod 29 | def shp(cls,name,extra = {}): 30 | parts = [name + '.shp'] 31 | parts.append(name + '.shx') 32 | parts.append(name + '.prj') 33 | parts.append(name + '.cpg') 34 | parts.append(name + '.dbf') 35 | return cls('shp',parts,extra) 36 | 37 | def size(self): 38 | total = 0 39 | for part in self.parts: 40 | total = total + os.path.getsize(part) 41 | return total 42 | 43 | def __str__(self): 44 | return '{0} {1} {2} {3}'.format(self.output_name,self.extra,','.join(self.parts),GetHumanReadable(self.size())) 45 | 46 | def __repr__(self): 47 | return self.__str__() -------------------------------------------------------------------------------- /osm_export_tool/cmd.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | import argparse 5 | import osm_export_tool.tabular as tabular 6 | import osm_export_tool.nontabular as nontabular 7 | from osm_export_tool.mapping import Mapping 8 | from osm_export_tool.geometry import load_geometry 9 | 10 | def main(): 11 | parser = argparse.ArgumentParser(description='Export OSM data in other file formats.') 12 | parser.add_argument('osm_file', help='OSM .pbf or .xml input file') 13 | parser.add_argument('output_name', help='Output prefix') 14 | parser.add_argument('-f','--formats', dest='formats',default='gpkg',help='List of formats e.g. gpkg,shp,kml (default: gpkg)') 15 | parser.add_argument('-m','--mapping', dest='mapping',help='YAML mapping of features schema. If not specified, a default is used.') 16 | parser.add_argument('--clip', dest='clip',help='GeoJSON or POLY file to clip geometries.') 17 | parser.add_argument('-v','--verbose', action='store_true') 18 | parser.add_argument('--omit-osm-ids', action='store_true') 19 | parsed = parser.parse_args() 20 | 21 | mapping_txt = None 22 | if parsed.mapping: 23 | with open(parsed.mapping,'r') as f: 24 | mapping_txt = f.read() 25 | else: 26 | default_mapping = os.path.join(os.path.dirname(__file__), 'mappings/default.yml') 27 | with open(default_mapping,'r') as f: 28 | mapping_txt = f.read() 29 | 30 | mapping = Mapping(mapping_txt,default_osm_id=not parsed.omit_osm_ids) 31 | 32 | clipping_geom = None 33 | if parsed.clip: 34 | with open(parsed.clip,'r') as f: 35 | clipping_geom = load_geometry(f.read()) 36 | 37 | formats = parsed.formats.split(',') 38 | 39 | tabular_outputs = [] 40 | if 'gpkg' in formats: 41 | tabular_outputs.append(tabular.Geopackage(parsed.output_name,mapping)) 42 | if 'shp' in formats: 43 | tabular_outputs.append(tabular.Shapefile(parsed.output_name,mapping)) 44 | if 'kml' in formats: 45 | tabular_outputs.append(tabular.Kml(parsed.output_name,mapping)) 46 | 47 | nontabular_outputs = [] 48 | if 'mwm' in formats: 49 | nontabular_outputs.append(nontabular.Omim()) 50 | if 'img' in formats: 51 | nontabular_outputs.append(nontabular.GarminIMG()) 52 | if 'obf' in formats: 53 | nontabular_outputs.append(nontabular.Osmand()) 54 | 55 | if len(tabular_outputs) > 0: 56 | h = tabular.Handler(tabular_outputs,mapping,clipping_geom=clipping_geom) 57 | start_time = time.time() 58 | h.apply_file(parsed.osm_file, locations=True, idx='sparse_file_array') 59 | 60 | for output in tabular_outputs: 61 | output.finalize() 62 | print('Completed in {0} seconds.'.format(time.time() - start_time)) 63 | 64 | for output in tabular_outputs: 65 | for file in output.files: 66 | print(file) 67 | 68 | if len(nontabular_outputs) > 0: 69 | for output in nontabular_outputs: 70 | output.run(pbf,tmpdir) -------------------------------------------------------------------------------- /osm_export_tool/geometry.py: -------------------------------------------------------------------------------- 1 | import json 2 | from shapely.geometry import shape, MultiPolygon, Polygon 3 | 4 | 5 | def parse_poly(lines): 6 | """Parse an Osmosis polygon filter file. 7 | Accept a sequence of lines from a polygon file, return a shapely.geometry.MultiPolygon object. 8 | https://wiki.openstreetmap.org/wiki/Osmosis/Polygon_Filter_File_Python_Parsing 9 | """ 10 | in_ring = False 11 | coords = [] 12 | for (index, line) in enumerate(lines): 13 | if index == 0: 14 | # first line is junk. 15 | continue 16 | elif index == 1: 17 | # second line is the first polygon ring. 18 | coords.append([[], []]) 19 | ring = coords[-1][0] 20 | in_ring = True 21 | elif in_ring and line.strip() == "END": 22 | # we are at the end of a ring, perhaps with more to come. 23 | in_ring = False 24 | elif in_ring: 25 | # we are in a ring and picking up new coordinates. 26 | ring.append(list(map(float, line.split()))) 27 | elif not in_ring and line.strip() == "END": 28 | # we are at the end of the whole polygon. 29 | break 30 | elif not in_ring and line.startswith("!"): 31 | # we are at the start of a polygon part hole. 32 | coords[-1][1].append([]) 33 | ring = coords[-1][1][-1] 34 | in_ring = True 35 | elif not in_ring: 36 | # we are at the start of a polygon part. 37 | coords.append([[], []]) 38 | ring = coords[-1][0] 39 | in_ring = True 40 | 41 | return MultiPolygon(coords) 42 | 43 | 44 | def load_geometry(txt): 45 | try: 46 | j = json.loads(txt) 47 | if j["type"] == "FeatureCollection": 48 | print("Warning: using first feature of --clip FeatureCollection.") 49 | return shape(j["features"][0]["geometry"]) 50 | else: 51 | return shape(j) 52 | except json.decoder.JSONDecodeError: 53 | pass 54 | return parse_poly(txt.split("\n")) 55 | -------------------------------------------------------------------------------- /osm_export_tool/mapping.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | import pyparsing 3 | from osm_export_tool import GeomType 4 | from osm_export_tool.sql import Matcher 5 | 6 | class InvalidMapping(Exception): 7 | pass 8 | 9 | class Theme: 10 | def __init__(self,name,d,default_osm_id): 11 | self.name = name 12 | 13 | # set geometry types. 14 | self.points = False 15 | self.lines = False 16 | self.polygons = False 17 | 18 | if not isinstance(d,dict): 19 | if isinstance(d,list): 20 | raise InvalidMapping('theme {0} must be YAML dict (types: , select:) , not list (- types, - select)'.format(name)) 21 | raise InvalidMapping('Theme value must be dict') 22 | 23 | if 'types' not in d: 24 | self.points = True 25 | self.lines = True 26 | self.polygons = True 27 | else: 28 | for t in d['types']: 29 | if t not in ['points','lines','polygons']: 30 | raise InvalidMapping('types: for theme {0} must be list containing one or more of: points, lines, polygons'.format(name)) 31 | if 'points' in d['types']: 32 | self.points = True 33 | if 'lines' in d['types']: 34 | self.lines = True 35 | if 'polygons' in d['types']: 36 | self.polygons = True 37 | 38 | 39 | if 'select' not in d: 40 | raise InvalidMapping('missing select: for theme {0}'.format(name)) 41 | self.keys = set(d['select']) 42 | 43 | self.osm_id = default_osm_id 44 | if 'osm_id' in self.keys: 45 | self.osm_id = True 46 | self.keys.remove('osm_id') 47 | 48 | if 'where' in d: 49 | try: 50 | if not d['where']: 51 | raise InvalidMapping('where: for theme {0} is invalid'.format(name)) 52 | if isinstance(d['where'],list): 53 | self.matcher = Matcher.null() 54 | for w in d['where']: 55 | self.matcher = self.matcher.union(Matcher.from_sql(w)) 56 | else: 57 | self.matcher = Matcher.from_sql(d['where']) 58 | except pyparsing.ParseException: 59 | raise InvalidMapping('Invalid SQL: {0}'.format(d['where'])) 60 | else: 61 | self.matcher = Matcher.null() 62 | for key in self.keys: 63 | self.matcher = self.matcher.union(Matcher.any(key)) 64 | 65 | extra = d.copy() 66 | if 'where' in extra: 67 | del extra['where'] 68 | if 'select' in d: 69 | del extra['select'] 70 | if 'types' in d: 71 | del extra['types'] 72 | self.extra = extra 73 | 74 | def matches(self,geom_type,tags): 75 | if geom_type == GeomType.POINT and not self.points: 76 | return False 77 | if geom_type == GeomType.LINE and not self.lines: 78 | return False 79 | if geom_type == GeomType.POLYGON and not self.polygons: 80 | return False 81 | 82 | return self.matcher.matches(tags) 83 | 84 | def __repr__(self): 85 | return self.name 86 | 87 | 88 | class Mapping: 89 | def __init__(self,y,default_osm_id=True): 90 | doc = yaml.safe_load(y) 91 | 92 | if not isinstance(doc,dict): 93 | raise InvalidMapping('YAML must be dict') 94 | 95 | self.themes = [] 96 | for theme_name, theme_dict in doc.items(): 97 | self.themes.append(Theme(theme_name,theme_dict,default_osm_id=default_osm_id)) 98 | 99 | @classmethod 100 | def validate(cls,y,**kwargs): 101 | try: 102 | return cls(y,kwargs), None 103 | except (yaml.scanner.ScannerError, yaml.parser.ParserError, InvalidMapping) as se: 104 | errors = [str(se)] 105 | return None, errors 106 | 107 | -------------------------------------------------------------------------------- /osm_export_tool/mappings/HDX.yml: -------------------------------------------------------------------------------- 1 | Buildings: 2 | hdx: 3 | tags: building, shelter, osm, openstreetmap 4 | types: 5 | - polygons 6 | select: 7 | - name 8 | - building 9 | - building:levels 10 | - building:materials 11 | - addr:full 12 | - addr:housenumber 13 | - addr:street 14 | - addr:city 15 | - office 16 | where: building IS NOT NULL 17 | 18 | Roads: 19 | hdx: 20 | tags: roads, transportation, osm, openstreetmap 21 | types: 22 | - lines 23 | - polygons 24 | select: 25 | - name 26 | - highway 27 | - surface 28 | - smoothness 29 | - width 30 | - lanes 31 | - oneway 32 | - bridge 33 | - layer 34 | where: highway IS NOT NULL 35 | 36 | Waterways: 37 | hdx: 38 | tags: rivers, hydrology, waterbodies, osm, openstreetmap 39 | types: 40 | - lines 41 | - polygons 42 | select: 43 | - name 44 | - waterway 45 | - covered 46 | - width 47 | - depth 48 | - layer 49 | - blockage 50 | - tunnel 51 | - natural 52 | - water 53 | where: waterway IS NOT NULL OR water IS NOT NULL OR natural IN ('water','wetland','bay') 54 | 55 | Points of Interest: 56 | hdx: 57 | tags: poi, points of interest, facilities, osm, openstreetmap 58 | types: 59 | - points 60 | - polygons 61 | select: 62 | - name 63 | - amenity 64 | - man_made 65 | - shop 66 | - tourism 67 | - opening_hours 68 | - beds 69 | - rooms 70 | - addr:full 71 | - addr:housenumber 72 | - addr:street 73 | - addr:city 74 | where: amenity IS NOT NULL OR man_made IS NOT NULL OR shop IS NOT NULL OR tourism IS NOT NULL 75 | -------------------------------------------------------------------------------- /osm_export_tool/mappings/HDX_v2.yml: -------------------------------------------------------------------------------- 1 | Buildings: 2 | hdx: 3 | tags: buildings, geodata 4 | caveats: OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive 5 | types: 6 | - polygons 7 | select: 8 | - name 9 | - building 10 | - building:levels 11 | - building:materials 12 | - addr:full 13 | - addr:housenumber 14 | - addr:street 15 | - addr:city 16 | - office 17 | - source 18 | where: building IS NOT NULL 19 | 20 | Roads: 21 | hdx: 22 | tags: roads, transportation, geodata 23 | caveats: OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive 24 | types: 25 | - lines 26 | - polygons 27 | select: 28 | - name 29 | - highway 30 | - surface 31 | - smoothness 32 | - width 33 | - lanes 34 | - oneway 35 | - bridge 36 | - layer 37 | - source 38 | where: highway IS NOT NULL 39 | 40 | Waterways: 41 | hdx: 42 | tags: rivers, water bodies - hydrography, geodata 43 | caveats: OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive 44 | types: 45 | - lines 46 | - polygons 47 | select: 48 | - name 49 | - waterway 50 | - covered 51 | - width 52 | - depth 53 | - layer 54 | - blockage 55 | - tunnel 56 | - natural 57 | - water 58 | - source 59 | where: waterway IS NOT NULL OR water IS NOT NULL OR natural IN ('water','wetland','bay') 60 | 61 | Points of Interest: 62 | hdx: 63 | tags: facilities and infrastructure, points of interest - poi, geodata 64 | caveats: OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive 65 | types: 66 | - points 67 | - polygons 68 | select: 69 | - name 70 | - amenity 71 | - man_made 72 | - shop 73 | - tourism 74 | - opening_hours 75 | - beds 76 | - rooms 77 | - addr:full 78 | - addr:housenumber 79 | - addr:street 80 | - addr:city 81 | - source 82 | where: amenity IS NOT NULL OR man_made IS NOT NULL OR shop IS NOT NULL OR tourism IS NOT NULL 83 | 84 | Airports: 85 | hdx: 86 | tags: airports, helicopter landing zone - hlz, aviation, facilities and infrastructure, transportation, geodata 87 | caveats: OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive 88 | types: 89 | - points 90 | - lines 91 | - polygons 92 | select: 93 | - name 94 | - aeroway 95 | - building 96 | - emergency 97 | - emergency:helipad 98 | - operator:type 99 | - capacity:persons 100 | - addr:full 101 | - addr:city 102 | - source 103 | where: aeroway IS NOT NULL OR building = 'aerodrome' OR emergency:helipad IS NOT NULL OR emergency = 'landing_site' 104 | 105 | 106 | Sea Ports: 107 | hdx: 108 | tags: ports, logistics, facilities and infrastructure, transportation, geodata 109 | caveats: OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive 110 | types: 111 | - points 112 | - lines 113 | - polygons 114 | select: 115 | - name 116 | - amenity 117 | - building 118 | - port 119 | - operator:type 120 | - addr:full 121 | - addr:city 122 | - source 123 | where: amenity = 'ferry_terminal' OR building = 'ferry_terminal' OR port IS NOT NULL 124 | 125 | Education Facilities: 126 | hdx: 127 | tags: education facilities - schools, education, geodata 128 | caveats: OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive 129 | types: 130 | - points 131 | - polygons 132 | select: 133 | - name 134 | - amenity 135 | - building 136 | - operator:type 137 | - capacity:persons 138 | - addr:full 139 | - addr:city 140 | - source 141 | where: amenity IN ('kindergarten', 'school', 'college', 'university') OR building IN ('kindergarten', 'school', 'college', 'university') 142 | 143 | Health Facilities: 144 | hdx: 145 | tags: health facilities, health, geodata 146 | caveats: OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive 147 | types: 148 | - points 149 | - polygons 150 | select: 151 | - name 152 | - amenity 153 | - building 154 | - healthcare 155 | - healthcare:speciality 156 | - operator:type 157 | - capacity:persons 158 | - addr:full 159 | - addr:city 160 | - source 161 | where: healthcare IS NOT NULL OR amenity IN ('doctors', 'dentist', 'clinic', 'hospital', 'pharmacy') 162 | 163 | Populated Places: 164 | hdx: 165 | tags: populated places - settlements, geodata 166 | caveats: OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive 167 | types: 168 | - points 169 | - polygons 170 | select: 171 | - name 172 | - place 173 | - population 174 | - is_in 175 | - source 176 | where: place IN ('isolated_dwelling', 'town', 'village', 'hamlet', 'city') 177 | 178 | Financial Services: 179 | hdx: 180 | tags: financial institutions, services, geodata 181 | caveats: OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive 182 | types: 183 | - points 184 | - polygons 185 | select: 186 | - name 187 | - amenity 188 | - operator 189 | - network 190 | - addr:full 191 | - addr:city 192 | - source 193 | where: amenity IN ('mobile_money_agent','bureau_de_change','bank','microfinance','atm','sacco','money_transfer','post_office') 194 | 195 | 196 | Railways: 197 | hdx: 198 | tags: facilities and infrastructure,railways,transportation, geodata 199 | caveats: OpenStreetMap data is crowd sourced and cannot be considered to be exhaustive 200 | types: 201 | - lines 202 | - points 203 | - polygons 204 | select: 205 | - name 206 | - railway 207 | - ele 208 | - operator:type 209 | - layer 210 | - addr:full 211 | - addr:city 212 | - source 213 | where: railway IN ('rail','subway','station') 214 | -------------------------------------------------------------------------------- /osm_export_tool/mappings/InAWARE.yml: -------------------------------------------------------------------------------- 1 | banks: 2 | types: 3 | - points 4 | - polygons 5 | select: 6 | - amenity 7 | - building 8 | - name 9 | - addr:full 10 | - addr:city 11 | - capacity:persons 12 | - building:levels 13 | - building:structure 14 | - building:material 15 | - building:floor 16 | - building:roof 17 | - access:roof 18 | - building:condition 19 | - ground_floor:height 20 | - backup_generator 21 | - source 22 | where: 23 | - amenity = 'bank' OR building = 'bank' 24 | 25 | communication_towers: 26 | types: 27 | - points 28 | - polygons 29 | select: 30 | - man_made 31 | - tower:type 32 | - name 33 | - height 34 | - operator 35 | - communication:mobile 36 | - communication:radio 37 | - addr:city 38 | - source 39 | where: 40 | - man_made = 'tower' AND tower:type = 'communication' 41 | 42 | airports: 43 | types: 44 | - points 45 | - polygons 46 | select: 47 | - aeroway 48 | - building 49 | - name 50 | - addr:full 51 | - addr:city 52 | - capacity:persons 53 | - building:levels 54 | - building:structure 55 | - building:material 56 | - building:floor 57 | - building:roof 58 | - access:roof 59 | - building:condition 60 | - backup_generator 61 | - source 62 | where: 63 | - aeroway = 'aerodrome' OR building = 'aerodrome' 64 | 65 | ferry_terminals: 66 | types: 67 | - points 68 | - polygons 69 | select: 70 | - amenity 71 | - building 72 | - name 73 | - addr:full 74 | - addr:city 75 | - capacity:persons 76 | - building:levels 77 | - building:structure 78 | - building:material 79 | - building:floor 80 | - building:roof 81 | - access:roof 82 | - building:condition 83 | - backup_generator 84 | - source 85 | where: 86 | - amenity = 'ferry_terminal' OR building = 'ferry_terminal' 87 | 88 | train_stations: 89 | types: 90 | - points 91 | - polygons 92 | select: 93 | - railway 94 | - name 95 | - ele 96 | - operator 97 | - addr:full 98 | - addr:city 99 | - building 100 | - source 101 | where: 102 | - railway = 'station' OR building = 'train_station' 103 | 104 | bus_stations: 105 | types: 106 | - points 107 | - polygons 108 | select: 109 | - amenity 110 | - name 111 | - addr:full 112 | - addr:city 113 | - source 114 | where: 115 | - amenity = 'bus_station' OR building = 'bus_station' 116 | 117 | roads: 118 | types: 119 | - lines 120 | select: 121 | - highway 122 | - name 123 | - layer 124 | - width 125 | - lanes 126 | - surface 127 | - smoothness 128 | - motorcycle 129 | - oneway 130 | - ref 131 | - source 132 | where: 133 | - highway IN ('motorway','trunk','primary','secondary','tertiary','service','residential','pedestrian','path','living_street','track') 134 | 135 | railways: 136 | types: 137 | - lines 138 | select: 139 | - railway 140 | - name 141 | - layer 142 | - source 143 | where: 144 | - railway = 'rail' 145 | 146 | fire_hydrants: 147 | types: 148 | - points 149 | select: 150 | - emergency 151 | - fire_hydrant:type 152 | - name 153 | - operator 154 | - addr:city 155 | - source 156 | where: 157 | - emergency = 'fire_hydrant' 158 | 159 | water_towers: 160 | types: 161 | - points 162 | - polygons 163 | select: 164 | - man_made 165 | - name 166 | - operator 167 | - addr:city 168 | - source 169 | where: 170 | - man_made = 'water_tower' 171 | 172 | pump_houses: 173 | types: 174 | - points 175 | - polygons 176 | select: 177 | - man_made 178 | - building 179 | - name 180 | - addr:full 181 | - addr:city 182 | - operator 183 | - pump:unit 184 | - elevation 185 | - capacity:pump 186 | - building:levels 187 | - building:structure 188 | - building:material 189 | - building:floor 190 | - building:roof 191 | - access:roof 192 | - building:condition 193 | - backup_generator 194 | - source 195 | where: 196 | - man_made = 'pumping_station' OR building = 'pumping_station' 197 | 198 | reservoirs: 199 | types: 200 | - points 201 | - polygons 202 | select: 203 | - landuse 204 | - name 205 | - operator 206 | - addr:city 207 | - source 208 | where: 209 | - landuse = 'reservoir' 210 | 211 | water_gates: 212 | types: 213 | - points 214 | - polygons 215 | select: 216 | - waterway 217 | - name 218 | - operator 219 | - floodgate:unit 220 | - elevation 221 | - condition 222 | - addr:city 223 | - source 224 | where: 225 | - waterway = 'floodgate' 226 | 227 | springs: 228 | types: 229 | - points 230 | - polygons 231 | select: 232 | - natural 233 | - name 234 | - operator 235 | - addr:city 236 | - source 237 | where: 238 | - natural = 'spring' 239 | 240 | embankments: 241 | types: 242 | - lines 243 | select: 244 | - man_made 245 | - name 246 | - material 247 | - source 248 | where: 249 | - man_made = 'embankment' 250 | 251 | waterways: 252 | types: 253 | - points 254 | - lines 255 | - polygons 256 | select: 257 | - name 258 | - waterway 259 | - width 260 | - source 261 | where: 262 | - waterway IS NOT NULL 263 | 264 | power_towers: 265 | types: 266 | - points 267 | select: 268 | - power 269 | - name 270 | - addr:city 271 | - operator 272 | - source 273 | where: 274 | - power = 'tower' 275 | 276 | electrical_substations: 277 | types: 278 | - points 279 | - polygons 280 | select: 281 | - power 282 | - substation 283 | - building 284 | - name 285 | - addr:city 286 | - rating 287 | - operator 288 | - source 289 | where: 290 | - power = 'substation' OR building = 'power_substation' 291 | 292 | power_plants: 293 | types: 294 | - polygons 295 | select: 296 | - power 297 | - building 298 | - name 299 | - operator 300 | - addr:full 301 | - addr:city 302 | - source 303 | where: 304 | - power = 'plant' OR building = 'power_plant' 305 | 306 | gas_stations: 307 | types: 308 | - points 309 | - polygons 310 | select: 311 | - amenity 312 | - name 313 | - addr:full 314 | - addr:city 315 | - operator 316 | - source 317 | where: 318 | - amenity = 'fuel' 319 | 320 | kindergartens: 321 | types: 322 | - points 323 | - polygons 324 | select: 325 | - amenity 326 | - building 327 | - name 328 | - addr:full 329 | - addr:city 330 | - operator:type 331 | - capcity:persons 332 | - building:levels 333 | - building:structure 334 | - building:material 335 | - building:floor 336 | - building:roof 337 | - access:roof 338 | - building:condition 339 | - ground_floor:height 340 | - backup_generator 341 | - source 342 | where: 343 | - amenity = 'kindergarten' OR building = 'kindergarten' 344 | 345 | schools: 346 | types: 347 | - points 348 | - polygons 349 | select: 350 | - school:type_idn 351 | - amenity 352 | - building 353 | - name 354 | - addr:full 355 | - addr:city 356 | - operator:type 357 | - capacity:persons 358 | - building:levels 359 | - building:structure 360 | - building:material 361 | - building:floor 362 | - building:roof 363 | - access:roof 364 | - building:condition 365 | - ground_floor:height 366 | - backup_generator 367 | - evacuation_center 368 | - shelter_type 369 | - water_source 370 | - kitchen:facilities 371 | - toilet:facilities 372 | - toilets:number 373 | - source 374 | where: 375 | - amenity = 'school' OR building = 'school' 376 | 377 | colleges: 378 | types: 379 | - points 380 | - polygons 381 | select: 382 | - amenity 383 | - building 384 | - name 385 | - addr:full 386 | - addr:city 387 | - operator:type 388 | - capacity:persons 389 | - building:levels 390 | - building:structure 391 | - building:material 392 | - building:floor 393 | - building:roof 394 | - access:roof 395 | - building:condition 396 | - ground_floor:height 397 | - backup_generator 398 | - evacuation_center 399 | - shelter_type 400 | - water_source 401 | - kitchen:facilities 402 | - toilet:facilities 403 | - toilets:number 404 | - source 405 | where: 406 | - amenity = 'college' OR building = 'college' 407 | 408 | universities: 409 | types: 410 | - points 411 | - polygons 412 | select: 413 | - amenity 414 | - building 415 | - name 416 | - addr:full 417 | - addr:city 418 | - operator:type 419 | - capacity:persons 420 | - building:levels 421 | - building:structure 422 | - building:material 423 | - building:floor 424 | - building:roof 425 | - access:roof 426 | - building:condition 427 | - ground_floor:height 428 | - backup_generator 429 | - evacuation_center 430 | - shelter_type 431 | - water_source 432 | - kitchen:facilities 433 | - toilet:facilities 434 | - toilets:number 435 | - source 436 | where: 437 | - amenity = 'university' OR building = 'university' 438 | 439 | places_of_worship: 440 | types: 441 | - points 442 | - polygons 443 | select: 444 | - amenity 445 | - religion 446 | - name 447 | - addr:full 448 | - addr:city 449 | - building 450 | - capacity:persons 451 | - building:levels 452 | - building:structure 453 | - building:material 454 | - building:floor 455 | - building:roof 456 | - access:roof 457 | - building:condition 458 | - ground_floor:height 459 | - backup_generator 460 | - evacuation_center 461 | - shelter_type 462 | - water_source 463 | - kitchen:facilities 464 | - toilet:facilities 465 | - toilets:number 466 | - source 467 | where: 468 | - amenity = 'place_of_worship' OR building IN ('mosque','church','temple') 469 | 470 | supermarkets: 471 | types: 472 | - points 473 | - polygons 474 | select: 475 | - shop 476 | - building 477 | - name 478 | - addr:full 479 | - addr:city 480 | - capacity:persons 481 | - building:levels 482 | - building:structure 483 | - building:material 484 | - building:floor 485 | - building:roof 486 | - access:roof 487 | - ground_floor:height 488 | - building:condition 489 | - backup_generator 490 | - source 491 | where: 492 | - shop = 'supermarket' OR building = 'supermarket' 493 | 494 | traditional_marketplaces: 495 | types: 496 | - points 497 | - polygons 498 | select: 499 | - amenity 500 | - building 501 | - name 502 | - addr:full 503 | - addr:city 504 | - capacity:persons 505 | - building:levels 506 | - building:structure 507 | - building:material 508 | - building:floor 509 | - building:roof 510 | - access:roof 511 | - building:condition 512 | - ground_floor:height 513 | - backup_generator 514 | - source 515 | where: 516 | - amenity = 'marketplace' OR building = 'marketplace' 517 | 518 | clinics: 519 | types: 520 | - points 521 | - polygons 522 | select: 523 | - amenity 524 | - name 525 | - addr:full 526 | - addr:city 527 | - operator:type 528 | - building 529 | - capacity:persons 530 | - building:levels 531 | - building:structure 532 | - building:material 533 | - building:floor 534 | - building:roof 535 | - access:roof 536 | - building:condition 537 | - ground_floor:height 538 | - backup_generator 539 | - evacuation_center 540 | - shelter_type 541 | - water_source 542 | - kitchen:facilities 543 | - toilet:facilities 544 | - toilets:number 545 | - source 546 | where: 547 | - amenity = 'clinic' or building = 'clinic' 548 | 549 | hospitals: 550 | types: 551 | - points 552 | - polygons 553 | select: 554 | - amenity 555 | - name 556 | - addr:full 557 | - addr:city 558 | - operator:type 559 | - building 560 | - capacity:persons 561 | - building:levels 562 | - building:structure 563 | - building:material 564 | - building:floor 565 | - building:roof 566 | - access:roof 567 | - building:condition 568 | - ground_floor:height 569 | - backup_generator 570 | - evacuation_center 571 | - shelter_type 572 | - water_source 573 | - kitchen:facilities 574 | - toilet:facilities 575 | - toilets:number 576 | - source 577 | where: 578 | - amenity = 'hospital' OR building = 'hospital' 579 | 580 | police_stations: 581 | types: 582 | - points 583 | - polygons 584 | select: 585 | - amenity 586 | - building 587 | - name 588 | - addr:full 589 | - addr:city 590 | - capacity:persons 591 | - building:levels 592 | - building:structure 593 | - building:material 594 | - building:floor 595 | - building:roof 596 | - access:roof 597 | - building:condition 598 | - ground_floor:height 599 | - backup_generator 600 | - source 601 | where: 602 | - amenity = 'police' OR building = 'police' 603 | 604 | fire_stations: 605 | types: 606 | - points 607 | - polygons 608 | select: 609 | - amenity 610 | - building 611 | - name 612 | - addr:full 613 | - addr:city 614 | - capacity:persons 615 | - building:levels 616 | - building:structure 617 | - building:material 618 | - building:floor 619 | - building:roof 620 | - access:roof 621 | - building:condition 622 | - ground_floor:height 623 | - backup_generator 624 | - source 625 | where: 626 | - amenity = 'fire_station' OR building = 'fire_station' 627 | 628 | sport_facilities: 629 | types: 630 | - points 631 | - polygons 632 | select: 633 | - leisure 634 | - sport 635 | - name 636 | - building 637 | - addr:full 638 | - addr:city 639 | - capacity:persons 640 | - building:levels 641 | - building:structure 642 | - building:material 643 | - building:floor 644 | - building:roof 645 | - access:roof 646 | - building:condition 647 | - ground_floor:height 648 | - backup_generator 649 | - evacuation_center 650 | - shelter_type 651 | - water_source 652 | - kitchen:facilities 653 | - toilet:facilities 654 | - toilets:number 655 | - source 656 | where: 657 | - leisure IN ('stadium','sports_centre','pitch','swimming_pool') OR building IN ('stadium','sports_centre') 658 | 659 | parks: 660 | types: 661 | - points 662 | - polygons 663 | select: 664 | - leisure 665 | - landuse 666 | - name 667 | - addr:full 668 | - addr:city 669 | - evacuation_center 670 | - shelter_type 671 | - water_source 672 | - kitchen:facilities 673 | - toilet:facilities 674 | - toilets:number 675 | - source 676 | where: 677 | - leisure = 'park' OR landuse='recreation_gound' 678 | 679 | local_government_offices: 680 | types: 681 | - points 682 | - polygons 683 | select: 684 | - office 685 | - building 686 | - admin_level 687 | - name 688 | - addr:full 689 | - addr:city 690 | - capacity:persons 691 | - building:levels 692 | - building:structure 693 | - building:material 694 | - building:floor 695 | - building:roof 696 | - access:roof 697 | - building:condition 698 | - ground_floor:height 699 | - backup_generator 700 | - evacuation_center 701 | - shelter_type 702 | - water_source 703 | - kitchen:facilities 704 | - toilet:facilities 705 | - toilets:number 706 | - source 707 | where: 708 | - office = 'government' OR building IN ('governor_office', 'townhall','subdistrict_office','village_office','community_group_office') 709 | 710 | 711 | government_offices: 712 | types: 713 | - points 714 | - polygons 715 | select: 716 | - office 717 | - building 718 | - name 719 | - addr:full 720 | - addr:city 721 | - admin_level 722 | - capacity:persons 723 | - building:levels 724 | - building:structure 725 | - building:material 726 | - building:floor 727 | - building:roof 728 | - access:roof 729 | - building:condition 730 | - ground_floor:height 731 | - backup_generator 732 | - source 733 | where: 734 | - office = 'government' OR building = 'government_office' 735 | 736 | administrative_boundaries: 737 | types: 738 | - polygons 739 | select: 740 | - type 741 | - boundary 742 | - name 743 | - admin_level 744 | - is_in:province 745 | - is_in:city 746 | - is_in:town 747 | - is_in:municipality 748 | - is_in:village 749 | - is_in:RW 750 | - flood_prone 751 | - landslide_prone 752 | - source 753 | where: 754 | - type = 'boundary' AND boundary = 'administrative' 755 | -------------------------------------------------------------------------------- /osm_export_tool/mappings/default.yml: -------------------------------------------------------------------------------- 1 | admin: 2 | types: 3 | - polygons 4 | select: 5 | - name 6 | - boundary 7 | - admin_level 8 | where: 9 | - boundary = 'administrative' 10 | 11 | aeroways: 12 | types: 13 | - lines 14 | select: 15 | - aeroway 16 | where: 17 | - aeroway IN ('runway','taxiway') 18 | 19 | amenities: 20 | types: 21 | - points 22 | select: 23 | - name 24 | - amenity 25 | where: 26 | - amenity IN ('university','school','library','fuel','hospital','fire_station','police','townhall') 27 | 28 | barrierpoints: 29 | types: 30 | - points 31 | select: 32 | - name 33 | - barrier 34 | where: 35 | - barrier IN ('block','bollard','cattle_grid','chain','cycle_barrier','entrance','horse_stile','gate','spikes','lift_gate','kissing_gate','fence','yes','wire_fence','toll_booth','stile') 36 | 37 | barrierways: 38 | types: 39 | - lines 40 | select: 41 | - name 42 | - barrier 43 | where: 44 | - barrier IN ('city_wall','fence','hedge','retaining_wall','wall','bollard','gate','spikes','lift_gate','kissing_gate','embankment','yes','wire_fence') 45 | 46 | buildings: 47 | types: 48 | - polygons 49 | select: 50 | - name 51 | - building 52 | where: 53 | - building IS NOT NULL 54 | 55 | housenumbers: 56 | types: 57 | - points 58 | select: 59 | - name 60 | - addr:housenumber 61 | - addr:street 62 | - addr:postcode 63 | - addr:city 64 | where: 65 | - addr:housenumber IS NOT NULL 66 | 67 | housenumbers_interpolated: 68 | types: 69 | - points 70 | select: 71 | - name 72 | - addr:interpolation 73 | - addr:street 74 | - addr:postcode 75 | - addr:city 76 | - addr:inclusion 77 | where: 78 | - addr:interpolation IS NOT NULL 79 | 80 | landusages: 81 | types: 82 | - polygons 83 | select: 84 | - name 85 | - aeroway 86 | - amenity 87 | - barrier 88 | - highway 89 | - landuse 90 | - leisure 91 | - man_made 92 | - military 93 | - natural 94 | - place 95 | - tourism 96 | where: 97 | - aeroway IN ('runway','taxiway') 98 | - amenity IN ('university','school','college','library','fuel','parking','cinema','theatre','place_of_worship','hospital') 99 | - barrier = 'hedge' 100 | - highway IN ('pedestrian','footway') 101 | - landuse IN ('park','forest','residential','retail','commercial','industrial','railway','cemetery','grass','farmyard','farm','farmland','orchard','vineyard','wood','meadow','village_green','recreation_ground','allotments','quarry') 102 | - leisure IN ('park','garden','playground','golf_course','sports_centre','pitch','stadium','common','nature_reserve') 103 | - man_made = 'pier' 104 | - military = 'barracks' 105 | - natural IN ('wood','land','scrub','wetland','heath') 106 | - place = 'island' 107 | - tourism = 'zoo' 108 | 109 | places: 110 | types: 111 | - points 112 | select: 113 | - name 114 | - place 115 | - population 116 | where: 117 | - place IN ('country','state','region','county','city','town','village','hamlet','suburb','locality') 118 | 119 | roads: 120 | types: 121 | - lines 122 | select: 123 | - name 124 | - tunnel 125 | - bridge 126 | - oneway 127 | - ref 128 | - access 129 | - service 130 | - railway 131 | - highway 132 | - man_made 133 | where: 134 | - railway IN ('rail','tram','light_rail','subway','narrow_gauge','preserved','funicular','monorail','disused') 135 | - highway IN ('motorway','motorway_link','trunk','trunk_link','primary','primary_link','secondary','secondary_link','tertiary','tertiary_link','road','path','track','service','footway','bridleway','cycleway','steps','pedestrian','living_street','unclassified','residential','raceway') 136 | - man_made IN ('pier','groyne') 137 | 138 | transport_areas: 139 | types: 140 | - polygons 141 | select: 142 | - name 143 | - aeroway 144 | - railway 145 | where: 146 | - aeroway IN ('aerodrome','terminal','helipad','apron') 147 | - railway IN ('station','platform') 148 | 149 | transport_points: 150 | types: 151 | - points 152 | select: 153 | - name 154 | - aeroway 155 | - highway 156 | - railway 157 | where: 158 | - aeroway IN ('aerodrome','terminal','helipad','gate') 159 | - highway IN ('motorway_junction','turning_circle','bus_stop') 160 | - railway IN ('station','halt','tram_stop','crossing','level_crossing','subway_entrance') 161 | 162 | waterareas: 163 | types: 164 | - polygons 165 | select: 166 | - name 167 | - amenity 168 | - landuse 169 | - leisure 170 | - natural 171 | - waterway 172 | where: 173 | - amenity = 'swimming_pool' 174 | - landuse IN ('basin','reservoir') 175 | - leisure = 'swimming_pool' 176 | - natural = 'water' 177 | - waterway = 'riverbank' 178 | 179 | waterways: 180 | types: 181 | - lines 182 | select: 183 | - name 184 | - barrier 185 | - waterway 186 | where: 187 | - barrier = 'ditch' 188 | - waterway IN ('stream','river','canal','drain','ditch') 189 | 190 | -------------------------------------------------------------------------------- /osm_export_tool/mappings/simple.yml: -------------------------------------------------------------------------------- 1 | buildings: 2 | types: 3 | - polygons 4 | select: 5 | - name 6 | - building 7 | where: 8 | - building = 'yes' 9 | -------------------------------------------------------------------------------- /osm_export_tool/nontabular.py: -------------------------------------------------------------------------------- 1 | import os 2 | from os.path import join 3 | import pathlib 4 | import subprocess 5 | from osm_export_tool import File 6 | import landez 7 | 8 | def osmand(input_pbf,map_creator_dir,tempdir=None,jvm_mem=[256,2048]): 9 | BATCH_XML = """ 10 | 11 | 13 | 14 | 23 | 24 | 25 | """ 26 | CLASSPATH = "{map_creator_dir}/OsmAndMapCreator.jar:{map_creator_dir}/lib/OsmAnd-core.jar:{map_creator_dir}/lib/*.jar" 27 | 28 | pathlib.Path(join(tempdir,'osmand')).mkdir(parents=True, exist_ok=True) 29 | 30 | try: 31 | os.link(input_pbf,join(tempdir,'osmand','osmand.osm.pbf')) 32 | except: 33 | pass 34 | 35 | with open(join(tempdir,'batch.xml'),'w') as b: 36 | b.write(BATCH_XML.format(tempdir=tempdir)) 37 | 38 | subprocess.check_call([ 39 | 'java', 40 | '-Xms{0}M'.format(jvm_mem[0]), 41 | '-Xmx{0}M'.format(jvm_mem[1]), 42 | '-cp', 43 | CLASSPATH.format(map_creator_dir=map_creator_dir), 44 | 'net.osmand.util.IndexBatchCreator', 45 | join(tempdir,'batch.xml') 46 | ]) 47 | return [File('osmand_obf',[join(tempdir,'Osmand_2.obf')])] 48 | 49 | def garmin(input_pbf,splitter_jar,mkgmap_jar,tempdir=None,jvm_mem=[256,2048]): 50 | """ 51 | Converts PBF to Garmin IMG format. 52 | 53 | Splits pbf into smaller tiles, generates .img files for each split, 54 | then patches the .img files back into a single .img file 55 | suitable for deployment to a Garmin GPS unit. 56 | NOTE: disabled poly bounds: see https://github.com/hotosm/osm-export-tool2/issues/248 57 | """ 58 | subprocess.check_call([ 59 | 'java', 60 | '-Xms{0}M'.format(jvm_mem[0]), 61 | '-Xmx{0}M'.format(jvm_mem[1]), 62 | '-jar', 63 | splitter_jar, 64 | '--output-dir=' + tempdir, 65 | input_pbf 66 | ]) 67 | 68 | # Generate the IMG file. 69 | # get the template.args file created by splitter 70 | # see: http://wiki.openstreetmap.org/wiki/Mkgmap/help/splitter 71 | subprocess.check_call([ 72 | 'java', 73 | '-Xms{0}M'.format(jvm_mem[0]), 74 | '-Xmx{0}M'.format(jvm_mem[1]), 75 | '-jar', 76 | mkgmap_jar, 77 | '--gmapsupp', 78 | '--output-dir=' + tempdir, 79 | '--description="HOT Export Garmin Map"', 80 | '--mapname=80000111', 81 | '--family-name="HOT Export Tool"', 82 | '--family-id=2', 83 | '--series-name="HOT Export Tool"', 84 | '--index', 85 | '--route', 86 | '--generate-sea=extend-sea-sectors', 87 | '--draw-priority=100', 88 | '--unicode', 89 | '--read-config={0}/template.args'.format(tempdir) 90 | ]) 91 | return [File('garmin',[join(tempdir,'gmapsupp.img')])] 92 | 93 | def mwm(input_pbf,output_dir,generate_mwm_path,generator_tool_path,osmconvert_path='osmconvert'): 94 | base_name = (os.path.basename(input_pbf).split(os.extsep))[0] 95 | env = os.environ.copy() 96 | env.update(OSMCONVERT=osmconvert_path,TARGET=output_dir,GENERATOR_TOOL=generator_tool_path) 97 | subprocess.check_call([ 98 | generate_mwm_path, 99 | input_pbf 100 | ],env=env) 101 | return [File('mwm',[join(output_dir,base_name + '.mwm')])] 102 | 103 | def mbtiles(geom,filepath,tiles_url,minzoom,maxzoom): 104 | mb = landez.MBTilesBuilder(cache=False,tiles_url=tiles_url,tiles_headers={'User-Agent':'github.com/hotosm/osm-export-tool'},filepath=filepath) 105 | mb.add_coverage(bbox=geom.bounds, 106 | zoomlevels=[minzoom,maxzoom]) 107 | mb.run() 108 | return [File('mbtiles',[filepath],{'minzoom':minzoom,'maxzoom':maxzoom,'source':tiles_url})] 109 | -------------------------------------------------------------------------------- /osm_export_tool/package.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from os.path import basename 4 | import zipfile 5 | import tarfile 6 | import io 7 | from shapely.geometry import mapping 8 | from osm_export_tool import File 9 | 10 | def create_package(destination,files,boundary_geom=None,output_name='zip'): 11 | with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED, True) as z: 12 | if boundary_geom: 13 | z.writestr("clipping_boundary.geojson", json.dumps(mapping(boundary_geom))) 14 | for file in files: 15 | for part in file.parts: 16 | z.write(part, os.path.basename(part)) 17 | 18 | return File(output_name,[destination]) 19 | 20 | def create_posm_bundle(destination,files,title,name,description,geom): 21 | contents = {} 22 | with tarfile.open(destination, "w|gz") as bundle: 23 | for file in files: 24 | for part in file.parts: 25 | if file.output_name == 'shp': 26 | target = 'data/' + basename(part) 27 | contents[target] = {'Type':'ESRI Shapefile'} 28 | elif file.output_name == 'kml': 29 | target = 'data/' + basename(part) 30 | contents[target] = {'Type':'KML'} 31 | elif file.output_name == 'gpkg': 32 | target = 'data/' + basename(part) 33 | contents[target] = {'Type':'Geopackage'} 34 | elif file.output_name == 'osmand_obf': 35 | target = 'navigation/' + basename(part) 36 | contents[target] = {'Type':'OsmAnd'} 37 | elif file.output_name == 'garmin': 38 | target = 'navigation/' + basename(part) 39 | contents[target] = {'Type':'Garmin IMG'} 40 | elif file.output_name == 'mwm': 41 | target = 'navigation/' + basename(part) 42 | contents[target] = {'Type':'Maps.me'} 43 | elif file.output_name == 'osm_pbf': 44 | target = 'osm/' + basename(part) 45 | contents[target] = {'Type':'OSM/PBF'} 46 | elif file.output_name == 'mbtiles': 47 | target = 'tiles/' + basename(part) 48 | contents[target] = { 49 | 'type':'MBTiles', 50 | 'minzoom':file.extra['minzoom'], 51 | 'maxzoom':file.extra['maxzoom'], 52 | 'source':file.extra['source'] 53 | } 54 | bundle.add(part,target) 55 | 56 | data = json.dumps({ 57 | 'title':title, 58 | 'name':name, 59 | 'description':description, 60 | 'bbox':geom.bounds, 61 | 'contents':contents 62 | },indent=2).encode() 63 | tarinfo = tarfile.TarInfo('manifest.json') 64 | tarinfo.size = len(data) 65 | bundle.addfile(tarinfo, io.BytesIO(data)) 66 | 67 | return File('bundle',[destination]) 68 | -------------------------------------------------------------------------------- /osm_export_tool/sources.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import pathlib 4 | import ast 5 | import shutil 6 | import subprocess 7 | import time 8 | from sre_constants import SUCCESS 9 | from string import Template 10 | from xml.dom import ValidationErr 11 | 12 | import requests 13 | import shapely.geometry 14 | from deepdiff import DeepDiff 15 | from requests.exceptions import Timeout 16 | 17 | from osm_export_tool.sql import to_prefix 18 | 19 | # path must return a path to an .osm.pbf or .osm.xml on the filesystem 20 | MAX_RETRIES = 5 21 | RETRY_DELAY = 60 22 | 23 | 24 | class Pbf: 25 | def __init__(self, path): 26 | self._path = path 27 | 28 | def fetch(self): 29 | pass 30 | 31 | def path(self): 32 | return self._path 33 | 34 | 35 | class OsmExpress: 36 | def __init__( 37 | self, osmx_path, db_path, geom, output_path, use_existing=True, tempdir=None 38 | ): 39 | self.osmx_path = osmx_path 40 | self.db_path = db_path 41 | self.geom = geom 42 | self.output_path = output_path 43 | self.use_existing = use_existing 44 | self.tempdir = tempdir 45 | 46 | def fetch(self): 47 | region_json = os.path.join(self.tempdir, "region.json") 48 | with open(region_json, "w") as f: 49 | f.write(json.dumps(shapely.geometry.mapping(self.geom))) 50 | subprocess.check_call( 51 | [ 52 | self.osmx_path, 53 | "extract", 54 | self.db_path, 55 | self.output_path, 56 | "--region", 57 | region_json, 58 | ] 59 | ) 60 | os.remove(region_json) 61 | 62 | def path(self): 63 | if os.path.isfile(self.output_path) and self.use_existing: 64 | return self.output_path 65 | else: 66 | self.fetch() 67 | return self.output_path 68 | 69 | 70 | class OsmiumTool: 71 | def __init__( 72 | self, 73 | osmium_path, 74 | source_path, 75 | geom, 76 | output_path, 77 | use_existing=True, 78 | tempdir=None, 79 | mapping=None, 80 | ): 81 | self.osmium_path = osmium_path 82 | self.source_path = source_path 83 | self.geom = geom 84 | self.output_path = output_path 85 | self.use_existing = use_existing 86 | self.tempdir = tempdir 87 | self.mapping = mapping 88 | 89 | @classmethod 90 | def parts(cls, expr): 91 | def _parts(prefix): 92 | op = prefix[0] 93 | if op == "=": 94 | return ["{0}={1}".format(prefix[1], prefix[2])] 95 | if op == "!=": 96 | return ["{0}!={1}".format(prefix[1], prefix[2])] 97 | if op in ["<", ">", "<=", ">="] or op == "notnull": 98 | raise ValueError("{0} where clause not supported".format(op)) 99 | if op == "in": 100 | x = "{0}={1}".format(prefix[1], ",".join(prefix[2])) 101 | return [x] 102 | if op == "and" or op == "or": 103 | return _parts(prefix[1]) + _parts(prefix[2]) 104 | 105 | return _parts(expr) 106 | 107 | @staticmethod 108 | def get_element_filter(theme, part): 109 | elements = [] 110 | if theme.points: 111 | elements.append("n/{0}".format(part)) # node 112 | if theme.lines: 113 | elements.append("w/{0}".format(part)) # way 114 | if theme.polygons: 115 | elements.append("r/{0}".format(part)) # relation 116 | 117 | return elements 118 | 119 | @classmethod 120 | def filters(cls, mapping): 121 | filters_set = set() 122 | tags = set() 123 | for t in mapping.themes: 124 | prefix = t.matcher.expr 125 | parts = cls.parts(prefix) 126 | for part in parts: 127 | [filters_set.add(e) for e in OsmiumTool.get_element_filter(t, part)] 128 | key = [t for t in t.keys if t in part] 129 | if len(key) == 1: 130 | tags.add(key[0]) 131 | 132 | return filters_set 133 | 134 | def tags_filter(self, filters, planet_as_source): 135 | source_path = self.output_path 136 | if planet_as_source is True: 137 | source_path = self.source_path 138 | 139 | cmd = [self.osmium_path, "tags-filter", source_path, "-o", self.output_path] 140 | 141 | for f in filters: 142 | cmd.insert(3, f) 143 | 144 | if planet_as_source is False: 145 | cmd.append("--overwrite") 146 | 147 | subprocess.check_call(cmd) 148 | 149 | def fetch(self): 150 | region_json = os.path.join(self.tempdir, "region.json") 151 | with open(region_json, "w") as f: 152 | f.write( 153 | json.dumps( 154 | {"type": "Feature", "geometry": shapely.geometry.mapping(self.geom)} 155 | ) 156 | ) 157 | subprocess.check_call( 158 | [ 159 | self.osmium_path, 160 | "extract", 161 | "-p", 162 | region_json, 163 | self.source_path, 164 | "-o", 165 | self.output_path, 166 | "--overwrite", 167 | ] 168 | ) 169 | os.remove(region_json) 170 | 171 | def path(self): 172 | if os.path.isfile(self.output_path) and self.use_existing: 173 | return self.output_path 174 | 175 | planet_as_source = True 176 | if self.geom.area < 6e4: 177 | self.fetch() 178 | planet_as_source = False 179 | 180 | if self.mapping is not None: 181 | filters = OsmiumTool.filters(self.mapping) 182 | self.tags_filter(filters, planet_as_source) 183 | 184 | return self.output_path 185 | 186 | 187 | class Overpass: 188 | @classmethod 189 | def filters(cls, mapping): 190 | nodes = set() 191 | ways = set() 192 | relations = set() 193 | for t in mapping.themes: 194 | parts = cls.parts(t.matcher.expr) 195 | if t.points: 196 | for part in parts: 197 | nodes.add(part) 198 | if t.lines: 199 | for part in parts: 200 | ways.add(part) 201 | if t.polygons: 202 | for part in parts: 203 | ways.add(part) 204 | relations.add(part) 205 | return nodes, ways, relations 206 | 207 | # force quoting of strings to handle keys with colons 208 | @classmethod 209 | def parts(cls, expr): 210 | def _parts(prefix): 211 | op = prefix[0] 212 | if op == "=": 213 | return ["['{0}'='{1}']".format(prefix[1], prefix[2])] 214 | if op == "!=": 215 | return ["['{0}'!='{1}']".format(prefix[1], prefix[2])] 216 | if op in ["<", ">", "<=", ">="] or op == "notnull": 217 | return ["['{0}']".format(prefix[1])] 218 | if op == "in": 219 | x = "['{0}'~'{1}']".format(prefix[1], "|".join(prefix[2])) 220 | return [x] 221 | if op == "and" or op == "or": 222 | return _parts(prefix[1]) + _parts(prefix[2]) 223 | 224 | return _parts(expr) 225 | 226 | @classmethod 227 | def sql(cls, str): 228 | return cls.parts(to_prefix(str)) 229 | 230 | def __init__( 231 | self, 232 | hostname, 233 | geom, 234 | path, 235 | use_existing=True, 236 | tempdir=None, 237 | osmconvert_path="osmconvert", 238 | mapping=None, 239 | use_curl=False, 240 | ): 241 | self.hostname = hostname 242 | self._path = path 243 | self.geom = geom 244 | self.use_existing = use_existing 245 | self.osmconvert_path = osmconvert_path 246 | self.tmp_path = os.path.join(tempdir, "tmp.osm.xml") 247 | self.mapping = mapping 248 | self.use_curl = use_curl 249 | self.tempdir = tempdir 250 | 251 | def fetch(self): 252 | base_template = Template( 253 | "[maxsize:$maxsize][timeout:$timeout];$query;out meta;" 254 | ) 255 | 256 | if self.geom.geom_type == "Polygon": 257 | geom = 'poly:"{0}"'.format( 258 | " ".join(["{1} {0}".format(*x) for x in self.geom.exterior.coords]) 259 | ) 260 | else: 261 | bounds = self.geom.bounds 262 | west = max(bounds[0], -180) 263 | south = max(bounds[1], -90) 264 | east = min(bounds[2], 180) 265 | north = min(bounds[3], 90) 266 | geom = "{1},{0},{3},{2}".format(west, south, east, north) 267 | 268 | if self.mapping: 269 | query = """( 270 | ( 271 | {0} 272 | ); 273 | ( 274 | {1} 275 | );>; 276 | ( 277 | {2} 278 | );>>;>;)""" 279 | nodes, ways, relations = Overpass.filters(self.mapping) 280 | nodes = "\n".join(["node({0}){1};".format(geom, f) for f in nodes]) 281 | ways = "\n".join(["way({0}){1};".format(geom, f) for f in ways]) 282 | relations = "\n".join( 283 | ["relation({0}){1};".format(geom, f) for f in relations] 284 | ) 285 | query = query.format(nodes, ways, relations) 286 | else: 287 | query = "(node({0});<;>>;>;)".format(geom) 288 | 289 | data = base_template.substitute(maxsize=2147483648, timeout=1600, query=query) 290 | 291 | if self.use_curl: 292 | with open(os.path.join(self.tempdir, "query.txt"), "w") as query_txt: 293 | query_txt.write(data) 294 | subprocess.check_call( 295 | [ 296 | "curl", 297 | "-X", 298 | "POST", 299 | "-d", 300 | "@" + os.path.join(self.tempdir, "query.txt"), 301 | os.path.join(self.hostname, "api", "interpreter"), 302 | "-o", 303 | self.tmp_path, 304 | ] 305 | ) 306 | else: 307 | with requests.post( 308 | os.path.join(self.hostname, "api", "interpreter"), 309 | data=data, 310 | stream=True, 311 | ) as r: 312 | with open(self.tmp_path, "wb") as f: 313 | shutil.copyfileobj(r.raw, f) 314 | 315 | with open(self.tmp_path, "r") as f: 316 | sample = [next(f) for x in range(6)] 317 | if "DOCTYPE html" in sample[1]: 318 | raise Exception("Overpass failure") 319 | if "remark" in sample[5]: 320 | raise Exception(sample[5]) 321 | # run osmconvert on the file 322 | try: 323 | subprocess.check_call( 324 | [self.osmconvert_path, self.tmp_path, "--out-pbf", "-o=" + self._path] 325 | ) 326 | except subprocess.CalledProcessError as e: 327 | raise ValidationErr(e) 328 | os.remove(self.tmp_path) 329 | 330 | def path(self): 331 | if os.path.isfile(self._path) and self.use_existing: 332 | return self._path 333 | else: 334 | self.fetch() 335 | return self._path 336 | 337 | 338 | class Galaxy: 339 | """Transfers Yaml Language to Galaxy Query Make a request and sends response back from fetch()""" 340 | 341 | @classmethod 342 | def hdx_filters(cls, t): 343 | geometryType = [] 344 | or_filter, and_filter, point_filter, line_filter, poly_filter = ( 345 | {}, 346 | {}, 347 | {}, 348 | {}, 349 | {}, 350 | ) 351 | point_columns, line_columns, poly_columns = [], [], [] 352 | 353 | parts, and_clause = cls.parts(t.matcher.expr) 354 | if len(and_clause) > 0: 355 | ### FIX ME to support and clause with multiple condition 356 | temp_and_clause = [] 357 | for clause in and_clause: 358 | for ause in clause: 359 | temp_and_clause.append(ause) 360 | 361 | and_clause = temp_and_clause 362 | for cl in and_clause: 363 | if cl in parts: 364 | parts.remove(cl) 365 | # ----- 366 | and_filter = cls.remove_duplicates( 367 | cls.where_filter(temp_and_clause, and_filter) 368 | ) 369 | 370 | or_filter = cls.remove_duplicates(cls.where_filter(parts, or_filter)) 371 | if t.points: 372 | point_columns = cls.attribute_filter(t) 373 | geometryType.append("point") 374 | point_filter = {"join_or": or_filter, "join_and": and_filter} 375 | 376 | if t.lines: 377 | line_columns = cls.attribute_filter(t) 378 | geometryType.append("line") 379 | line_filter = {"join_or": or_filter, "join_and": and_filter} 380 | 381 | if t.polygons: 382 | poly_columns = cls.attribute_filter(t) 383 | geometryType.append("polygon") 384 | poly_filter = {"join_or": or_filter, "join_and": and_filter} 385 | 386 | return ( 387 | point_filter, 388 | line_filter, 389 | poly_filter, 390 | geometryType, 391 | point_columns, 392 | line_columns, 393 | poly_columns, 394 | ) 395 | 396 | @classmethod 397 | def filters(cls, mapping): 398 | geometryType = [] 399 | or_filter, and_filter, point_filter, line_filter, poly_filter = ( 400 | {}, 401 | {}, 402 | {}, 403 | {}, 404 | {}, 405 | ) 406 | point_columns, line_columns, poly_columns = [], [], [] 407 | 408 | for t in mapping.themes: 409 | parts, and_clause = cls.parts(t.matcher.expr) 410 | 411 | if len(and_clause) > 0: 412 | ### FIX ME to support and clause with multiple condition 413 | temp_and_clause = [] 414 | for clause in and_clause: 415 | for ause in clause: 416 | temp_and_clause.append(ause) 417 | 418 | and_clause = temp_and_clause 419 | for cl in and_clause: 420 | if cl in parts: 421 | parts.remove(cl) 422 | # ----- 423 | and_filter = cls.remove_duplicates( 424 | cls.where_filter(temp_and_clause, and_filter) 425 | ) 426 | 427 | or_filter = cls.remove_duplicates(cls.where_filter(parts, or_filter)) 428 | 429 | if t.points: 430 | point_columns = cls.attribute_filter(t) 431 | geometryType.append("point") 432 | point_filter = {"join_or": or_filter, "join_and": and_filter} 433 | 434 | if t.lines: 435 | line_columns = cls.attribute_filter(t) 436 | geometryType.append("line") 437 | line_filter = {"join_or": or_filter, "join_and": and_filter} 438 | 439 | if t.polygons: 440 | poly_columns = cls.attribute_filter(t) 441 | geometryType.append("polygon") 442 | poly_filter = {"join_or": or_filter, "join_and": and_filter} 443 | 444 | return ( 445 | point_filter, 446 | line_filter, 447 | poly_filter, 448 | geometryType, 449 | point_columns, 450 | line_columns, 451 | poly_columns, 452 | ) 453 | 454 | @classmethod 455 | def remove_duplicates(cls, entries_dict): 456 | for key, value in entries_dict.items(): 457 | entries_dict[key] = list(dict.fromkeys(value)) 458 | return entries_dict 459 | 460 | # force quoting of strings to handle keys with colons 461 | @classmethod 462 | def parts(cls, expr, and_clause=[]): 463 | def _parts(prefix): 464 | op = prefix[0] 465 | if op == "=": 466 | return [""" "{0}":["{1}"] """.format(prefix[1], prefix[2])] 467 | if ( 468 | op == "!=" 469 | ): # fixme this will require improvement in rawdata api is not implemented yet 470 | pass 471 | # return ["['{0}'!='{1}']".format(prefix[1],prefix[2])] 472 | if op in ["<", ">", "<=", ">="] or op == "notnull": 473 | return [""" "{0}":[] """.format(prefix[1])] 474 | if op == "in": 475 | x = """ "{0}":["{1}"]""".format(prefix[1], """ "," """.join(prefix[2])) 476 | return [x] 477 | if op == "and": 478 | and_clause.append(_parts(prefix[1]) + _parts(prefix[2])) 479 | return _parts(prefix[1]) + _parts(prefix[2]) 480 | if op == "or": 481 | return _parts(prefix[1]) + _parts(prefix[2]) 482 | 483 | return _parts(expr), and_clause 484 | 485 | @classmethod 486 | def attribute_filter(cls, theme): 487 | columns = theme.keys 488 | return list(columns) 489 | 490 | @classmethod 491 | def where_filter(cls, parts, filter_dict): 492 | for part in parts: 493 | part_dict = json.loads(f"""{'{'}{part.strip()}{'}'}""") 494 | for key, value in part_dict.items(): 495 | if key not in filter_dict: 496 | filter_dict[key] = value 497 | else: 498 | if ( 499 | filter_dict.get(key) != [] 500 | ): # only add other values if not null condition is not applied to that key 501 | if ( 502 | value == [] 503 | ): # if incoming value is not null i.e. key = * ignore previously added values 504 | filter_dict[key] = value 505 | else: 506 | filter_dict[ 507 | key 508 | ] += value # if value was not previously = * then and value is not =* then add values 509 | 510 | return filter_dict 511 | 512 | def __init__(self, hostname, geom, mapping=None, file_name="", access_token=None,userinfo=False): 513 | self.hostname = hostname 514 | self.geom = geom 515 | self.mapping = mapping 516 | self.file_name = file_name 517 | self.access_token = access_token 518 | self.userinfo=userinfo 519 | 520 | def fetch( 521 | self, 522 | output_format, 523 | is_hdx_export=False, 524 | all_feature_filter_json=None, 525 | min_zoom=None, 526 | max_zoom=None, 527 | ): 528 | if all_feature_filter_json: 529 | with open(all_feature_filter_json, encoding="utf-8") as all_features: 530 | all_features_filters = json.loads(all_features.read()) 531 | geom = shapely.geometry.mapping(self.geom) 532 | 533 | def format_response(res_item): 534 | if isinstance(res_item, str): 535 | return ast.literal_eval(res_item) 536 | return res_item 537 | 538 | if self.mapping: 539 | if is_hdx_export: 540 | # hdx block 541 | fullresponse = [] 542 | for t in self.mapping.themes: 543 | ( 544 | point_filter, 545 | line_filter, 546 | poly_filter, 547 | geometryType_filter, 548 | point_columns, 549 | line_columns, 550 | poly_columns, 551 | ) = Galaxy.hdx_filters(t) 552 | osmTags = point_filter 553 | if point_filter == line_filter == poly_filter: 554 | osmTags = point_filter # master filter that will be applied to all type of osm elements : current implementation of galaxy api 555 | else: 556 | osmTags = {} 557 | if point_columns == line_columns == poly_columns: 558 | columns = point_columns 559 | else: 560 | columns = [] 561 | if len(geometryType_filter) == 0: 562 | geometryType_filter = ["point", "line", "polygon"] 563 | 564 | for geomtype in geometryType_filter: 565 | geomtype_to_pass = [geomtype] 566 | formatted_file_name = f"""{self.file_name.lower()}_{t.name.lower()}_{geomtype.lower()}s_{output_format.lower()}""" 567 | if ( 568 | osmTags 569 | ): # if it is a master filter i.e. filter same for all type of feature 570 | if columns: 571 | request_body = { 572 | "fileName": formatted_file_name, 573 | "geometry": geom, 574 | "outputType": output_format, 575 | "geometryType": geomtype_to_pass, 576 | "filters": { 577 | "tags": {"all_geometry": osmTags}, 578 | "attributes": {"all_geometry": columns}, 579 | }, 580 | } 581 | else: 582 | request_body = { 583 | "fileName": formatted_file_name, 584 | "geometry": geom, 585 | "outputType": output_format, 586 | "geometryType": geomtype_to_pass, 587 | "filters": { 588 | "tags": {"all_geometry": osmTags}, 589 | "attributes": { 590 | "point": point_columns, 591 | "line": line_columns, 592 | "polygon": poly_columns, 593 | }, 594 | }, 595 | } 596 | else: 597 | if columns: 598 | request_body = { 599 | "fileName": formatted_file_name, 600 | "geometry": geom, 601 | "outputType": output_format, 602 | "geometryType": geomtype_to_pass, 603 | "filters": { 604 | "tags": { 605 | "point": point_filter, 606 | "line": line_filter, 607 | "polygon": poly_filter, 608 | }, 609 | "attributes": {"all_geometry": columns}, 610 | }, 611 | } 612 | else: 613 | request_body = { 614 | "fileName": formatted_file_name, 615 | "geometry": geom, 616 | "outputType": output_format, 617 | "geometryType": geomtype_to_pass, 618 | "filters": { 619 | "tags": { 620 | "point": point_filter, 621 | "line": line_filter, 622 | "polygon": poly_filter, 623 | }, 624 | "attributes": { 625 | "point": point_columns, 626 | "line": line_columns, 627 | "polygon": poly_columns, 628 | }, 629 | }, 630 | } 631 | # sending post request and saving response as response object 632 | 633 | headers = { 634 | "accept": "application/json", 635 | "Content-Type": "application/json", 636 | } 637 | if self.access_token: 638 | headers["access-token"] = self.access_token 639 | # print(request_body) 640 | try: 641 | if all_feature_filter_json: 642 | if ( 643 | len( 644 | DeepDiff( 645 | request_body["filters"], 646 | all_features_filters, 647 | ignore_order=True, 648 | ) 649 | ) 650 | < 1 651 | ): # that means user is selecting all the options available on export tool 652 | request_body["filters"] = {} 653 | 654 | with requests.Session() as req_session: 655 | # print("printing before sending") 656 | # print(json.dumps(request_body)) 657 | request_body["uuid"] = "false" 658 | for retry in range(MAX_RETRIES): 659 | r = req_session.post( 660 | url=f"{self.hostname}v1/snapshot/", 661 | data=json.dumps(request_body), 662 | headers=headers, 663 | timeout=60 * 5, 664 | ) 665 | 666 | if r.status_code == 429: # Rate limited 667 | print( 668 | f"Rate limited, retrying in {RETRY_DELAY} seconds..." 669 | ) 670 | time.sleep(RETRY_DELAY) 671 | elif r.status_code != 200: # Unexpected status code 672 | if r.status_code == 422: # Unprocessable Entity 673 | try: 674 | error_message = r.json().get("detail")[ 675 | 0 676 | ]["msg"] 677 | except ( 678 | json.JSONDecodeError, 679 | KeyError, 680 | IndexError, 681 | ): 682 | error_message = "Unknown error occurred" 683 | raise ValueError( 684 | f"Error {r.status_code}: {error_message}" 685 | ) 686 | else: 687 | r.raise_for_status() # Raise other non-200 errors 688 | else: # Success 689 | break 690 | if r.ok: 691 | res = r.json() 692 | else: 693 | raise ValueError(r.content) 694 | url = f"{self.hostname}v1{res['track_link']}" 695 | success = False 696 | while not success: 697 | with requests.Session() as api: 698 | r = api.get(url) 699 | r.raise_for_status() 700 | if r.ok: 701 | res = r.json() 702 | if res["status"] == "FAILURE": 703 | raise ValueError( 704 | 705 | "Task failed from raw data api" 706 | ) 707 | if res["status"] == "SUCCESS": 708 | success = True 709 | response_back = res["result"] 710 | response_back["theme"] = t.name 711 | response_back["output_name"] = output_format 712 | fullresponse.append(response_back) 713 | time.sleep( 714 | 0.5 715 | ) # wait one half sec before making another request 716 | else : 717 | time.sleep(2) # Check every 2s for hdx 718 | 719 | except requests.exceptions.RequestException as ex: 720 | raise ex 721 | 722 | return fullresponse 723 | else: 724 | ( 725 | point_filter, 726 | line_filter, 727 | poly_filter, 728 | geometryType_filter, 729 | point_columns, 730 | line_columns, 731 | poly_columns, 732 | ) = Galaxy.filters(self.mapping) 733 | osmTags = point_filter 734 | if point_filter == line_filter == poly_filter: 735 | osmTags = point_filter # master filter that will be applied to all type of osm elements : current implementation of galaxy api 736 | else: 737 | osmTags = {} 738 | if point_columns == line_columns == poly_columns: 739 | columns = point_columns 740 | else: 741 | columns = [] 742 | 743 | if ( 744 | osmTags 745 | ): # if it is a master filter i.e. filter same for all type of feature 746 | attribute_meta = ( 747 | {"all_geometry": columns} 748 | if columns 749 | else { 750 | "point": point_columns, 751 | "line": line_columns, 752 | "polygon": poly_columns, 753 | } 754 | ) 755 | 756 | request_body = { 757 | "fileName": self.file_name, 758 | "geometry": geom, 759 | "outputType": output_format, 760 | "geometryType": geometryType_filter, 761 | "filters": { 762 | "tags": {"all_geometry": osmTags}, 763 | "attributes": attribute_meta, 764 | }, 765 | } 766 | else: 767 | if columns: 768 | request_body = { 769 | "fileName": self.file_name, 770 | "geometry": geom, 771 | "outputType": output_format, 772 | "geometryType": geometryType_filter, 773 | "filters": { 774 | "tags": { 775 | "point": point_filter, 776 | "line": line_filter, 777 | "polygon": poly_filter, 778 | }, 779 | "attributes": {"all_geometry": columns}, 780 | }, 781 | } 782 | else: 783 | request_body = { 784 | "fileName": self.file_name, 785 | "geometry": geom, 786 | "outputType": output_format, 787 | "geometryType": geometryType_filter, 788 | "filters": { 789 | "tags": { 790 | "point": point_filter, 791 | "line": line_filter, 792 | "polygon": poly_filter, 793 | }, 794 | "attributes": { 795 | "point": point_columns, 796 | "line": line_columns, 797 | "polygon": poly_columns, 798 | }, 799 | }, 800 | } 801 | 802 | if all_feature_filter_json: 803 | if ( 804 | len( 805 | DeepDiff( 806 | request_body["filters"], 807 | all_features_filters, 808 | ignore_order=True, 809 | ) 810 | ) 811 | < 1 812 | ): # that means user is selecting all the options available on export tool 813 | request_body["filters"] = {} 814 | 815 | else: 816 | request_body = { 817 | "fileName": self.file_name, 818 | "geometry": geom, 819 | "outputType": output_format, 820 | } 821 | if output_format in ["mbtiles","pmtiles","mvt"]: 822 | request_body["minZoom"] = min_zoom 823 | request_body["maxZoom"] = max_zoom 824 | else : # use stintersects 825 | request_body["useStWithin"]= False 826 | request_body['includeUserMetadata'] = self.userinfo 827 | headers = {"accept": "application/json", "Content-Type": "application/json"} 828 | if self.access_token: 829 | headers["access-token"] = self.access_token 830 | # print(request_body) 831 | try: 832 | with requests.Session() as req_session: 833 | # print("printing before sending") 834 | # print(json.dumps(request_body)) 835 | for retry in range(MAX_RETRIES): 836 | r = req_session.post( 837 | url=f"{self.hostname}v1/snapshot/", 838 | data=json.dumps(request_body), 839 | headers=headers, 840 | timeout=60 * 5, 841 | ) 842 | 843 | if r.status_code == 429: # Rate limited 844 | print(f"Rate limited, retrying in {RETRY_DELAY} seconds...") 845 | time.sleep(RETRY_DELAY) 846 | elif r.status_code != 200: # Unexpected status code 847 | print(json.dumps(request_body)) 848 | r.raise_for_status() 849 | else: # Success 850 | break 851 | if r.ok: 852 | res = r.json() 853 | else: 854 | raise ValueError(r.content) 855 | url = f"{self.hostname}v1{res['track_link']}" 856 | success = False 857 | while not success: 858 | with requests.Session() as api: 859 | r = api.get(url) 860 | r.raise_for_status() 861 | if r.ok: 862 | res = r.json() 863 | if res.get("status") == "FAILURE": 864 | raise ValueError("Task failed from raw data api") 865 | if res.get("status") == "SUCCESS": 866 | if res.get('result'): 867 | result=format_response(res['result']) 868 | if result.get('download_url'): 869 | success = True 870 | return [result] 871 | time.sleep(1) # Check each 1 seconds 872 | 873 | except requests.exceptions.RequestException as ex: 874 | raise ex 875 | -------------------------------------------------------------------------------- /osm_export_tool/sql.py: -------------------------------------------------------------------------------- 1 | from pyparsing import Word, delimitedList, Optional, \ 2 | Group, alphas, nums, alphanums, ParseException, Forward, oneOf, quotedString, \ 3 | ZeroOrMore, Keyword 4 | 5 | class InvalidSQL(Exception): 6 | pass 7 | 8 | 9 | ident = Word( alphas, alphanums + "_:" ) 10 | columnName = (ident | quotedString())("columnName") 11 | 12 | whereExpression = Forward() 13 | and_ = Keyword("and", caseless=True)('and') 14 | or_ = Keyword("or", caseless=True)('or') 15 | in_ = Keyword("in", caseless=True)("in") 16 | isnotnull = Keyword("is not null",caseless=True)('notnull') 17 | binop = oneOf("= != < > >= <=", caseless=True)('binop') 18 | intNum = Word( nums ) 19 | 20 | columnRval = (intNum | quotedString.setParseAction(lambda x:x[0][1:-1]))('rval*') 21 | whereCondition = Group( 22 | ( columnName + isnotnull ) | 23 | ( columnName + binop + columnRval ) | 24 | ( columnName + in_ + "(" + delimitedList( columnRval ) + ")" ) | 25 | ( "(" + whereExpression + ")" ) 26 | )('condition') 27 | whereExpression << Group(whereCondition + ZeroOrMore( ( and_ | or_ ) + whereExpression ) )('expression') 28 | 29 | class SQLValidator(object): 30 | """ Parses a subset of SQL to define feature selections. 31 | This validates the SQL to make sure the user can't do anything dangerous.""" 32 | 33 | def __init__(self,s): 34 | self._s = s 35 | self._errors = [] 36 | self._parse_result = None 37 | 38 | @property 39 | def valid(self): 40 | try: 41 | self._parse_result = whereExpression.parseString(self._s,parseAll=True) 42 | except InvalidSQL as e: 43 | self._errors.append(str(e)) 44 | return False 45 | except ParseException as e: 46 | self._errors.append("SQL could not be parsed.") 47 | return False 48 | return True 49 | 50 | @property 51 | def errors(self): 52 | return self._errors 53 | 54 | @property 55 | def column_names(self): 56 | # takes a dictionary, returns a list 57 | def column_names_in_dict(d): 58 | result = [] 59 | for key, value in d.items(): 60 | if 'columnName' == key: 61 | result = result + [value] 62 | if isinstance(value,dict): 63 | result = result + column_names_in_dict(value) 64 | return result 65 | return column_names_in_dict(self._parse_result.asDict()) 66 | 67 | def strip_quotes(token): 68 | if token[0] == '"' and token[-1] == '"': 69 | token = token[1:-1] 70 | if token[0] == "'" and token[-1] == "'": 71 | token = token[1:-1] 72 | return token 73 | 74 | def _match(d,tags): 75 | if len(d) == 0: 76 | return False 77 | op = d[0] 78 | if op == 'or': 79 | return _match(d[1],tags) or _match(d[2],tags) 80 | elif op == 'and': 81 | return _match(d[1],tags) and _match(d[2],tags) 82 | elif op == '=': 83 | return d[1] in tags and tags[d[1]] == d[2] 84 | elif op == 'notnull': 85 | return d[1] in tags 86 | elif op == 'in': 87 | return (d[1] in tags) and (tags[d[1]] in d[2]) 88 | elif op == '!=': 89 | return d[1] not in tags or tags[d[1]] != d[2] 90 | elif op == '>': 91 | return d[1] in tags and str(tags[d[1]]) > str(d[2]) 92 | elif op == '<': 93 | return d[1] in tags and str(tags[d[1]]) < str(d[2]) 94 | elif op == '>=': 95 | return d[1] in tags and str(tags[d[1]]) >= str(d[2]) 96 | elif op == '<=': 97 | return d[1] in tags and str(tags[d[1]]) <= str(d[2]) 98 | raise Exception 99 | 100 | def to_prefix(sql): 101 | def prefixform(d): 102 | if 'or' in d: 103 | return ('or',prefixform(d['condition']),prefixform(d['expression'])) 104 | elif 'and' in d: 105 | return ('and',prefixform(d['condition']),prefixform(d['expression'])) 106 | elif 'condition' in d: 107 | return prefixform(d['condition']) 108 | elif 'expression' in d: 109 | return prefixform(d['expression']) 110 | elif 'binop' in d: 111 | return (d['binop'],strip_quotes(d['columnName']),d['rval'][0]) 112 | elif 'notnull' in d: 113 | return ('notnull',strip_quotes(d['columnName'])) 114 | elif 'in' in d: 115 | return ('in',strip_quotes(d['columnName']),d['rval']) 116 | return prefixform(whereExpression.parseString(sql,parseAll=True).asDict()) 117 | 118 | class Matcher: 119 | def __init__(self,expr): 120 | self.expr = expr 121 | 122 | def matches(self,tags): 123 | return _match(self.expr,tags) 124 | 125 | # returns a new matcher 126 | def union(self,other_matcher): 127 | if other_matcher.expr == (): 128 | return Matcher(self.expr) 129 | if self.expr == (): 130 | return Matcher(other_matcher.expr) 131 | return Matcher(('or',self.expr,other_matcher.expr)) 132 | 133 | @classmethod 134 | def any(cls,tag_name): 135 | return Matcher(('notnull',tag_name)) 136 | 137 | @classmethod 138 | def null(cls): 139 | return Matcher(()) 140 | 141 | @classmethod 142 | def from_sql(cls,sql): 143 | return cls(to_prefix(sql)) 144 | 145 | # only used for display and debugging 146 | def to_sql(self): 147 | def expr_to_sql(e): 148 | if e[0] == '=': 149 | return "{0} = '{1}'".format(e[1],e[2]) 150 | if e[0] == 'notnull': 151 | return "{0} IS NOT NULL".format(e[1]) 152 | if e[0] == '!=': 153 | return "{0} != '{1}'".format(e[1],e[2]) 154 | if e[0] == '>=': 155 | return "{0} >= {1}".format(e[1],e[2]) 156 | if e[0] == '<=': 157 | return "{0} <= {1}".format(e[1],e[2]) 158 | if e[0] == '>': 159 | return "{0} > {1}".format(e[1],e[2]) 160 | if e[0] == '<': 161 | return "{0} < {1}".format(e[1],e[2]) 162 | if e[0] == 'in': 163 | parts = ','.join(["'" + x + "'" for x in e[2]]) 164 | return '{0} IN ({1})'.format(e[1],parts) 165 | if e[0] == 'and': 166 | return expr_to_sql(e[1]) + ' AND ' + expr_to_sql(e[2]) 167 | if e[0] == 'or': 168 | return expr_to_sql(e[1]) + ' OR ' + expr_to_sql(e[2]) 169 | return expr_to_sql(self.expr) 170 | -------------------------------------------------------------------------------- /osm_export_tool/tabular.py: -------------------------------------------------------------------------------- 1 | from base64 import b64decode 2 | import os 3 | import re 4 | 5 | import osmium as o 6 | import osgeo.ogr as ogr 7 | import osgeo.osr as osr 8 | from shapely.wkb import loads, dumps 9 | from shapely.prepared import prep 10 | 11 | from osm_export_tool import GeomType, File 12 | 13 | fab = o.geom.WKBFactory() 14 | create_geom = lambda b : ogr.CreateGeometryFromWkb(bytes.fromhex(b)) 15 | epsg_4326 = osr.SpatialReference() 16 | epsg_4326.ImportFromEPSG(4326) 17 | 18 | CLOSED_WAY_KEYS = ['aeroway','amenity','boundary','building','building:part','craft','geological','historic','landuse','leisure','military','natural','office','place','shop','sport','tourism'] 19 | CLOSED_WAY_KEYVALS = {'highway':'platform','public_transport':'platform'} 20 | def closed_way_is_polygon(tags): 21 | for key in CLOSED_WAY_KEYS: 22 | if key in tags: 23 | return True 24 | for key, val in CLOSED_WAY_KEYVALS.items(): 25 | if key in tags and tags[key] == val: 26 | return True 27 | return False 28 | 29 | def make_filename(s): 30 | return s.lower().replace(' ','_') 31 | 32 | class Kml: 33 | class Layer: 34 | def __init__(self,driver,file_name,ogr_geom_type,theme): 35 | self.columns = theme.keys 36 | self.ds = driver.CreateDataSource(file_name + '.kml') 37 | self.ogr_layer = self.ds.CreateLayer(theme.name, epsg_4326, ogr_geom_type) 38 | 39 | if theme.osm_id: 40 | self.osm_id = True 41 | field_name = ogr.FieldDefn('osm_id', ogr.OFTInteger64) 42 | field_name.SetWidth(80) 43 | self.ogr_layer.CreateField(field_name) 44 | else: 45 | self.osm_id = False 46 | 47 | for column in self.columns: 48 | field_name = ogr.FieldDefn(column, ogr.OFTString) 49 | field_name.SetWidth(80) 50 | self.ogr_layer.CreateField(field_name) 51 | 52 | self.defn = self.ogr_layer.GetLayerDefn() 53 | 54 | def __init__(self,output_name,mapping): 55 | driver = ogr.GetDriverByName('KML') 56 | 57 | self.files = [] 58 | self.layers = {} 59 | for t in mapping.themes: 60 | name = output_name + '_' + make_filename(t.name) 61 | if t.points: 62 | self.layers[(t.name,GeomType.POINT)] = Kml.Layer(driver,name + '_points',ogr.wkbPoint,t) 63 | self.files.append(File('kml',[name + '_points.kml'],{'theme':t.name})) 64 | if t.lines: 65 | self.layers[(t.name,GeomType.LINE)] = Kml.Layer(driver,name + '_lines',ogr.wkbLineString,t) 66 | self.files.append(File('kml',[name + '_lines.kml'],{'theme':t.name})) 67 | if t.polygons: 68 | self.layers[(t.name,GeomType.POLYGON)] = Kml.Layer(driver,name + '_polygons',ogr.wkbMultiPolygon,t) 69 | self.files.append(File('kml',[name + '_polygons.kml'],{'theme':t.name})) 70 | 71 | def write(self,osm_id,layer_name,geom_type,geom,tags): 72 | layer = self.layers[(layer_name,geom_type)] 73 | feature = ogr.Feature(layer.defn) 74 | feature.SetGeometry(geom) 75 | if layer.osm_id: 76 | feature.SetField('osm_id',osm_id) 77 | for col in layer.columns: 78 | if col in tags: 79 | feature.SetField(col,tags[col]) 80 | layer.ogr_layer.CreateFeature(feature) 81 | 82 | def finalize(self): 83 | self.layers = None 84 | self.ds = None 85 | 86 | class Shapefile: 87 | class Layer: 88 | def __init__(self,driver,file_name,ogr_geom_type,theme): 89 | def launderName(col): 90 | return re.sub(r'[^a-zA-Z0-9_]', '', col)[0:10] 91 | 92 | self.columns = theme.keys 93 | self.ds = driver.CreateDataSource(file_name + '.shp') 94 | self.ogr_layer = self.ds.CreateLayer(theme.name, epsg_4326, ogr_geom_type,options=['ENCODING=UTF-8']) 95 | 96 | if theme.osm_id: 97 | self.osm_id = True 98 | field_name = ogr.FieldDefn('osm_id', ogr.OFTInteger64) 99 | field_name.SetWidth(80) 100 | self.ogr_layer.CreateField(field_name) 101 | else: 102 | self.osm_id = False 103 | 104 | self.launderedNames = {} 105 | for column in self.columns: 106 | laundered_name = launderName(column) 107 | field_name = ogr.FieldDefn(laundered_name, ogr.OFTString) 108 | field_name.SetWidth(80) 109 | self.ogr_layer.CreateField(field_name) 110 | self.launderedNames[column] = laundered_name 111 | 112 | self.defn = self.ogr_layer.GetLayerDefn() 113 | 114 | def __init__(self,output_name,mapping): 115 | driver = ogr.GetDriverByName('ESRI Shapefile') 116 | 117 | self.files = [] 118 | self.layers = {} 119 | for t in mapping.themes: 120 | name = output_name + '_' + make_filename(t.name) 121 | if t.points: 122 | self.layers[(t.name,GeomType.POINT)] = Shapefile.Layer(driver,name + '_points',ogr.wkbPoint,t) 123 | self.files.append(File.shp(name + '_points',{'theme':t.name})) 124 | if t.lines: 125 | self.layers[(t.name,GeomType.LINE)] = Shapefile.Layer(driver,name + '_lines',ogr.wkbLineString,t) 126 | self.files.append(File.shp(name + '_lines',{'theme':t.name})) 127 | if t.polygons: 128 | self.layers[(t.name,GeomType.POLYGON)] = Shapefile.Layer(driver,name + '_polygons',ogr.wkbMultiPolygon,t) 129 | self.files.append(File.shp(name + '_polygons',{'theme':t.name})) 130 | 131 | def write(self,osm_id,layer_name,geom_type,geom,tags): 132 | layer = self.layers[(layer_name,geom_type)] 133 | feature = ogr.Feature(layer.defn) 134 | feature.SetGeometry(geom) 135 | if layer.osm_id: 136 | feature.SetField('osm_id',osm_id) 137 | for col in layer.columns: 138 | if col in tags: 139 | feature.SetField(layer.launderedNames[col],tags[col]) 140 | layer.ogr_layer.CreateFeature(feature) 141 | 142 | def finalize(self): 143 | self.layers = None 144 | self.ds = None 145 | 146 | class Geopackage: 147 | class Layer: 148 | def __init__(self,ds,theme): 149 | self.ogr_layer = ds.CreateLayer(theme.name, epsg_4326, ogr.wkbUnknown,options=['SPATIAL_INDEX=NO']) 150 | 151 | if theme.osm_id: 152 | self.osm_id = True 153 | field_name = ogr.FieldDefn('osm_id', ogr.OFTInteger64) 154 | field_name.SetWidth(80) 155 | self.ogr_layer.CreateField(field_name) 156 | else: 157 | self.osm_id = False 158 | 159 | self.columns = theme.keys 160 | for column_name in self.columns: 161 | field_name = ogr.FieldDefn(column_name, ogr.OFTString) 162 | field_name.SetWidth(80) 163 | self.ogr_layer.CreateField(field_name) 164 | self.defn = self.ogr_layer.GetLayerDefn() 165 | 166 | def __init__(self,output_name,mapping): 167 | driver = ogr.GetDriverByName('GPKG') 168 | self.ds = driver.CreateDataSource(output_name + '.gpkg') 169 | self.ds.StartTransaction() 170 | 171 | self.files = [File('gpkg',[output_name + '.gpkg'])] 172 | self.layers = {} 173 | for theme in mapping.themes: 174 | layer = Geopackage.Layer(self.ds,theme) 175 | if theme.points: 176 | self.layers[(theme.name,GeomType.POINT)] = layer 177 | if theme.lines: 178 | self.layers[(theme.name,GeomType.LINE)] = layer 179 | if theme.polygons: 180 | self.layers[(theme.name,GeomType.POLYGON)] = layer 181 | 182 | def write(self,osm_id,layer_name,geom_type,geom,tags): 183 | layer = self.layers[(layer_name,geom_type)] 184 | feature = ogr.Feature(layer.defn) 185 | feature.SetGeometry(geom) 186 | if layer.osm_id: 187 | feature.SetField('osm_id',osm_id) 188 | for column_name in layer.columns: 189 | if column_name in tags: 190 | feature.SetField(column_name,tags[column_name]) 191 | layer.ogr_layer.CreateFeature(feature) 192 | 193 | def finalize(self): 194 | self.ds.CommitTransaction() 195 | self.layers = None 196 | self.ds = None 197 | 198 | # special case where each theme is a separate geopackage, for legacy reasons 199 | class MultiGeopackage: 200 | class Layer: 201 | def __init__(self,output_name,theme): 202 | driver = ogr.GetDriverByName('GPKG') 203 | self.ds = driver.CreateDataSource(output_name + '_' + make_filename(theme.name) + '.gpkg') 204 | self.ds.StartTransaction() 205 | self.ogr_layer = self.ds.CreateLayer(theme.name, epsg_4326, ogr.wkbUnknown,options=['SPATIAL_INDEX=NO']) 206 | 207 | if theme.osm_id: 208 | self.osm_id = True 209 | field_name = ogr.FieldDefn('osm_id', ogr.OFTInteger64) 210 | field_name.SetWidth(80) 211 | self.ogr_layer.CreateField(field_name) 212 | else: 213 | self.osm_id = False 214 | 215 | self.columns = theme.keys 216 | for column_name in self.columns: 217 | field_name = ogr.FieldDefn(column_name, ogr.OFTString) 218 | field_name.SetWidth(80) 219 | self.ogr_layer.CreateField(field_name) 220 | self.defn = self.ogr_layer.GetLayerDefn() 221 | 222 | def __init__(self,output_name,mapping): 223 | self.files = [] 224 | self.layers = {} 225 | for theme in mapping.themes: 226 | layer = MultiGeopackage.Layer(output_name, theme) 227 | self.files.append(File('gpkg',[output_name + '_' + make_filename(theme.name) + '.gpkg'],{'theme':theme.name})) 228 | if theme.points: 229 | self.layers[(theme.name,GeomType.POINT)] = layer 230 | if theme.lines: 231 | self.layers[(theme.name,GeomType.LINE)] = layer 232 | if theme.polygons: 233 | self.layers[(theme.name,GeomType.POLYGON)] = layer 234 | 235 | def write(self,osm_id,layer_name,geom_type,geom,tags): 236 | layer = self.layers[(layer_name,geom_type)] 237 | feature = ogr.Feature(layer.defn) 238 | feature.SetGeometry(geom) 239 | if layer.osm_id: 240 | feature.SetField('osm_id',osm_id) 241 | for column_name in layer.columns: 242 | if column_name in tags: 243 | feature.SetField(column_name,tags[column_name]) 244 | layer.ogr_layer.CreateFeature(feature) 245 | 246 | def finalize(self): 247 | for k, layer in self.layers.items(): 248 | layer.ds.CommitTransaction() 249 | self.layers = None 250 | 251 | class Handler(o.SimpleHandler): 252 | def __init__(self,outputs,mapping,clipping_geom=None, polygon_centroid=False): 253 | super(Handler, self).__init__() 254 | self.outputs = outputs 255 | self.mapping = mapping 256 | self.clipping_geom = clipping_geom 257 | self.polygon_centroid = polygon_centroid 258 | 259 | if clipping_geom: 260 | self.prepared_clipping_geom=None 261 | self.prepared_clipping_geom = prep(clipping_geom) 262 | 263 | def node(self,n): 264 | if len(n.tags) == 0: 265 | return 266 | geom = None 267 | for theme in self.mapping.themes: 268 | if theme.matches(GeomType.POINT,n.tags): 269 | if not geom: 270 | wkb = fab.create_point(n) 271 | if self.clipping_geom: 272 | sg = loads(bytes.fromhex(wkb)) 273 | if not self.prepared_clipping_geom.contains(sg): 274 | return 275 | geom = create_geom(wkb) 276 | for output in self.outputs: 277 | output.write(n.id,theme.name,GeomType.POINT,geom,n.tags) 278 | 279 | def way(self, w): 280 | if len(w.tags) == 0: 281 | return 282 | if w.is_closed() and closed_way_is_polygon(w.tags): # this will be handled in area() 283 | return 284 | try: 285 | # NOTE: it is possible this is actually a MultiLineString 286 | # in the case where a LineString is clipped by the clipping geom, 287 | # or the way is self-intersecting 288 | # but GDAL and QGIS seem to handle it OK. 289 | linestring = None 290 | for theme in self.mapping.themes: 291 | if theme.matches(GeomType.LINE,w.tags): 292 | if not linestring: 293 | wkb = fab.create_linestring(w) 294 | if self.clipping_geom: 295 | sg = loads(bytes.fromhex(wkb)) 296 | if not self.prepared_clipping_geom.intersects(sg): 297 | return 298 | if not self.prepared_clipping_geom.contains_properly(sg): 299 | sg = self.clipping_geom.intersection(sg) 300 | linestring = ogr.CreateGeometryFromWkb(dumps(sg)) 301 | else: 302 | linestring = create_geom(wkb) 303 | for output in self.outputs: 304 | output.write(w.id,theme.name,GeomType.LINE,linestring,w.tags) 305 | except RuntimeError: 306 | print("Incomplete way: {0}".format(w.id)) 307 | 308 | def area(self,a): 309 | if len(a.tags) == 0: 310 | return 311 | if not closed_way_is_polygon(a.tags): 312 | return 313 | osm_id = a.orig_id() if a.from_way() else -a.orig_id() 314 | try: 315 | geom_type = GeomType.POLYGON 316 | multipolygon = None 317 | for theme in self.mapping.themes: 318 | if theme.matches(GeomType.POLYGON,a.tags): 319 | if not multipolygon: 320 | wkb = fab.create_multipolygon(a) 321 | if self.clipping_geom: 322 | sg = loads(bytes.fromhex(wkb)) 323 | if not self.prepared_clipping_geom.intersects(sg): 324 | return 325 | if not self.prepared_clipping_geom.contains_properly(sg): 326 | sg = self.clipping_geom.intersection(sg) 327 | multipolygon = ogr.CreateGeometryFromWkb(dumps(sg)) 328 | else: 329 | multipolygon = create_geom(wkb) 330 | 331 | geom = multipolygon 332 | if self.polygon_centroid is True: 333 | geom = multipolygon.Centroid() 334 | geom_type = GeomType.POINT 335 | 336 | for output in self.outputs: 337 | output.write(osm_id,theme.name,geom_type,geom,a.tags) 338 | except RuntimeError: 339 | print('Invalid area: {0}'.format(a.orig_id())) 340 | 341 | 342 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | osmium~=2.15.2 2 | pyparsing~=2.4.0 3 | pyyaml~=5.1.1 4 | shapely~=1.6.4 5 | requests~=2.26.0 6 | landez~=2.5.0 7 | deepdiff~=5.8.1 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | requirements = [ 7 | "osmium~=3.5.0", 8 | "pyparsing~=2.4", 9 | "pyyaml", 10 | "shapely~=1.6", 11 | "requests>=2.22.0", 12 | "landez~=2.5.0", 13 | ] 14 | 15 | setuptools.setup( 16 | name="osm-export-tool-python", 17 | version="2.0.16", 18 | author="Hot Tech Team", 19 | author_email="sysadmin@hotosm.org", 20 | description="Convert OpenStreetMap data into GIS and mobile mapping file formats.", 21 | license="BSD-3-Clause", 22 | long_description=long_description, 23 | long_description_content_type="text/markdown", 24 | url="https://github.com/hotosm/osm-export-tool-python", 25 | packages=setuptools.find_packages(), 26 | classifiers=[ 27 | "Programming Language :: Python :: 3", 28 | "License :: OSI Approved :: BSD License", 29 | "Operating System :: OS Independent", 30 | ], 31 | scripts=["bin/osm-export-tool"], 32 | install_requires=requirements, 33 | requires_python=">=3.0", 34 | package_data={"osm_export_tool": ["mappings/*.yml"]}, 35 | ) 36 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hotosm/osm-export-tool-python/3fd3ab46f9d1143df45249332d63b165bf7e9ce3/test/__init__.py -------------------------------------------------------------------------------- /test/test_mapping.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from osm_export_tool.mapping import Mapping 3 | from osm_export_tool import GeomType 4 | 5 | class TestMapping(unittest.TestCase): 6 | def test_basic_mapping(self): 7 | y = ''' 8 | buildings: 9 | select: 10 | - name 11 | ''' 12 | m = Mapping(y) 13 | self.assertEqual(len(m.themes),1) 14 | theme = m.themes[0] 15 | self.assertEqual(theme.name,'buildings') 16 | self.assertTrue(theme.points) 17 | self.assertTrue(theme.lines) 18 | self.assertTrue(theme.polygons) 19 | self.assertTrue('name' in theme.keys) 20 | 21 | def test_geom_types(self): 22 | y = ''' 23 | buildings: 24 | types: 25 | - points 26 | select: 27 | - name 28 | ''' 29 | m = Mapping(y) 30 | self.assertTrue(m.themes[0].points) 31 | self.assertFalse(m.themes[0].lines) 32 | self.assertFalse(m.themes[0].polygons) 33 | y = ''' 34 | buildings: 35 | types: 36 | - lines 37 | select: 38 | - name 39 | ''' 40 | m = Mapping(y) 41 | self.assertFalse(m.themes[0].points) 42 | self.assertTrue(m.themes[0].lines) 43 | self.assertFalse(m.themes[0].polygons) 44 | y = ''' 45 | buildings: 46 | types: 47 | - polygons 48 | select: 49 | - name 50 | ''' 51 | m = Mapping(y) 52 | self.assertFalse(m.themes[0].points) 53 | self.assertFalse(m.themes[0].lines) 54 | self.assertTrue(m.themes[0].polygons) 55 | 56 | 57 | def test_key_selections(self): 58 | y = ''' 59 | buildings: 60 | types: 61 | - polygons 62 | select: 63 | - addr:housenumber 64 | ''' 65 | m = Mapping(y) 66 | self.assertTrue('addr:housenumber' in m.themes[0].keys) 67 | 68 | def test_where(self): 69 | y = ''' 70 | buildings: 71 | types: 72 | - polygons 73 | select: 74 | - addr:housenumber 75 | where: 76 | - building = 'yes' 77 | ''' 78 | m = Mapping(y) 79 | self.assertFalse(m.themes[0].matches(GeomType.POINT,{'building':'yes'})) 80 | self.assertFalse(m.themes[0].matches(GeomType.POLYGON,{'building':'no'})) 81 | self.assertTrue(m.themes[0].matches(GeomType.POLYGON,{'building':'yes'})) 82 | 83 | def test_default_matcher(self): 84 | y = ''' 85 | buildings: 86 | types: 87 | - polygons 88 | select: 89 | - addr:housenumber 90 | ''' 91 | m = Mapping(y) 92 | self.assertTrue(m.themes[0].matches(GeomType.POLYGON,{'addr:housenumber':'1234'})) 93 | 94 | def test_multiple_matchers(self): 95 | y = ''' 96 | buildings: 97 | types: 98 | - polygons 99 | select: 100 | - addr:housenumber 101 | where: 102 | - building = 'yes' 103 | - amenity = 'parking' 104 | ''' 105 | m = Mapping(y) 106 | self.assertTrue(m.themes[0].matches(GeomType.POLYGON,{'building':'yes'})) 107 | self.assertTrue(m.themes[0].matches(GeomType.POLYGON,{'amenity':'parking'})) 108 | 109 | def test_nonlist_matcher(self): 110 | y = ''' 111 | buildings: 112 | types: 113 | - polygons 114 | select: 115 | - addr:housenumber 116 | where: building = 'yes' 117 | ''' 118 | m = Mapping(y) 119 | self.assertTrue(m.themes[0].matches(GeomType.POLYGON,{'building':'yes'})) 120 | 121 | def test_gt(self): 122 | y = ''' 123 | buildings: 124 | types: 125 | - polygons 126 | select: 127 | - building 128 | where: height > 20 129 | ''' 130 | m = Mapping(y) 131 | self.assertTrue(m.themes[0].matches(GeomType.POLYGON,{'height':21})) 132 | self.assertFalse(m.themes[0].matches(GeomType.POLYGON,{'height':20})) 133 | 134 | def test_default_osm_id(self): 135 | y = ''' 136 | buildings: 137 | types: 138 | - polygons 139 | select: 140 | - building 141 | ''' 142 | m = Mapping(y) 143 | self.assertTrue(m.themes[0].osm_id) 144 | self.assertFalse('osm_id' in m.themes[0].keys) 145 | m = Mapping(y,default_osm_id=False) 146 | self.assertFalse(m.themes[0].osm_id) 147 | 148 | def test_osm_id_override(self): 149 | y = ''' 150 | buildings: 151 | types: 152 | - polygons 153 | select: 154 | - building 155 | - osm_id 156 | ''' 157 | m = Mapping(y) 158 | self.assertTrue(m.themes[0].osm_id) 159 | self.assertFalse('osm_id' in m.themes[0].keys) 160 | m = Mapping(y,default_osm_id=False) 161 | self.assertTrue(m.themes[0].osm_id) 162 | self.assertFalse('osm_id' in m.themes[0].keys) 163 | 164 | def test_duplicate_key(self): 165 | y = ''' 166 | buildings: 167 | types: 168 | - polygons 169 | select: 170 | - building 171 | - building 172 | ''' 173 | m = Mapping(y) 174 | self.assertTrue(len(m.themes[0].keys) == 1) 175 | 176 | def test_extra(self): 177 | y = ''' 178 | buildings: 179 | foo: 180 | bar: baz 181 | select: 182 | - building 183 | ''' 184 | m = Mapping(y) 185 | self.assertEqual(m.themes[0].extra,{'foo':{'bar':'baz'}}) 186 | 187 | class TestMappingValidation(unittest.TestCase): 188 | def test_empty_yaml(self): 189 | y = ''' 190 | ''' 191 | m, errors = Mapping.validate(y) 192 | self.assertTrue(m is None) 193 | self.assertTrue(len(errors) == 1) 194 | 195 | def test_bad_yaml(self): 196 | y = ''' 197 | buildings 198 | types: 199 | - polygons 200 | select: 201 | - building 202 | ''' 203 | m, errors = Mapping.validate(y) 204 | self.assertTrue(m is None) 205 | self.assertTrue(len(errors) == 1) 206 | 207 | def test_no_select(self): 208 | y = ''' 209 | buildings: 210 | types: 211 | - polygons 212 | ''' 213 | m, errors = Mapping.validate(y) 214 | self.assertTrue(m is None) 215 | self.assertTrue(len(errors) == 1) 216 | 217 | def test_invalid_type(self): 218 | y = ''' 219 | buildings: 220 | types: 221 | - polygon 222 | select: 223 | - building 224 | ''' 225 | m, errors = Mapping.validate(y) 226 | self.assertTrue(m is None) 227 | self.assertTrue(len(errors) == 1) 228 | 229 | def test_empty_sql(self): 230 | y = ''' 231 | buildings: 232 | types: 233 | - polygons 234 | select: 235 | - building 236 | where: 237 | ''' 238 | m, errors = Mapping.validate(y) 239 | self.assertTrue(m is None) 240 | self.assertTrue(len(errors) == 1) 241 | 242 | def test_invalid_sql(self): 243 | y = ''' 244 | buildings: 245 | types: 246 | - polygons 247 | select: 248 | - building 249 | where: XXX aaa 250 | ''' 251 | m, errors = Mapping.validate(y) 252 | self.assertTrue(m is None) 253 | self.assertTrue(len(errors) == 1) 254 | 255 | def test_wrong_yaml_list(self): 256 | y = ''' 257 | buildings: 258 | types: polygons 259 | select: 260 | - building 261 | ''' 262 | m, errors = Mapping.validate(y) 263 | self.assertTrue(m is None) 264 | self.assertTrue(len(errors) == 1) 265 | 266 | def test_invalid_yaml_parse(self): 267 | y = ''' 268 | buildings: 269 | - types: 270 | - polygons 271 | select: 272 | - building 273 | ''' 274 | m, errors = Mapping.validate(y) 275 | self.assertTrue(m is None) 276 | self.assertTrue(len(errors) == 1) 277 | 278 | def test_wrong_yaml_parse(self): 279 | y = ''' 280 | buildings: 281 | - types: 282 | - polygons 283 | - select: 284 | - building 285 | ''' 286 | m, errors = Mapping.validate(y) 287 | self.assertTrue(m is None) 288 | self.assertTrue(len(errors) == 1) 289 | self.assertTrue('must be YAML dict' in errors[0]) 290 | 291 | def test_wrong_toplevel_themes(self): 292 | y = ''' 293 | - buildings: 294 | types: 295 | - polygons 296 | select: 297 | - building 298 | ''' 299 | m, errors = Mapping.validate(y) 300 | self.assertTrue(m is None) 301 | self.assertTrue(len(errors) == 1) 302 | -------------------------------------------------------------------------------- /test/test_sources.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from osm_export_tool.sources import Overpass 3 | from osm_export_tool.mapping import Mapping 4 | 5 | class TestMappingToOverpass(unittest.TestCase): 6 | def test_mapping(self): 7 | y = ''' 8 | buildings: 9 | types: 10 | - points 11 | select: 12 | - column1 13 | where: column2 IS NOT NULL 14 | 15 | other1: 16 | types: 17 | - points 18 | - polygons 19 | select: 20 | - column1 21 | - irrelevant 22 | where: column2 IS NOT NULL AND column3 IN ('foo','bar') 23 | 24 | other2: 25 | types: 26 | - lines 27 | select: 28 | - column5:key 29 | ''' 30 | mapping = Mapping(y) 31 | nodes, ways, relations = Overpass.filters(mapping) 32 | self.assertCountEqual(nodes,["['column3'~'foo|bar']","['column2']"]) 33 | # force quoting of strings to handle keys with colons 34 | self.assertCountEqual(ways,["['column5:key']","['column3'~'foo|bar']","['column2']"]) 35 | self.assertCountEqual(relations,["['column3'~'foo|bar']","['column2']"]) 36 | 37 | class TestSQLToOverpass(unittest.TestCase): 38 | def test_basic(self): 39 | s = Overpass.sql("name = 'somename'") 40 | self.assertEqual(s,["['name'='somename']"]) 41 | s = Overpass.sql("level > 4") 42 | self.assertEqual(s,["['level']"]) 43 | 44 | def test_basic_list(self): 45 | s = Overpass.sql("name IN ('val1','val2')") 46 | self.assertEqual(s,["['name'~'val1|val2']"]) 47 | 48 | def test_whitespace(self): 49 | s = Overpass.sql("name = 'some value'") 50 | self.assertEqual(s,["['name'='some value']"]) 51 | 52 | def test_notnull(self): 53 | s = Overpass.sql("name is not null") 54 | self.assertEqual(s,["['name']"]) 55 | 56 | def test_and_or(self): 57 | s = Overpass.sql("name1 = 'foo' or name2 = 'bar'") 58 | self.assertEqual(s,["['name1'='foo']","['name2'='bar']"]) 59 | s = Overpass.sql("(name1 = 'foo' and name2 = 'bar') or name3 = 'baz'") 60 | self.assertEqual(s,["['name1'='foo']","['name2'='bar']","['name3'='baz']"]) -------------------------------------------------------------------------------- /test/test_sql.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from osm_export_tool.sql import SQLValidator, Matcher 3 | 4 | class TestSql(unittest.TestCase): 5 | 6 | def test_basic(self): 7 | s = SQLValidator("name = 'a name'") 8 | self.assertTrue(s.valid) 9 | 10 | def test_identifier_list(self): 11 | s = SQLValidator("natural in ('water','cliff')") 12 | self.assertTrue(s.valid) 13 | 14 | def test_float_value(self): 15 | s = SQLValidator("height > 20") 16 | self.assertTrue(s.valid) 17 | 18 | def test_not_null(self): 19 | s = SQLValidator("height IS NOT NULL") 20 | self.assertTrue(s.valid) 21 | 22 | def test_and_or(self): 23 | s = SQLValidator("height IS NOT NULL and height > 20") 24 | self.assertTrue(s.valid) 25 | s = SQLValidator("height IS NOT NULL or height > 20") 26 | self.assertTrue(s.valid) 27 | s = SQLValidator("height IS NOT NULL or height > 20 and height < 30") 28 | self.assertTrue(s.valid) 29 | 30 | def test_parens(self): 31 | s = SQLValidator("(admin IS NOT NULL and level > 4)") 32 | self.assertTrue(s.valid) 33 | s = SQLValidator("(admin IS NOT NULL and level > 4) AND height is not null") 34 | self.assertTrue(s.valid) 35 | 36 | def test_colons_etc(self): 37 | s = SQLValidator("addr:housenumber IS NOT NULL") 38 | self.assertTrue(s.valid) 39 | s = SQLValidator("admin_level IS NOT NULL") 40 | self.assertTrue(s.valid) 41 | 42 | def test_invalid_sql(self): 43 | s = SQLValidator("drop table planet_osm_polygon") 44 | self.assertFalse(s.valid) 45 | self.assertEqual(s.errors,['SQL could not be parsed.']) 46 | s = SQLValidator("(drop table planet_osm_polygon)") 47 | self.assertFalse(s.valid) 48 | self.assertEqual(s.errors,['SQL could not be parsed.']) 49 | s = SQLValidator ("") 50 | self.assertFalse(s.valid) 51 | self.assertEqual(s.errors,['SQL could not be parsed.']) 52 | s = SQLValidator("name = 'a name'; blah") 53 | self.assertFalse(s.valid) 54 | self.assertEqual(s.errors,['SQL could not be parsed.']) 55 | 56 | def test_column_names(self): 57 | s = SQLValidator("(admin IS NOT NULL and level > 4) AND height is not null") 58 | self.assertTrue(s.valid) 59 | self.assertEqual(s.column_names,['admin','level','height']) 60 | 61 | class TestMatcher(unittest.TestCase): 62 | def test_matcher_binop(self): 63 | m = Matcher.from_sql("building = 'yes'") 64 | self.assertTrue(m.matches({'building':'yes'})) 65 | self.assertFalse(m.matches({'building':'no'})) 66 | 67 | m = Matcher.from_sql("building != 'yes'") 68 | self.assertFalse(m.matches({'building':'yes'})) 69 | self.assertTrue(m.matches({'building':'no'})) 70 | 71 | def test_matcher_colon(self): 72 | m = Matcher.from_sql("addr:housenumber = 1") 73 | self.assertTrue(m.matches({'addr:housenumber':'1'})) 74 | 75 | m = Matcher.from_sql("building != 'yes'") 76 | self.assertFalse(m.matches({'building':'yes'})) 77 | self.assertTrue(m.matches({'building':'no'})) 78 | 79 | def test_matcher_doublequote(self): 80 | m = Matcher.from_sql("\"addr:housenumber\" = 1") 81 | self.assertTrue(m.matches({'addr:housenumber':'1'})) 82 | 83 | m = Matcher.from_sql("\"addr:housenumber\" IN ('foo')") 84 | self.assertTrue(m.matches({'addr:housenumber':'foo'})) 85 | 86 | m = Matcher.from_sql("\"addr:housenumber\" IS NOT NULL") 87 | self.assertTrue(m.matches({'addr:housenumber':'foo'})) 88 | 89 | def test_matcher_or(self): 90 | m = Matcher.from_sql("building = 'yes' OR amenity = 'bank'") 91 | self.assertTrue(m.matches({'building':'yes'})) 92 | self.assertTrue(m.matches({'amenity':'bank'})) 93 | self.assertFalse(m.matches({})) 94 | 95 | def test_matcher_and(self): 96 | m = Matcher.from_sql("building = 'yes' AND amenity = 'bank'") 97 | self.assertFalse(m.matches({'building':'yes'})) 98 | self.assertFalse(m.matches({'amenity':'bank'})) 99 | 100 | def test_matcher_is_not_null(self): 101 | m = Matcher.from_sql("building IS NOT NULL") 102 | self.assertTrue(m.matches({'building':'one'})) 103 | self.assertTrue(m.matches({'building':'two'})) 104 | self.assertFalse(m.matches({})) 105 | 106 | def test_in(self): 107 | m = Matcher.from_sql("building IN ('one','two')") 108 | self.assertTrue(m.matches({'building':'one'})) 109 | self.assertTrue(m.matches({'building':'two'})) 110 | self.assertFalse(m.matches({})) 111 | self.assertFalse(m.matches({'building':'three'})) 112 | 113 | def test_any(self): 114 | m = Matcher.any("building"); 115 | self.assertTrue(m.matches({'building':'one'})) 116 | 117 | def test_union(self): 118 | m = Matcher.any("building").union(Matcher.any("parking")) 119 | self.assertTrue(m.matches({'building':'one'})) 120 | self.assertTrue(m.matches({'parking':'one'})) 121 | 122 | def test_null(self): 123 | m = Matcher.null() 124 | self.assertFalse(m.matches({'building':'one'})) 125 | 126 | def test_to_sql(self): 127 | sql = "building = 'yes'" 128 | self.assertEqual(Matcher.from_sql(sql).to_sql(),sql) 129 | sql = "building IS NOT NULL" 130 | self.assertEqual(Matcher.from_sql(sql).to_sql(),sql) 131 | sql = "building IN ('one','two')" 132 | self.assertEqual(Matcher.from_sql(sql).to_sql(),sql) 133 | sql = "building != 'yes'" 134 | self.assertEqual(Matcher.from_sql(sql).to_sql(),sql) 135 | sql = "building >= 0" 136 | self.assertEqual(Matcher.from_sql(sql).to_sql(),sql) 137 | sql = "building <= 0" 138 | self.assertEqual(Matcher.from_sql(sql).to_sql(),sql) 139 | sql = "building > 0" 140 | self.assertEqual(Matcher.from_sql(sql).to_sql(),sql) 141 | sql = "building < 0" 142 | self.assertEqual(Matcher.from_sql(sql).to_sql(),sql) 143 | sql = "building > 0 AND building < 5" 144 | self.assertEqual(Matcher.from_sql(sql).to_sql(),sql) 145 | sql = "building > 0 OR building < 5" 146 | self.assertEqual(Matcher.from_sql(sql).to_sql(),sql) 147 | --------------------------------------------------------------------------------