├── LICENSE.md ├── README.md ├── process-boundaries.js └── run.py /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c), MapBox 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | - Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | - Redistributions in binary form must reproduce the above copyright notice, this 10 | list of conditions and the following disclaimer in the documentation and/or 11 | other materials provided with the distribution. 12 | - Neither the name "Development Seed" nor the names of its contributors may be 13 | used to endorse or promote products derived from this software without 14 | specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 20 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 23 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MapBox OSM Boundaries 2 | 3 | This program will process and import boundary relations from a OSM PBF file. The process is optimized for rendering - instead of building (multi)polygons, boundary relation way members are imported as unconnected linestrings allowing different segments of a single boundary to be styled independently. This also avoids overlapping lines where different boundaries and admin levels meet, and allows renderers like Mapnik to draw dashed lines correctly. 4 | 5 | ### Data processing 6 | 7 | The data is manipulated and simplified for easier styling: 8 | 9 | - The lowest `admin_level` value of any of a way's parent relations will be the `admin_level` value of the resulting linestring. 10 | - Various tag combinations are checked to see if each way is a maritime boundary. This information is simplified to a single `maritime` field with a value of either `0` (false) or `1` (true). 11 | - Various tag combinations are checked to see if each way is a disputed boundary. This information is simplified to a single `disputed` field with a value of either `0` (false) or `1` (true). 12 | - Boundaries that are also coastlines (`natural=coastline`) are not imported. 13 | - Boundaries that are closure segments (`closure_segment=yes`) are not imported. (Closure segments are ways added at the limits of the projection to close boundaries for valid multipolygon building. They are not actual borders.) 14 | - Geometries are imported to a Spherical Mercator projection (900913). 15 | 16 | ### Known issues 17 | 18 | - boundaries that are not part of any `boundary=administrative` relation are ignored. 19 | 20 | ## Dependencies 21 | 22 | - Python & [Psycopg2](http://initd.org/psycopg/docs/) in a Unixy environment 23 | - [Osmosis](http://wiki.openstreetmap.org/wiki/Osmosis) (requires version >= __0.42__ for planet files newer than Feb 9 2013) 24 | - [PostgreSQL](http://postgresql.org) (tested with 9.2) 25 | - [PostGIS](http://postgis.refractions.net) (tested with 2.0) 26 | - [Osmium](http://github.com/joto/osmium/) - make sure `osmjs` is compiled and in your PATH 27 | 28 | ## Running 29 | 30 | 1. Make sure you have a PostgreSQL database set up with PostGIS enabled. 31 | 2. Run `run.py -f 2 -t 4 data.osm.pbf` with appropriate options set for your database and desired admin levels. See `run.py --help` for available options. 32 | 33 | The process will take quite some time and require lots of free disk space for temporary storage. Processing a full planet file might take over six hours and require at least 60 GB of free disk space. 34 | -------------------------------------------------------------------------------- /process-boundaries.js: -------------------------------------------------------------------------------- 1 | // Usage: 2 | // osmjs -l sparsetable -j process-boundaries.js boundaries.osm | psql 3 | 4 | var ways_table = 'carto_boundary'; 5 | 6 | is_maritime = function(tags) { 7 | if (tags['maritime']) { 8 | return 1 9 | } 10 | var maritime_tags = [ 11 | 'boundary_type', 12 | 'border_type', 13 | 'boundary' 14 | ]; 15 | var maritime_vals = [ 16 | 'eez', 17 | 'maritime', 18 | 'territorial_waters', 19 | 'territorial waters' 20 | ]; 21 | for (i = 0; i < maritime_tags.length; i++) { 22 | if (maritime_vals.indexOf(tags[maritime_tags[i]]) >= 0) { 23 | return 1; 24 | } 25 | } 26 | return 0; 27 | } 28 | 29 | is_disputed = function(tags) { 30 | if (tags['disputed'] || tags['dispute'] 31 | || tags['border_status'] === 'dispute') { 32 | return 1 33 | } 34 | return 0 35 | } 36 | 37 | Osmium.Callbacks.way = function() { 38 | // This will import all ways in the OSM file except coastlines. We assume 39 | // that we are only looking at ways that belong in boundary relations. 40 | 41 | // ignore coastlines & closure segments 42 | if (this.tags['natural'] == 'coastline' 43 | || this.tags['closure_segment']) { 44 | return; 45 | } 46 | 47 | var geometry = this.geom.toHexWKB(true); 48 | // Catch failed geometries, skip them 49 | if (geometry == undefined) { 50 | return; 51 | } else { 52 | geometry = ['st_transform(\'', geometry, '\'::geometry, 900913)'].join(''); 53 | } 54 | 55 | print(['INSERT INTO ', ways_table, ' (osm_id, maritime, disputed, geom) ', 56 | 'VALUES (', this.id, ', ', is_maritime(this.tags), ', ', 57 | is_disputed(this.tags), ', ', geometry, ');'].join('')); 58 | } 59 | 60 | Osmium.Callbacks.relation = function() { 61 | // For each relation we check a few key tags, then make any updates to 62 | // its way members as necessary. 63 | 64 | var rel_id = this.id, 65 | way_ids = [], 66 | admin_level 67 | 68 | try { 69 | admin_level = parseInt(this.tags['admin_level']); 70 | } catch(e) {} 71 | 72 | 73 | for (var i=0; i < this.members.length; i++) { 74 | // build a list of way members for processing 75 | if (this.members[i].type = 'w') { 76 | way_ids.push(this.members[i].ref); 77 | } 78 | } 79 | 80 | if (way_ids.length === 0) { 81 | // relation has no way members, no updates needed 82 | return; 83 | } 84 | 85 | way_ids = way_ids.join(', '); 86 | 87 | if (typeof admin_level === 'number') { 88 | print(['UPDATE', ways_table, 'SET admin_level =', admin_level, 89 | 'WHERE osm_id in (', way_ids, ') AND (admin_level >', 90 | admin_level, 'OR admin_level IS NULL);'].join(' ')); 91 | } 92 | 93 | if (is_maritime(this.tags)) { 94 | print(['UPDATE', ways_table, 'SET maritime = 1 WHERE osm_id in (', 95 | way_ids, ');'].join(' ')); 96 | } 97 | 98 | if (is_disputed(this.tags)) { 99 | print(['UPDATE', ways_table, 'SET disputed = 1 WHERE osm_id in (', 100 | way_ids, ');'].join(' ')); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | import os.path 6 | import psycopg2 7 | import subprocess 8 | 9 | 10 | ## Arguments & help 11 | 12 | ap = argparse.ArgumentParser(description='Process OSM administrative ' + 13 | 'boundaries as individual ways.') 14 | ap.add_argument('-d', dest='db_name', default='osm', 15 | help='PostgreSQL database.') 16 | ap.add_argument('-U', dest='db_user', default='postgres', 17 | help='PostgreSQL user name.') 18 | ap.add_argument('-H', dest='db_host', default='localhost', 19 | help='PostgreSQL host.') 20 | ap.add_argument('-p', dest='db_port', default='5432', 21 | help='PostgreSQL port.') 22 | ap.add_argument('-f', dest='min_admin_level', type=int, default=2, 23 | help='Minimum admin_level to retrieve.') 24 | ap.add_argument('-t', dest='max_admin_level', type=int, default=4, 25 | help='Maximum admin_level to retrieve.') 26 | ap.add_argument(dest='osm_input', metavar='planet.osm.pbf', 27 | help='An OpenStreetMap PBF file to process.') 28 | args = ap.parse_args() 29 | 30 | 31 | ## PostgreSQL setup 32 | 33 | # Set up the db connection 34 | con = psycopg2.connect("dbname={0} user={1} host={2} port={3}".format( 35 | args.db_name, args.db_user, args.db_host, args.db_port)) 36 | cur = con.cursor() 37 | 38 | # Set up PostgeSQL table 39 | boundary_table = 'carto_boundary' 40 | 41 | cur.execute(''' 42 | create table if not exists {0} ( 43 | osm_id bigint primary key, 44 | admin_level smallint, 45 | maritime smallint, 46 | disputed smallint, 47 | geom geometry(Geometry,900913), 48 | geom_gen1 geometry(Geometry,900913), 49 | geom_gen0 geometry(Geometry,900913) 50 | );'''.format(boundary_table)) 51 | con.commit() 52 | 53 | 54 | ## Process & import the boundaries with osmjs 55 | 56 | if args.min_admin_level == args.max_admin_level: 57 | admin_levels = args.min_admin_level; 58 | outfile = 'osm_admin_{0}.osm.pbf'.format(admin_levels) 59 | elif args.min_admin_level < args.max_admin_level: 60 | admin_levels = ','.join(str(i) for i in range( 61 | args.min_admin_level, args.max_admin_level + 1)) 62 | outfile = 'osm_admin_{0}-{1}.osm.pbf'.format( 63 | args.min_admin_level, args.max_admin_level) 64 | else: 65 | print('Error: max admin level cannot be be less than min admin level') 66 | exit(1) 67 | 68 | if os.path.exists(outfile): 69 | print('Found existing file {0}; skipping filtering.'.format(outfile)) 70 | else: 71 | subprocess.call(['''osmosis \ 72 | --read-pbf {0} \ 73 | --tf accept-relations admin_level={1} \ 74 | --tf accept-relations boundary=administrative \ 75 | --used-way \ 76 | --used-node \ 77 | --write-pbf {2}'''.format( 78 | args.osm_input, 79 | admin_levels, 80 | outfile)], 81 | shell=True) 82 | 83 | subprocess.call(['osmjs -l sparsetable -r -j process-boundaries.js {0} | psql -h {1} -p {2} -U {3} -d {4} > /dev/null'.format( 84 | outfile, 85 | args.db_host, 86 | args.db_port, 87 | args.db_user, 88 | args.db_name)], 89 | shell=True) 90 | 91 | 92 | ## Create simplified geometries 93 | 94 | cur.execute('update {0} set geom_gen1 = st_simplify(geom, 200);'.format( 95 | boundary_table)) 96 | con.commit() 97 | 98 | cur.execute('update {0} set geom_gen0 = st_simplify(geom, 1000);'.format( 99 | boundary_table)) 100 | con.commit() 101 | 102 | cur.close() 103 | con.close() 104 | --------------------------------------------------------------------------------