├── examples ├── gnisNamesOnWater │ ├── README.md │ ├── gnisschema.sql │ ├── fetchosmdata.py │ ├── fetchexternaldata.py │ ├── postgisloaddata.py │ └── conflate.py ├── massMissingBuildings │ ├── README.md │ ├── postgisloaddata.py │ ├── go.py │ ├── fetchosmdata.py │ ├── fetchexternaldata.py │ ├── stage.py │ └── conflate.py ├── removeAreaFromBuilding │ ├── README.md │ ├── go.py │ ├── fetchosmdata.py │ ├── postgisloaddata.py │ └── conflate.py ├── massMissingPonds │ ├── ogrtranslation.py │ ├── README.md │ ├── go.py │ ├── fetchosmdata.py │ ├── fetchexternaldata.py │ ├── postgisloaddata.py │ └── conflate.py └── palisIdNamesOnWater │ ├── README.md │ ├── fetchosmdata.py │ ├── fetchexternaldata.py │ ├── postgisloaddata.py │ └── conflate.py ├── .gitignore ├── prep ├── findlandusereservoir.py ├── findschoolswithnobuildings.py ├── findaerowwaywithnobuildings.py └── findoverlappingbuildings.py ├── upload ├── osm2osc.py ├── osmsplit.py └── osmupload.py └── README.md /examples/gnisNamesOnWater/README.md: -------------------------------------------------------------------------------- 1 | 2 | Attempt to name lakes/ponds without names using gnis data. Still a work in progress. 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # files/directories that git should ignore when running git status 2 | ogr2osm/* 3 | osmdata/* 4 | externaldata/* 5 | temp/ 6 | stage/* 7 | upload/* 8 | *~ 9 | 10 | 11 | -------------------------------------------------------------------------------- /examples/massMissingBuildings/README.md: -------------------------------------------------------------------------------- 1 | MassGIS 2013 Building Import 2 | ======================= 3 | These scripts are rewritten from MassGIS 2013 import. 4 | 5 | http://wiki.openstreetmap.org/wiki/MassGIS_Buildings_Import 6 | 7 | - still a work in progres.... 8 | -------------------------------------------------------------------------------- /examples/removeAreaFromBuilding/README.md: -------------------------------------------------------------------------------- 1 | 2 | Wiki is here [http://wiki.openstreetmap.org/wiki/MassGIS_Import_Tag_Cleanup_2013], this is step 1. 3 | 4 | ======================= 5 | 6 | - Remove area=yes, the building=* does not need an area tag. 7 | - remove source tag with broken url. MassGIS Buildings (http://www.mass.gov/mgis/lidarbuildingfp2d.htm). Will move it to the changeset. 8 | 9 | 10 | -------------------------------------------------------------------------------- /examples/massMissingPonds/ogrtranslation.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A translation function for ogr2osm, All it does is remove empty tags, can't 3 | seem to figure out how to do that from the ogr2shp script. 4 | ''' 5 | 6 | def filterTags(attrs): 7 | if not attrs: 8 | return 9 | tags = {} 10 | 11 | for tag in attrs : 12 | if attrs[tag] != '' : 13 | tags[tag] = attrs[tag] 14 | 15 | return tags 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /examples/massMissingBuildings/postgisloaddata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import os 6 | import sys 7 | 8 | outputFile = "osmdata/massachusetts-latest.osm.bz2" 9 | 10 | # load osm file into postGIS 11 | if ( os.system("osm2pgsql " + outputFile ) ) : 12 | print("Error loading " + outputFile + " into postGIS gis database.") 13 | sys.exit(1) 14 | 15 | print("Success!!") 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /examples/massMissingPonds/README.md: -------------------------------------------------------------------------------- 1 | MassGIS Lake and Pond Import 2 | ======================= 3 | 4 | The import wiki is [here][http://wiki.openstreetmap.org/wiki/MassGIS_DEP_Wetlands_Import_2013]. It can be used as a template and to give more context around this code. 5 | 6 | It is designed to import the remaining ponds and lakes into MA, using the MassGIS as its data source. It makes efforts to skip ponds and lakes that are already imported. It is looks up the name from the PALIS_ID. Everything up to the staging step is completed. 7 | 8 | -------------------------------------------------------------------------------- /examples/gnisNamesOnWater/gnisschema.sql: -------------------------------------------------------------------------------- 1 | -- Jason Remillard - This file is in the public domain. 2 | 3 | -- create schema for gnis features, we only care about the id, name, type, and lat/lon. 4 | -- The rest of the data is not imported to save time/space. 5 | 6 | DROP TABLE IF EXISTS gnis; 7 | 8 | CREATE TABLE gnis ( 9 | featureid int, 10 | name varchar(128), 11 | featureclass varchar(16) 12 | ); 13 | 14 | SELECT AddGeometryColumn( '', 'gnis', 'geom', 4326, 'POINT', 2); 15 | 16 | CREATE INDEX idx_gnis_geom ON gnis USING gist (geom); 17 | -------------------------------------------------------------------------------- /examples/massMissingPonds/go.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | # Script to run the entire export data processing workflow. 5 | 6 | import os 7 | import sys 8 | 9 | if ( os.system("./fetchosmdata.py.py") ) : 10 | sys.exit(1) 11 | 12 | if ( os.system("./fetchexternaldata.py") ) : 13 | sys.exit(1) 14 | 15 | if ( os.system("./postgisloaddata.py") ) : 16 | sys.exit(1) 17 | 18 | #if ( os.system("./conflate.py")) : 19 | # os.exit(1) 20 | 21 | #if ( os.system("./stage.py")) : 22 | # os.exit(1) 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /examples/massMissingBuildings/go.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | # Script to run the entire export data processing workflow. 5 | 6 | import os 7 | import sys 8 | 9 | if ( os.system("./fetchosmdata.py.py") ) : 10 | sys.exit(1) 11 | 12 | if ( os.system("./fetchexternaldata.py") ) : 13 | sys.exit(1) 14 | 15 | if ( os.system("./postgisloaddata.py") ) : 16 | sys.exit(1) 17 | 18 | #if ( os.system("./conflate.py")) : 19 | # os.exit(1) 20 | 21 | #if ( os.system("./stage.py")) : 22 | # os.exit(1) 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /examples/removeAreaFromBuilding/go.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | # Script to run the entire export data processing workflow. 5 | 6 | import os 7 | import sys 8 | 9 | if ( os.system("./fetchosmdata.py.py") ) : 10 | sys.exit(1) 11 | 12 | if ( os.system("./fetchexternaldata.py") ) : 13 | sys.exit(1) 14 | 15 | if ( os.system("./postgisloaddata.py") ) : 16 | sys.exit(1) 17 | 18 | #if ( os.system("./conflate.py")) : 19 | # os.exit(1) 20 | 21 | #if ( os.system("./stage.py")) : 22 | # os.exit(1) 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /examples/palisIdNamesOnWater/README.md: -------------------------------------------------------------------------------- 1 | MassGIS Import Tag Cleanup 2013 - Step 2 2 | =========================================================== 3 | 4 | This is step two in the MassGIS tag cleanup wiki. 5 | 6 | http://wiki.openstreetmap.org/wiki/MassGIS_Import_Tag_Cleanup_2013 7 | 8 | In 2010 many lakes, ponds, and wetlands around Boston and Cape Cod 9 | were imported with all of the shape file attributes. This is an 10 | automated edit script that deletes them from the natural=water way 11 | and looks up the name from PALIS_ID attribute. 12 | 13 | See the wiki for more info. 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /examples/gnisNamesOnWater/fetchosmdata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | # script to download current MA OSM data. 5 | 6 | import os 7 | import sys 8 | 9 | outputDir = "osmdata" 10 | 11 | outputFile = outputDir + "/massachusetts-latest.osm.bz2" 12 | srcURL = "http://download.geofabrik.de/north-america/us/massachusetts-latest.osm.bz2" 13 | 14 | # make sure our output dirs are setup, and any existing file is clean. 15 | if ( os.path.isdir(outputDir) == False) : 16 | os.mkdir(outputDir) 17 | 18 | if ( os.path.isfile(outputFile)) : 19 | os.remove(outputFile) 20 | 21 | # download MA OSM file extract. 22 | if ( os.system("wget -O " + outputFile + " " + srcURL ) ) : 23 | print("Error: Can't download MA OSM extract from " + srcURL) 24 | sys.exit(1) 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /examples/massMissingPonds/fetchosmdata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | # script to download current MA OSM data. 5 | 6 | import os 7 | import sys 8 | 9 | outputDir = "osmdata" 10 | 11 | outputFile = outputDir + "/massachusetts-latest.osm.bz2" 12 | srcURL = "http://download.geofabrik.de/north-america/us/massachusetts-latest.osm.bz2" 13 | 14 | # make sure our output dirs are setup, and any existing file is clean. 15 | if ( os.path.isdir(outputDir) == False) : 16 | os.mkdir(outputDir) 17 | 18 | if ( os.path.isfile(outputFile)) : 19 | os.remove(outputFile) 20 | 21 | # download MA OSM file extract. 22 | if ( os.system("wget -O " + outputFile + " " + srcURL ) ) : 23 | print("Error: Can't download MA OSM extract from " + srcURL) 24 | sys.exit(1) 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /examples/massMissingBuildings/fetchosmdata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | # script to download current MA OSM data. 5 | 6 | import os 7 | import sys 8 | 9 | outputDir = "osmdata" 10 | 11 | outputFile = outputDir + "/massachusetts-latest.osm.bz2" 12 | srcURL = "http://download.geofabrik.de/north-america/us/massachusetts-latest.osm.bz2" 13 | 14 | # make sure our output dirs are setup, and any existing file is clean. 15 | if ( os.path.isdir(outputDir) == False) : 16 | os.mkdir(outputDir) 17 | 18 | if ( os.path.isfile(outputFile)) : 19 | os.remove(outputFile) 20 | 21 | # download MA OSM file extract. 22 | if ( os.system("wget -O " + outputFile + " " + srcURL ) ) : 23 | print("Error: Can't download MA OSM extract from " + srcURL) 24 | sys.exit(1) 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /examples/palisIdNamesOnWater/fetchosmdata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | # script to download current MA OSM data. 5 | 6 | import os 7 | import sys 8 | 9 | outputDir = "osmdata" 10 | 11 | outputFile = outputDir + "/massachusetts-latest.osm.bz2" 12 | srcURL = "http://download.geofabrik.de/north-america/us/massachusetts-latest.osm.bz2" 13 | 14 | # make sure our output dirs are setup, and any existing file is clean. 15 | if ( os.path.isdir(outputDir) == False) : 16 | os.mkdir(outputDir) 17 | 18 | if ( os.path.isfile(outputFile)) : 19 | os.remove(outputFile) 20 | 21 | # download MA OSM file extract. 22 | if ( os.system("wget -O " + outputFile + " " + srcURL ) ) : 23 | print("Error: Can't download MA OSM extract from " + srcURL) 24 | sys.exit(1) 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /examples/removeAreaFromBuilding/fetchosmdata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | # script to download current MA OSM data. 5 | 6 | import os 7 | import sys 8 | 9 | outputDir = "osmdata" 10 | 11 | outputFile = outputDir + "/massachusetts-latest.osm.bz2" 12 | srcURL = "http://download.geofabrik.de/north-america/us/massachusetts-latest.osm.bz2" 13 | 14 | # make sure our output dirs are setup, and any existing file is clean. 15 | if ( os.path.isdir(outputDir) == False) : 16 | os.mkdir(outputDir) 17 | 18 | if ( os.path.isfile(outputFile)) : 19 | os.remove(outputFile) 20 | 21 | # download MA OSM file extract. 22 | if ( os.system("wget -O " + outputFile + " " + srcURL ) ) : 23 | print("Error: Can't download MA OSM extract from " + srcURL) 24 | sys.exit(1) 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /examples/palisIdNamesOnWater/fetchexternaldata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | # This script will download the PALIS_ID/palis-saris-lookups file with the 6 | # names in it. 7 | 8 | import os 9 | import sys 10 | import urllib 11 | import zipfile 12 | 13 | outputDir = "externaldata" 14 | 15 | # make sure our output dirs are setup, and any existing file is clean. 16 | if ( os.path.isdir(outputDir) == False) : 17 | os.mkdir(outputDir) 18 | 19 | # The "MassGIS Data - MassDEP 2010 Integrated List of Waters (305(b)/303(d))" download. 20 | outputFile = outputDir + "/wbs2010_shp.zip" 21 | srcURL = "http://wsgw.mass.gov/data/gispub/shape/state/wbs2010_shp.zip" 22 | 23 | if ( os.path.isfile(outputFile)) : 24 | os.remove(outputFile) 25 | 26 | # download MassGIS water and unzip. 27 | if ( os.system("wget -O " + outputFile + " " + srcURL ) ) : 28 | print("Error: Can't download " + srcURL) 29 | sys.exit(1) 30 | 31 | # unzip 32 | z=zipfile.ZipFile(outputFile,"r") 33 | zl=z.namelist() 34 | z.extractall(outputDir) 35 | z.close() 36 | os.remove(outputFile) 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /examples/removeAreaFromBuilding/postgisloaddata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import os 6 | import sys 7 | 8 | outputFile = "osmdata/massachusetts-latest.osm.bz2" 9 | 10 | if ( os.path.isdir("temp") == False) : 11 | os.mkdir("temp") 12 | 13 | os.system("rm temp/*") 14 | 15 | # if you are not using debian, these files are probably not at this path. 16 | if (os.system("psql gis -f /usr/share/doc/osmosis/examples/pgsnapshot_schema_0.6.sql")): 17 | print("Error loading pgsnapshot_schema_0.6.sql into into postGIS gis database.") 18 | sys.exit(1) 19 | 20 | if (os.system("psql gis -f /usr/share/doc/osmosis/examples/pgsnapshot_schema_0.6_linestring.sql")): 21 | print("Error loading pgsnapshot_schema_0.6_linestring.sql into postGIS gis database.") 22 | sys.exit(1) 23 | 24 | # load osm file into postGIS 25 | if ( os.system("bzcat " + outputFile + " | osmosis --read-xml - --wp user=\"mapping\" database=\"gis\"" ) ) : 26 | print("Error loading " + outputFile + " into postGIS gis database.") 27 | sys.exit(1) 28 | 29 | print("Success!!") 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /examples/gnisNamesOnWater/fetchexternaldata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | # Script to download full GNIS database, and unzip it. 6 | 7 | import os 8 | import sys 9 | import urllib 10 | import zipfile 11 | 12 | outputDir = "externaldata" 13 | 14 | # filename changes all of the time, pass ISO date in. 15 | gnisDate = "20130811" 16 | if ( len(sys.argv) > 1 ) : 17 | gnisDate = sys.argv[1] 18 | 19 | outputFile = outputDir + "/NationalFile.zip" 20 | srcURL = "http://geonames.usgs.gov/docs/stategaz/NationalFile_" + gnisDate + ".zip" 21 | 22 | # make sure our output dirs are setup, and any existing file is clean. 23 | if ( os.path.isdir(outputDir) == False) : 24 | os.mkdir(outputDir) 25 | if ( os.path.isfile(outputFile)) : 26 | os.remove(outputFile) 27 | 28 | # download MA OSM file extract. 29 | if ( os.system("wget -O " + outputFile + " " + srcURL ) ) : 30 | print("Error: Can't download NationalFile extract from " + srcURL) 31 | sys.exit(1) 32 | 33 | # unzip 34 | z=zipfile.ZipFile(outputFile,"r") 35 | zl=z.namelist() 36 | z.extractall(outputDir) 37 | z.close() 38 | os.remove(outputFile) 39 | os.rename( "externaldata/NationalFile_" + gnisDate + ".txt","externaldata/NationalFile.txt") 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /examples/massMissingBuildings/fetchexternaldata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import urllib 4 | import os 5 | import zipfile 6 | import glob 7 | 8 | # Got this script from somebody on the osm talk-us list and hacked it up. My changes 9 | # to the script are public domain. 10 | # 11 | # copies all building shape files from massgis, unzips them, and puts them into externaldata. 12 | 13 | outputDir = "externaldata" 14 | 15 | if ( os.path.isdir(outputDir) == False) : 16 | os.mkdir(outputDir) 17 | 18 | os.system("rm " + outputDir + "/*") 19 | 20 | base = "http://wsgw.mass.gov/data/gispub/shape/structures/structures_poly_" 21 | 22 | def download(): 23 | for i in range(1,352): 24 | i=str(i) 25 | n = i+".zip" 26 | localname = outputDir + "/buildings_" + n 27 | b=base+n 28 | urllib.urlretrieve(b,localname) 29 | print "downloaded town " + i 30 | st=os.stat(localname) 31 | if st.st_size>0: 32 | try : 33 | z=zipfile.ZipFile(localname,"r") 34 | except: 35 | urllib.urlretrieve(b,localname) 36 | z=zipfile.ZipFile(localname,"r") 37 | 38 | zl=z.namelist() 39 | z.extractall(outputDir) 40 | z.close() 41 | os.remove(localname) 42 | if __name__ == "__main__": 43 | download() 44 | -------------------------------------------------------------------------------- /examples/massMissingPonds/fetchexternaldata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | # Script to download major hydro data from MassGIS. Unfortunately 1/10000 hydro 6 | # data is contained in a windows .exe. The contents of the .exe should be downloaded 7 | # 8 | # "MassGIS Data - DEP Wetlands (1:12,000)" layer will need to be downloaded from a 9 | # windows box. It comes as an .exe, put it into external data. 10 | # 11 | # and placed into externaldata, this script will download the palis-saris-lookups file with the 12 | # names in it. 13 | 14 | import os 15 | import sys 16 | import urllib 17 | import zipfile 18 | 19 | outputDir = "externaldata" 20 | 21 | # make sure our output dirs are setup, and any existing file is clean. 22 | if ( os.path.isdir(outputDir) == False) : 23 | os.mkdir(outputDir) 24 | 25 | # The "MassGIS Data - MassDEP 2010 Integrated List of Waters (305(b)/303(d))" download. 26 | outputFile = outputDir + "/wbs2010_shp.zip" 27 | srcURL = "http://wsgw.mass.gov/data/gispub/shape/state/wbs2010_shp.zip" 28 | 29 | if ( os.path.isfile(outputFile)) : 30 | os.remove(outputFile) 31 | 32 | # download MassGIS water and unzip. 33 | if ( os.system("wget -O " + outputFile + " " + srcURL ) ) : 34 | print("Error: Can't download " + srcURL) 35 | sys.exit(1) 36 | 37 | # unzip 38 | z=zipfile.ZipFile(outputFile,"r") 39 | zl=z.namelist() 40 | z.extractall(outputDir) 41 | z.close() 42 | os.remove(outputFile) 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /examples/massMissingPonds/postgisloaddata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import os 6 | import sys 7 | 8 | outputFile = "osmdata/massachusetts-latest.osm.bz2" 9 | 10 | # load osm file into postGIS 11 | if ( os.system("osm2pgsql " + outputFile ) ) : 12 | print("Error loading " + outputFile + " into postGIS gis database.") 13 | sys.exit(1) 14 | 15 | if ( os.path.isdir("temp") == False) : 16 | os.mkdir("temp") 17 | 18 | projection = "900913" 19 | 20 | # the geometry 21 | os.system("rm temp/*") 22 | 23 | r = os.system("ogr2ogr -t_srs EPSG:" + projection + " -overwrite temp/wetland_poly.shp externaldata/WETLANDSDEP_POLY.shp"); 24 | if ( r ) : exit(r); 25 | 26 | r = os.system("shp2pgsql -D -I -s EPSG:" + projection + " -d temp/wetland_poly.shp massgis_wetlands | psql -q gis"); 27 | if ( r ) : exit(r); 28 | 29 | r = os.system("psql gis -c \"select UpdateGeometrySRID('massgis_wetlands','the_geom'," + projection +")\"") 30 | if ( r ) : exit(r); 31 | 32 | 33 | # the names (itegrated list - il) 34 | os.system("rm temp/*") 35 | r = os.system("ogr2ogr -t_srs EPSG:" + projection + " -overwrite temp/IL_2010_POLY.shp externaldata/IL_2010_POLY.shp"); 36 | if ( r ) : exit(r); 37 | 38 | r = os.system("shp2pgsql -D -I -s EPSG:" + projection + " -d temp/IL_2010_POLY.shp massgis_il | psql -q gis"); 39 | if ( r ) : exit(r); 40 | 41 | r = os.system("psql gis -c \"select UpdateGeometrySRID('massgis_il','the_geom'," + projection +")\"") 42 | if ( r ) : exit(r); 43 | 44 | 45 | print("Success!!") 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /examples/palisIdNamesOnWater/postgisloaddata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import os 6 | import sys 7 | 8 | outputFile = "osmdata/massachusetts-latest.osm.bz2" 9 | 10 | if ( os.path.isdir("temp") == False) : 11 | os.mkdir("temp") 12 | 13 | # the names (itegrated list - il) 14 | os.system("rm temp/*") 15 | 16 | projection = "900913" 17 | 18 | r = os.system("ogr2ogr -t_srs EPSG:" + projection + " -overwrite temp/IL_2010_POLY.shp externaldata/IL_2010_POLY.shp"); 19 | if ( r ) : exit(r); 20 | 21 | r = os.system("shp2pgsql -D -I -s EPSG:" + projection + " -d temp/IL_2010_POLY.shp massgis_il | psql -q gis"); 22 | if ( r ) : exit(r); 23 | 24 | r = os.system("psql gis -c \"select UpdateGeometrySRID('massgis_il','the_geom'," + projection +")\"") 25 | if ( r ) : exit(r); 26 | 27 | # if you are not using debian, these files are probably not at this path. 28 | os.system("rm temp/*") 29 | 30 | if (os.system("psql gis -f /usr/share/doc/osmosis/examples/pgsnapshot_schema_0.6.sql")): 31 | print("Error loading pgsnapshot_schema_0.6.sql into into postGIS gis database.") 32 | sys.exit(1) 33 | 34 | if (os.system("psql gis -f /usr/share/doc/osmosis/examples/pgsnapshot_schema_0.6_linestring.sql")): 35 | print("Error loading pgsnapshot_schema_0.6_linestring.sql into postGIS gis database.") 36 | sys.exit(1) 37 | 38 | # load osm file into postGIS 39 | if ( os.system("bzcat " + outputFile + " | osmosis --read-xml - --wp user=\"mapping\" database=\"gis\"" ) ) : 40 | print("Error loading " + outputFile + " into postGIS gis database.") 41 | sys.exit(1) 42 | 43 | print("Success!!") 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /examples/gnisNamesOnWater/postgisloaddata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import os 6 | import sys 7 | import csv 8 | import psycopg2 9 | 10 | gnisFile = "externaldata/NationalFile.txt" 11 | 12 | if (os.system("psql gis -f gnisschema.sql")): 13 | print("Error loading gnisschema.sql into into postGIS gis database.") 14 | sys.exit(1) 15 | 16 | con = psycopg2.connect(database='gis') 17 | cur = con.cursor() 18 | 19 | with open(gnisFile, 'rt') as f: 20 | reader = csv.reader(f,delimiter='|') 21 | for row in reader: 22 | if ( len(row) > 10 and (row[2] == 'Lake' or row[2] == 'Reservoir')) : 23 | cur.execute("insert into gnis (featureid, name, featureclass, geom) values(" + 24 | row[0] + "," + 25 | "'" + row[1].replace("'","''") + "'," + 26 | "'" + row[2].replace("'","''") + "'," + 27 | "ST_PointFromText('POINT(" + row[10] + ' ' + row[9] + ")', 4326))"); 28 | 29 | cur.close() 30 | 31 | con.commit() 32 | con.close() 33 | 34 | outputFile = "osmdata/massachusetts-latest.osm.bz2" 35 | 36 | if ( os.path.isdir("temp") == False) : 37 | os.mkdir("temp") 38 | 39 | os.system("rm temp/*") 40 | 41 | # if you are not using debian, these files are probably not at this path. 42 | if (os.system("psql gis -f /usr/share/doc/osmosis/examples/pgsnapshot_schema_0.6.sql")): 43 | print("Error loading pgsnapshot_schema_0.6.sql into into postGIS gis database.") 44 | sys.exit(1) 45 | 46 | if (os.system("psql gis -f /usr/share/doc/osmosis/examples/pgsnapshot_schema_0.6_linestring.sql")): 47 | print("Error loading pgsnapshot_schema_0.6_linestring.sql into postGIS gis database.") 48 | sys.exit(1) 49 | 50 | # load osm file into postGIS 51 | if ( os.system("bzcat " + outputFile + " | osmosis --read-xml - --wp user=\"mapping\" database=\"gis\"" ) ) : 52 | print("Error loading " + outputFile + " into postGIS gis database.") 53 | sys.exit(1) 54 | 55 | print("Success!!") 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /examples/gnisNamesOnWater/conflate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import psycopg2 6 | import sys 7 | import xml.etree.cElementTree as ElementTree 8 | import os 9 | 10 | 11 | con = psycopg2.connect(database='gis') 12 | cur = con.cursor() 13 | 14 | cur.execute( 15 | "CREATE TEMP TABLE water (id bigint, palisid text, name text); " 16 | "SELECT AddGeometryColumn( '', 'water', 'geom', 4326, 'POLYGON', 2); " 17 | "CREATE INDEX idx_water_geom ON water USING gist (geom); " 18 | ); 19 | 20 | cur.execute( 21 | "INSERT into water( id, name, palisid, geom) " 22 | " select " 23 | " id, " 24 | " tags::hstore -> 'name', " 25 | " tags::hstore -> 'massgis:PALIS_ID', " 26 | " ST_BuildArea(ways.linestring) " 27 | " from ways " 28 | " where " 29 | " tags @> 'natural=>water'::hstore and " 30 | " ST_IsValid( ST_BuildArea(ways.linestring) ) and " 31 | " ST_BuildArea(ways.linestring) is not null;" 32 | ); 33 | 34 | cur.execute( 35 | "select " 36 | " water.name, " 37 | " massgis_il.waterbody, " 38 | " gnis.name as gnis_name " 39 | "from water " 40 | "left join gnis on " 41 | " ST_Intersects(water.geom,gnis.geom) " 42 | "left join massgis_il on " 43 | " water.palisid = massgis_il.watercode " 44 | ); 45 | 46 | row = cur.fetchone() 47 | 48 | missingName = 0 49 | newName = 0 50 | existingName = 0 51 | differentName = 0 52 | 53 | while row is not None: 54 | 55 | line = '' 56 | if ( row[0] != None ) : 57 | line += row[0] 58 | line += "," 59 | if ( row[1] != None ) : 60 | line += row[1] 61 | line += "," 62 | if ( row[2] != None ) : 63 | line += row[2] 64 | 65 | if ( row[0] != None or row[1] != None or row[2] != None ) : 66 | if( row[0] != None ) : 67 | existingName += 1 68 | else: 69 | newName += 1 70 | 71 | #print(line) 72 | 73 | if (row[0] != None and row[1] != None and row[0] != row[1] ) : 74 | print line 75 | differentName += 1 76 | 77 | if (row[1] != None and row[2] != None and row[1] != row[2] ) : 78 | print line 79 | differentName += 1 80 | 81 | if (row[0] != None and row[2] != None and row[0] != row[2] ) : 82 | print line 83 | differentName += 1 84 | 85 | else: 86 | missingName += 1 87 | 88 | row = cur.fetchone() 89 | 90 | print("no name count %d" % (missingName)) 91 | print("new name count %d" % (newName)) 92 | print("different count %d" % (differentName)) 93 | print("same name count %d" % (existingName-differentName)) 94 | 95 | 96 | -------------------------------------------------------------------------------- /prep/findlandusereservoir.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import os 6 | import sys 7 | import tempfile 8 | import optparse 9 | import subprocess 10 | 11 | # run a command, return true if it worked, false if it did not work, used 12 | # to see if something is installed. Insures nothing goes to stdout,stderr 13 | # used to make sure the environment is setup. 14 | def commandCheck( args ) : 15 | ret = 1 16 | try: 17 | subprocess.check_output(args,stderr=subprocess.STDOUT) 18 | except: 19 | ret = 0 20 | return ret 21 | 22 | parser = optparse.OptionParser( 23 | description='Finds all landuse=reservoir ways. It is deprecated.' 24 | 'The dependencies are ogr2ogr, a postGIS database loaded with OSM ' 25 | 'data using with osm2pgsql, and ogr2osm. ogr2osm must be in the ' 26 | 'current directory, on path, or in ./ogr2osm directory. ', 27 | usage='findareowaywithnobuildings.py [options] [outputfile.osm]') 28 | 29 | parser.add_option('','--dbname',help='PostGIS database name, defaults to gis.',dest='database',default='gis') 30 | 31 | (param,filenames) = parser.parse_args() 32 | 33 | outputFile = 'landusereservoir.osm' 34 | if ( len ( filenames) > 0 ) : 35 | outputFile = filenames[0] 36 | 37 | # see if ogr2ogr is installed on path. 38 | if ( commandCheck(["ogr2ogr","--version"]) == False ) : 39 | print("error: ogr2ogr is not installed or not on the path.") 40 | sys.exit(1) 41 | 42 | # try to find ogr2osm.py, not packaged yet by anybody. 43 | ogr2osmCmd = "" 44 | if ( commandCheck(["../ogr2osm/ogr2osm.py","-h"])) : 45 | ogr2osmCmd = "../ogr2osm/ogr2osm.py" 46 | elif ( commandCheck(["python","../ogr2osm/ogr2osm.py","-h"])) : 47 | ogr2osmCmd = "python ../ogr2osm/ogr2osm.py" 48 | elif ( commandCheck(["ogr2osm","-h"])) : 49 | ogr2osmCmd = "ogr2osm" 50 | else : 51 | print("error: ogr2osm is not installed or is not on the path. See README.md for instructions.") 52 | sys.exit(1) 53 | 54 | if ( os.path.isfile(outputFile)) : 55 | os.remove(outputFile) 56 | 57 | # landuse=reservoir 58 | sql = ("select way " 59 | "from planet_osm_polygon as way " 60 | "where " 61 | " landuse = 'reservoir' and " 62 | " 'natural' != 'water' ") 63 | 64 | tempShapeFile = tempfile.mktemp(); 65 | 66 | if ( os.system("ogr2ogr -sql \"" + sql + "\" -overwrite -f 'ESRI Shapefile' " + tempShapeFile + ".shp PG:dbname=\"" + param.database + "\" ") ) : 67 | print("error: ogr2ogr command failed.") 68 | sys.exit(1) 69 | 70 | if ( os.system(ogr2osmCmd + " -f -o " + outputFile + " " + tempShapeFile + ".shp") ) : 71 | print("error: ogr2osm command failed.") 72 | os.remove(tempShapeFile + ".shp") 73 | os.remove(tempShapeFile + ".dbf") 74 | os.remove(tempShapeFile + ".shx") 75 | sys.exit(1) 76 | 77 | os.remove(tempShapeFile + ".shp") 78 | os.remove(tempShapeFile + ".dbf") 79 | os.remove(tempShapeFile + ".shx") 80 | 81 | print("Writing output file " + outputFile) 82 | 83 | sys.exit(0) 84 | 85 | 86 | -------------------------------------------------------------------------------- /examples/massMissingPonds/conflate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import sys, os, zipfile, subprocess 6 | 7 | # run a command, return true if it worked, false if it did not work, used 8 | # to see if something is installed. Insures nothing goes to stdout,stderr 9 | # used to make sure the environment is setup. 10 | def commandCheck( args ) : 11 | ret = 1 12 | try: 13 | subprocess.check_output(args,stderr=subprocess.STDOUT) 14 | except: 15 | ret = 0 16 | return ret 17 | 18 | if ( os.path.isdir("temp") == False) : 19 | os.mkdir("temp") 20 | os.system("rm temp/*") 21 | 22 | stageDir = "stage" 23 | 24 | if ( os.path.isdir(stageDir) == False) : 25 | os.mkdir(stageDir) 26 | 27 | # try to find ogr2osm.py, not packaged yet by anybody. 28 | ogr2osmCmd = "" 29 | if ( commandCheck(["../../ogr2osm/ogr2osm.py","-h"])) : 30 | ogr2osmCmd = "../../ogr2osm/ogr2osm.py" 31 | elif ( commandCheck(["python","../../ogr2osm/ogr2osm.py","-h"])) : 32 | ogr2osmCmd = "python ../../ogr2osm/ogr2osm.py" 33 | elif ( commandCheck(["ogr2osm","-h"])) : 34 | ogr2osmCmd = "ogr2osm" 35 | else : 36 | print("error: ogr2osm is not installed or is not on the path. See README.md for instructions.") 37 | sys.exit(1) 38 | 39 | # For each water poly, export it if it does not overlap with a water feature or a 40 | # waterway feature in OSM. Try to get the name from massgis_il, but leave it empty 41 | # if not present, and make a guess at the water type from name and area. 42 | sql = ("select " + 43 | " distinct ST_SimplifyPreserveTopology(massgis_wetlands.the_geom,1), " + 44 | " 'water' as natural, " + 45 | " case " + 46 | " when poly_code = 1 then 'reservoir' " + 47 | " else '' " + 48 | " end as water, " + 49 | " massgis_il.waterbody as name " + 50 | "from massgis_wetlands " + 51 | "left join massgis_il on " + 52 | " cast( massgis_wetlands.palis_id as text) = massgis_il.watercode " + 53 | "where " + 54 | " wetcode = 9 and areaacres > 1 and (poly_code = 1 or poly_code = 6) and " + 55 | " not exists (select * from planet_osm_polygon as osm " + 56 | " where " + 57 | " (osm.natural = 'water' or " + 58 | " osm.waterway != '' or " + 59 | " osm.landuse = 'reservoir') and " + 60 | " ST_IsValid( osm.way ) and " + 61 | " ST_Intersects(osm.way,massgis_wetlands.the_geom)) and" 62 | " not exists ( select * from planet_osm_line as osm " + 63 | " where " + 64 | " osm.waterway != '' and " + 65 | " ST_IsValid( osm.way ) and " + 66 | " ST_Intersects(osm.way,massgis_wetlands.the_geom))") 67 | 68 | r = os.system("ogr2ogr -sql \"" + sql + "\"" + 69 | " -overwrite -f 'ESRI Shapefile' temp/ponds_missing_from_osm.shp PG:dbname=gis " ) 70 | if ( r ) : exit(r) 71 | 72 | # ogrtranslation.py just kills the empty name tags. 73 | r = os.system(ogr2osmCmd + " -f -t ./ogrtranslation.py -o " + stageDir + "/ponds_missing_from_osm.osm temp/ponds_missing_from_osm.shp") 74 | if ( r ) : exit(r) 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /prep/findschoolswithnobuildings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import os 6 | import sys 7 | import tempfile 8 | import optparse 9 | import subprocess 10 | 11 | # run a command, return true if it worked, false if it did not work, used 12 | # to see if something is installed. Insures nothing goes to stdout,stderr 13 | # used to make sure the environment is setup. 14 | def commandCheck( args ) : 15 | ret = 1 16 | try: 17 | subprocess.check_output(args,stderr=subprocess.STDOUT) 18 | except: 19 | ret = 0 20 | return ret 21 | 22 | parser = optparse.OptionParser( 23 | description='Finds all amenity=school ways without a building=* way. ' 24 | 'The dependencies are ogr2ogr, a postGIS database loaded with OSM ' 25 | 'data using with osm2pgsql, and ogr2osm. ogr2osm must be in the ' 26 | 'current directory, on path, or in ./ogr2osm directory. ', 27 | usage='findschoolswithnobuildings.py [options] [outputfile.osm]') 28 | 29 | parser.add_option('','--dbname',help='PostGIS database name, defaults to gis.',dest='database',default='gis') 30 | 31 | (param,filenames) = parser.parse_args() 32 | 33 | outputFile = 'badschools.osm' 34 | if ( len ( filenames) > 0 ) : 35 | outputFile = filenames[0] 36 | 37 | # see if ogr2ogr is installed on path. 38 | if ( commandCheck(["ogr2ogr","--version"]) == False ) : 39 | print("error: ogr2ogr is not installed or not on the path.") 40 | sys.exit(1) 41 | 42 | # try to find ogr2osm.py, not packaged yet by anybody. 43 | ogr2osmCmd = "" 44 | if ( commandCheck(["../ogr2osm/ogr2osm.py","-h"])) : 45 | ogr2osmCmd = "../ogr2osm/ogr2osm.py" 46 | elif ( commandCheck(["python","../ogr2osm/ogr2osm.py","-h"])) : 47 | ogr2osmCmd = "python ../ogr2osm/ogr2osm.py" 48 | elif ( commandCheck(["ogr2osm","-h"])) : 49 | ogr2osmCmd = "ogr2osm" 50 | else : 51 | print("error: ogr2osm is not installed or is not on the path. See README.md for instructions.") 52 | sys.exit(1) 53 | 54 | if ( os.path.isfile(outputFile)) : 55 | os.remove(outputFile) 56 | 57 | sql = ('select ' 58 | ' way ' 59 | 'from planet_osm_polygon as way1 ' 60 | 'where ' 61 | ' way1.amenity = \'school\' and ' 62 | 'not exists ' 63 | ' (select * ' 64 | ' from planet_osm_polygon as way2 ' 65 | ' where ' 66 | ' way2.building != \'\' and ST_Intersects(way1.way,way2.way)' 67 | ' )') 68 | 69 | tempShapeFile = tempfile.mktemp(); 70 | 71 | if ( os.system("ogr2ogr -sql \"" + sql + "\" -overwrite -f 'ESRI Shapefile' " + tempShapeFile + ".shp PG:dbname=\"" + param.database + "\" ") ) : 72 | print("error: ogr2ogr command failed.") 73 | sys.exit(1) 74 | 75 | if ( os.system(ogr2osmCmd + " -f -o " + outputFile + " " + tempShapeFile + ".shp") ) : 76 | print("error: ogr2osm command failed.") 77 | os.remove(tempShapeFile + ".shp") 78 | os.remove(tempShapeFile + ".dbf") 79 | os.remove(tempShapeFile + ".shx") 80 | sys.exit(1) 81 | 82 | os.remove(tempShapeFile + ".shp") 83 | os.remove(tempShapeFile + ".dbf") 84 | os.remove(tempShapeFile + ".shx") 85 | 86 | print("Writing output file " + outputFile) 87 | 88 | sys.exit(0) 89 | 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /prep/findaerowwaywithnobuildings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import os 6 | import sys 7 | import tempfile 8 | import optparse 9 | import subprocess 10 | 11 | # run a command, return true if it worked, false if it did not work, used 12 | # to see if something is installed. Insures nothing goes to stdout,stderr 13 | # used to make sure the environment is setup. 14 | def commandCheck( args ) : 15 | ret = 1 16 | try: 17 | subprocess.check_output(args,stderr=subprocess.STDOUT) 18 | except: 19 | ret = 0 20 | return ret 21 | 22 | parser = optparse.OptionParser( 23 | description='Finds all aeroway = \'terminal\' ways without a building=* way. ' 24 | 'The dependencies are ogr2ogr, a postGIS database loaded with OSM ' 25 | 'data using with osm2pgsql, and ogr2osm. ogr2osm must be in the ' 26 | 'current directory, on path, or in ./ogr2osm directory. ', 27 | usage='findareowaywithnobuildings.py [options] [outputfile.osm]') 28 | 29 | parser.add_option('','--dbname',help='PostGIS database name, defaults to gis.',dest='database',default='gis') 30 | 31 | (param,filenames) = parser.parse_args() 32 | 33 | outputFile = 'badaeroways.osm' 34 | if ( len ( filenames) > 0 ) : 35 | outputFile = filenames[0] 36 | 37 | # see if ogr2ogr is installed on path. 38 | if ( commandCheck(["ogr2ogr","--version"]) == False ) : 39 | print("error: ogr2ogr is not installed or not on the path.") 40 | sys.exit(1) 41 | 42 | # try to find ogr2osm.py, not packaged yet by anybody. 43 | ogr2osmCmd = "" 44 | if ( commandCheck(["../ogr2osm/ogr2osm.py","-h"])) : 45 | ogr2osmCmd = "../ogr2osm/ogr2osm.py" 46 | elif ( commandCheck(["python","../ogr2osm/ogr2osm.py","-h"])) : 47 | ogr2osmCmd = "python ../ogr2osm/ogr2osm.py" 48 | elif ( commandCheck(["ogr2osm","-h"])) : 49 | ogr2osmCmd = "ogr2osm" 50 | else : 51 | print("error: ogr2osm is not installed or is not on the path. See README.md for instructions.") 52 | sys.exit(1) 53 | 54 | if ( os.path.isfile(outputFile)) : 55 | os.remove(outputFile) 56 | 57 | # get aeroway=terminals without building tags. 58 | sql = ('select way ' 59 | 'from planet_osm_polygon as way1 ' 60 | 'where ' 61 | ' way1.aeroway = \'terminal\' and ' 62 | 'not exists ' 63 | ' (select * ' 64 | ' from planet_osm_polygon as way2 ' 65 | ' where way2.building != \'\' and ST_Intersects(way1.way,way2.way) ' 66 | ' )') 67 | 68 | tempShapeFile = tempfile.mktemp(); 69 | 70 | if ( os.system("ogr2ogr -sql \"" + sql + "\" -overwrite -f 'ESRI Shapefile' " + tempShapeFile + ".shp PG:dbname=\"" + param.database + "\" ") ) : 71 | print("error: ogr2ogr command failed.") 72 | sys.exit(1) 73 | 74 | if ( os.system(ogr2osmCmd + " -f -o " + outputFile + " " + tempShapeFile + ".shp") ) : 75 | print("error: ogr2osm command failed.") 76 | os.remove(tempShapeFile + ".shp") 77 | os.remove(tempShapeFile + ".dbf") 78 | os.remove(tempShapeFile + ".shx") 79 | sys.exit(1) 80 | 81 | os.remove(tempShapeFile + ".shp") 82 | os.remove(tempShapeFile + ".dbf") 83 | os.remove(tempShapeFile + ".shx") 84 | 85 | print("Writing output file " + outputFile) 86 | 87 | sys.exit(0) 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /upload/osm2osc.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3 2 | # vim: fileencoding=utf-8 encoding=utf-8 et sw=4 3 | 4 | # Copyright (C) 2009 Jacek Konieczny 5 | # Copyright (C) 2009 Andrzej Zaborowski 6 | # 7 | # This program is free software; you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation; either version 2 of the License. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program; if not, write to the Free Software 18 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 19 | 20 | 21 | """ 22 | Convert .osm files to osmChange 0.3 format. 23 | """ 24 | 25 | __version__ = "$Revision: 21 $" 26 | 27 | import os 28 | import sys 29 | import traceback 30 | 31 | import http 32 | 33 | import xml.etree.cElementTree as ElementTree 34 | 35 | def osmsort(tree, order): 36 | list = tree[0:len(tree)] 37 | list.sort(lambda x, y: order.index(x.tag) - order.index(y.tag)) 38 | tree[0:len(tree)] = list 39 | 40 | try: 41 | if len(sys.argv) != 2: 42 | sys.stderr.write("Synopsis:\n") 43 | sys.stderr.write(" %s \n", (sys.argv[0],)) 44 | sys.exit(1) 45 | 46 | filename = sys.argv[1] 47 | if not os.path.exists(filename): 48 | sys.stderr.write("File %r doesn't exist!\n" % (filename,)) 49 | sys.exit(1) 50 | if filename.endswith(".osm"): 51 | filename_base = filename[:-4] 52 | else: 53 | filename_base = filename 54 | 55 | tree = ElementTree.parse(filename) 56 | root = tree.getroot() 57 | if root.tag != "osm" or root.attrib.get("version") != "0.6": 58 | sys.stderr.write("File %s is not a v0.6 osm file!\n" % (filename,)) 59 | sys.exit(1) 60 | 61 | output_attr = {"version": "0.3", "generator": root.attrib.get("generator")} 62 | output_root = ElementTree.Element("osmChange", output_attr) 63 | output_tree = ElementTree.ElementTree(output_root) 64 | 65 | operation = {} 66 | for opname in [ "create", "modify", "delete" ]: 67 | operation[opname] = ElementTree.SubElement(output_root, 68 | opname, output_attr) 69 | 70 | for element in root: 71 | if "id" in element.attrib and int(element.attrib["id"]) < 0: 72 | opname = "create" 73 | elif "action" in element.attrib: 74 | opname = element.attrib.pop("action") 75 | else: 76 | continue 77 | operation[opname].append(element) 78 | 79 | # Does this account for all cases? Also, is it needed? 80 | # (cases like relations containing relations... is that allowed?) 81 | #osmsort(operation["create"], [ "node", "way", "relation" ]) 82 | #osmsort(operation["delete"], [ "relation", "way", "node" ]) 83 | 84 | output_tree.write(filename_base + ".osc", "utf-8") 85 | except Exception as err: 86 | sys.stderr.write(repr(err) + "\n") 87 | traceback.print_exc(file = sys.stderr) 88 | sys.exit(1) 89 | -------------------------------------------------------------------------------- /prep/findoverlappingbuildings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import os 6 | import sys 7 | import tempfile 8 | import optparse 9 | import subprocess 10 | 11 | # run a command, return true if it worked, false if it did not work, used 12 | # to see if something is installed. Insures nothing goes to stdout,stderr 13 | # used to make sure the environment is setup. 14 | def commandCheck( args ) : 15 | ret = 1 16 | try: 17 | subprocess.check_output(args,stderr=subprocess.STDOUT) 18 | except: 19 | ret = 0 20 | return ret 21 | 22 | parser = optparse.OptionParser( 23 | description='Creates an .osm file containing overlapping buildings > 1 sq meter. ' 24 | 'The dependencies are ogr2ogr, a postGIS database loaded with OSM ' 25 | 'data using with osm2pgsql, and ogr2osm. ogr2osm must be in the ' 26 | 'current directory, on path, or in ./ogr2osm directory. ', 27 | usage='findoverlappingbuildings.py [options] [outputfile.osm]') 28 | 29 | parser.add_option('','--dbname',help='PostGIS database name, defaults to gis.',dest='database',default='gis') 30 | 31 | (param,filenames) = parser.parse_args() 32 | 33 | outputFile = 'overlappingbuildings.osm' 34 | if ( len ( filenames) > 0 ) : 35 | outputFile = filenames[0] 36 | 37 | # see if ogr2ogr is installed on path. 38 | if ( commandCheck(["ogr2ogr","--version"]) == False ) : 39 | print("error: ogr2ogr is not installed or not on the path.") 40 | sys.exit(1) 41 | 42 | # try to find ogr2osm.py, not packaged yet by anybody. 43 | ogr2osmCmd = "" 44 | if ( commandCheck(["../ogr2osm/ogr2osm.py","-h"])) : 45 | ogr2osmCmd = "../ogr2osm/ogr2osm.py" 46 | elif ( commandCheck(["python","../ogr2osm/ogr2osm.py","-h"])) : 47 | ogr2osmCmd = "python ../ogr2osm/ogr2osm.py" 48 | elif ( commandCheck(["ogr2osm","-h"])) : 49 | ogr2osmCmd = "ogr2osm" 50 | else : 51 | print("error: ogr2osm is not installed or is not on the path. See README.md for instructions.") 52 | sys.exit(1) 53 | 54 | if ( os.path.isfile(outputFile)) : 55 | os.remove(outputFile) 56 | 57 | sql = ("select way " 58 | "from planet_osm_polygon as way1 " 59 | "where " 60 | " way1.building <> '' and " 61 | " exists " 62 | " (select * " 63 | " from planet_osm_polygon as way2 " 64 | " where " 65 | " way1.osm_id <> way2.osm_id and " 66 | " way2.building <> '' and " 67 | " ST_Intersects(way1.way,way2.way) and " 68 | " ST_IsValid( way1.way) and " 69 | " ST_IsValid( way2.way) and " 70 | " ST_Area(ST_Intersection(way1.way, way2.way)) > 1 )") 71 | 72 | tempShapeFile = tempfile.mktemp(); 73 | 74 | if ( os.system("ogr2ogr -sql \"" + sql + "\" -overwrite -f 'ESRI Shapefile' " + tempShapeFile + ".shp PG:dbname=\"" + param.database + "\" ") ) : 75 | print("error: ogr2ogr command failed.") 76 | sys.exit(1) 77 | 78 | if ( os.system(ogr2osmCmd + " -f -o " + outputFile + " " + tempShapeFile + ".shp") ) : 79 | print("error: ogr2osm command failed.") 80 | os.remove(tempShapeFile + ".shp") 81 | os.remove(tempShapeFile + ".dbf") 82 | os.remove(tempShapeFile + ".shx") 83 | sys.exit(1) 84 | 85 | os.remove(tempShapeFile + ".shp") 86 | os.remove(tempShapeFile + ".dbf") 87 | os.remove(tempShapeFile + ".shx") 88 | 89 | print("Writing output file " + outputFile) 90 | 91 | sys.exit(0) 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /examples/removeAreaFromBuilding/conflate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import psycopg2 6 | import sys 7 | import xml.etree.cElementTree as ElementTree 8 | import os 9 | 10 | os.system("rm stage/*.osc") 11 | 12 | def removeSourceAndAreaTag() : 13 | 14 | con = psycopg2.connect(database='gis') 15 | cur = con.cursor() 16 | 17 | cur.execute( 18 | "select id,version,akeys(tags),avals(tags),nodes " 19 | "from ways " 20 | "where " 21 | " tags @> 'source=>\"MassGIS Buildings (http://www.mass.gov/mgis/lidarbuildingfp2d.htm)\"'::hstore and " 22 | " tags ? 'building'") 23 | 24 | output_root = ElementTree.Element('osmChange', {'version':'0.6' }) 25 | output_tree = ElementTree.ElementTree(output_root) 26 | output_op = ElementTree.SubElement(output_root, 'modify') 27 | 28 | count = 0 29 | fileCount = 1; 30 | row = cur.fetchone() 31 | 32 | while row is not None: 33 | 34 | wayid = row[0] 35 | version = row[1] 36 | 37 | output_way = ElementTree.SubElement(output_op, 'way', { 'id':str(wayid), 'version':str(version) }) 38 | 39 | ids = row[4] 40 | 41 | for v in ids: 42 | ElementTree.SubElement(output_way, 'nd', { 'ref':str(v) }) 43 | 44 | tags = row[2] 45 | values = row[3] 46 | 47 | for index,tag in enumerate(tags): 48 | if ( tag == "area" and values[index] == "yes" ) : 49 | # eat it 50 | False 51 | elif ( tag == "source" ) : 52 | # eat it 53 | False 54 | else: 55 | ElementTree.SubElement(output_way, 'tag', 56 | { 'k':tag.decode('utf-8'),'v':values[index].decode('utf-8') }) 57 | 58 | count += 1 59 | 60 | # 2000 elements per file 61 | if ( count > 2000 ) : 62 | filename = "stage/with-bad-source-%04i.osc" % (fileCount) 63 | output_tree.write(filename, "utf-8") 64 | fileCount += 1; 65 | count = 0 66 | 67 | output_root = ElementTree.Element('osmChange', {'version':'0.3' }) 68 | output_tree = ElementTree.ElementTree(output_root) 69 | output_op = ElementTree.SubElement(output_root, 'modify') 70 | 71 | row = cur.fetchone() 72 | 73 | cur.close() 74 | con.close() 75 | 76 | filename = "stage/with-bad-source-%04i.osc" % (fileCount) 77 | output_tree.write(filename, "utf-8") 78 | 79 | def removeJustAreaTag() : 80 | 81 | con = psycopg2.connect(database='gis') 82 | cur = con.cursor() 83 | 84 | cur.execute( 85 | "select id,version,akeys(tags),avals(tags),nodes " 86 | "from ways " 87 | "where " 88 | " not tags @> 'source=>\"MassGIS Buildings (http://www.mass.gov/mgis/lidarbuildingfp2d.htm)\"'::hstore and " 89 | " tags ? 'building' and " 90 | " tags ? 'area'") 91 | 92 | output_root = ElementTree.Element('osmChange', {'version':'0.3' }) 93 | output_tree = ElementTree.ElementTree(output_root) 94 | output_op = ElementTree.SubElement(output_root, 'modify') 95 | 96 | row = cur.fetchone() 97 | while row is not None: 98 | 99 | wayid = row[0] 100 | version = row[1] 101 | 102 | output_way = ElementTree.SubElement(output_op, 'way', { 'id':str(wayid), 'version':str(version) }) 103 | 104 | ids = row[4] 105 | 106 | for v in ids: 107 | ElementTree.SubElement(output_way, 'nd', { 'ref':str(v) }) 108 | 109 | tags = row[2] 110 | values = row[3] 111 | 112 | for index,tag in enumerate(tags): 113 | if ( tag == "area" and values[index] == "yes" ) : 114 | # eat it 115 | False 116 | else: 117 | ElementTree.SubElement(output_way, 'tag', 118 | { 'k':tag.decode('utf-8'),'v':values[index].decode('utf-8') }) 119 | 120 | row = cur.fetchone() 121 | 122 | cur.close() 123 | con.close() 124 | 125 | output_tree.write("stage/extra-area-leave-source.osc", "utf-8") 126 | 127 | 128 | removeSourceAndAreaTag() 129 | removeJustAreaTag() 130 | 131 | 132 | 133 | -------------------------------------------------------------------------------- /examples/massMissingBuildings/stage.py: -------------------------------------------------------------------------------- 1 | #LISCENSE 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License. 5 | # 6 | # This program is distributed in the hope that it will be useful, 7 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 8 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 | # GNU General Public License for more details. 10 | # 11 | # You should have received a copy of the GNU General Public License 12 | # along with this program; if not, write to the Free Software 13 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 14 | 15 | #INTRODUCTION 16 | # This program serves as an interface with the bulk 17 | # uploader program. It allows one to specify a directory 18 | # containing multiple osm files and have them all uploaded at 19 | # once. Please use this program carefully and reach a 20 | # a community consensus before massive imports. 21 | # ~ingalls 22 | 23 | #REQUIREMENTS 24 | # Python2 25 | # Python3 26 | 27 | import os 28 | import shutil 29 | import sys 30 | import optparse 31 | 32 | rootLoc = os.getcwd() 33 | version = "0.2" 34 | fileLoc = "" 35 | comment = "" 36 | num = 0 37 | 38 | #print "Enter Source Directory (Press enter for current)" 39 | #fileLoc = raw_input(":") 40 | sourceFiles = "src/" 41 | outputFiles = "upload/" 42 | 43 | #print "Enter Changeset Comment" 44 | #comment = raw_input(":") 45 | comment = 'Imported MassGIS Building Structures - %%filename%% - 4th data set'; 46 | source = 'Building Structures (2-D, from 2011-2012 Ortho Imagery) - Office of Geographic Information (MassGIS), Commonwealth of Massachusetts, Information Technology Division'; 47 | 48 | if not os.path.exists(rootLoc + "/" + outputFiles): 49 | os.makedirs(rootLoc + "/" + outputFiles) 50 | 51 | for osmfile in os.listdir(sourceFiles): 52 | if osmfile.endswith(".osm"): 53 | fullOsmFile = sourceFiles + osmfile 54 | smallOutputStatusFile = outputFiles + osmfile[:-4] + "-status.txt" 55 | largeOutputStatusFile = outputFiles + osmfile[:-4] + "-part0000-status.txt" 56 | 57 | if ( not os.path.exists(smallOutputStatusFile) and not os.path.exists(largeOutputStatusFile) ) : 58 | 59 | print("converting to osc: " + fullOsmFile) 60 | r = os.system("python3.1 osm2osc.py " + fullOsmFile) 61 | if ( r ) : 62 | sys.exit(1) 63 | 64 | fullOSCFile = fullOsmFile[:-4] + ".osc" 65 | 66 | # clean out older OSC files, may have changed maxElements up or down. 67 | if ( os.path.exists( outputFiles + osmfile[:-4] + ".osc" ) ) : 68 | os.remove( outputFiles + osmfile[:-4] + ".osc") 69 | os.remove( outputFiles + osmfile[:-4] + ".comment") 70 | os.remove( outputFiles + osmfile[:-4] + ".source") 71 | 72 | fileCount = 0 73 | while ( os.path.exists( outputFiles + osmfile[:-4] + "-part%04d.osc" % (fileCount,) )) : 74 | os.remove( outputFiles + osmfile[:-4] + "-part%04d.osc" % (fileCount,) ) 75 | os.remove( outputFiles + osmfile[:-4] + "-part%04d.comment" % (fileCount,) ) 76 | os.remove( outputFiles + osmfile[:-4] + "-part%04d.source" % (fileCount,) ) 77 | fileCount += 1 78 | 79 | print("splitting osc: " + fullOSCFile) 80 | r = os.system("python osmsplit.py --outputDir " + outputFiles + " --maxElements 10000 " + fullOSCFile) 81 | if ( r ) : 82 | sys.exit(1) 83 | os.remove( fullOSCFile) 84 | else : 85 | print("already uploaded skipping: " + fullOsmFile) 86 | 87 | # write out comment and source files. 88 | for oscfile in os.listdir(outputFiles): 89 | if oscfile.endswith(".osc"): 90 | commentFilename = outputFiles + oscfile[:-4] + ".comment" 91 | if ( not os.path.exists(commentFilename) ) : 92 | commentFile = open( commentFilename,"w") 93 | commentFile.write(comment.replace('%%filename%%',oscfile)) 94 | commentFile.close() 95 | 96 | sourceFilename = outputFiles + oscfile[:-4] + ".source" 97 | sourceFile = open( sourceFilename,"w") 98 | sourceFile.write(source) 99 | sourceFile.close() 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | -------------------------------------------------------------------------------- /examples/palisIdNamesOnWater/conflate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import psycopg2 6 | import sys 7 | import xml.etree.cElementTree as ElementTree 8 | import os 9 | 10 | if ( os.path.isdir('stage') == False) : 11 | os.mkdir('stage') 12 | 13 | os.system("rm stage/*.osc") 14 | 15 | def fixDEPImport() : 16 | 17 | con = psycopg2.connect(database='gis') 18 | cur = con.cursor() 19 | 20 | cur.execute( 21 | "select " 22 | " id, " 23 | " version, " 24 | " akeys(tags), " 25 | " avals(tags), " 26 | " nodes, " 27 | " massgis_il.waterbody " 28 | "from ways " 29 | "left join massgis_il on " 30 | " tags -> 'massgis:PALIS_ID' = massgis_il.watercode " 31 | "where " 32 | " tags @> 'natural=>water'::hstore") 33 | 34 | output_root = ElementTree.Element('osmChange', {'version':'0.6' }) 35 | output_tree = ElementTree.ElementTree(output_root) 36 | output_op = ElementTree.SubElement(output_root, 'modify') 37 | 38 | count = 0 39 | fileCount = 1; 40 | row = cur.fetchone() 41 | 42 | while row is not None: 43 | 44 | wayid = row[0] 45 | version = row[1] 46 | 47 | tags = row[2] 48 | values = row[3] 49 | ids = row[4] 50 | name = row[5] 51 | 52 | editWay = False 53 | hasName = False 54 | 55 | for index,tag in enumerate(tags): 56 | if ( tag == "massgis:PALID_ID" or tag == "massgis:WETCODE") : 57 | editWay = True 58 | if ( tag == "name") : 59 | hasName = True 60 | 61 | if ( editWay ) : 62 | 63 | output_way = ElementTree.SubElement(output_op, 'way', { 'id':str(wayid), 'version':str(version) }) 64 | 65 | for v in ids: 66 | ElementTree.SubElement(output_way, 'nd', { 'ref':str(v) }) 67 | 68 | for index,tag in enumerate(tags): 69 | if ( tag == "area" and values[index] == "yes" ) : 70 | # eat it 71 | False 72 | elif ( tag == "source" and values[index] == "DEP Wetlands (1:12,000) - April 2007 (http://www.mass.gov/mgis/wetdep.htm)") : 73 | # eat it 74 | False 75 | elif ( tag.find("massgis:") == 0) : 76 | # eat it 77 | False 78 | else: 79 | ElementTree.SubElement(output_way, 'tag', 80 | { 'k':tag.decode('utf-8'),'v':values[index].decode('utf-8') }) 81 | 82 | # If no name exists, look it up and add it in. If name exists, leave it. 83 | # I have already QA'ed the existing name agaist PALIS_ID, when both exist. 84 | # 100% sure the name in OSM should always be kept. 85 | if ( hasName == False and name != None) : 86 | ElementTree.SubElement(output_way, 'tag', 87 | { 'k':'name','v':name.decode('utf-8') }) 88 | 89 | count += 1 90 | 91 | # 2000 elements per file, not needed not that many 92 | #if ( count > 2000 ) : 93 | # filename = "stage/water-depimport-%04i.osc" % (fileCount) 94 | # output_tree.write(filename, "utf-8") 95 | # fileCount += 1; 96 | # count = 0 97 | 98 | # output_root = ElementTree.Element('osmChange', {'version':'0.3' }) 99 | # output_tree = ElementTree.ElementTree(output_root) 100 | # output_op = ElementTree.SubElement(output_root, 'modify') 101 | 102 | row = cur.fetchone() 103 | 104 | cur.close() 105 | con.close() 106 | 107 | filename = "stage/water-depimport-%04i.osc" % (fileCount) 108 | output_tree.write(filename, "utf-8") 109 | 110 | def fixOpenSpaceImport() : 111 | 112 | con = psycopg2.connect(database='gis') 113 | cur = con.cursor() 114 | 115 | cur.execute( 116 | "select " 117 | " id, " 118 | " version, " 119 | " akeys(tags), " 120 | " avals(tags), " 121 | " nodes " 122 | "from ways " 123 | "where " 124 | " tags @> 'natural=>water'::hstore") 125 | 126 | output_root = ElementTree.Element('osmChange', {'version':'0.6' }) 127 | output_tree = ElementTree.ElementTree(output_root) 128 | output_op = ElementTree.SubElement(output_root, 'modify') 129 | 130 | count = 0 131 | fileCount = 1; 132 | row = cur.fetchone() 133 | 134 | while row is not None: 135 | 136 | wayid = row[0] 137 | version = row[1] 138 | 139 | tags = row[2] 140 | values = row[3] 141 | ids = row[4] 142 | 143 | editWay = False 144 | 145 | for index,tag in enumerate(tags): 146 | if ( tag == "massgis:TOWN_ID" or tag == "massgis:OWNER_TYPE") : 147 | editWay = True 148 | 149 | if ( editWay ) : 150 | 151 | output_way = ElementTree.SubElement(output_op, 'way', { 'id':str(wayid), 'version':str(version) }) 152 | 153 | for v in ids: 154 | ElementTree.SubElement(output_way, 'nd', { 'ref':str(v) }) 155 | 156 | for index,tag in enumerate(tags): 157 | if ( tag == "area" and values[index] == "yes" ) : 158 | # eat it 159 | False 160 | elif ( tag == "source" and values[index] == "MassGIS OpenSpace (http://www.mass.gov/mgis/osp.htm)") : 161 | # eat it 162 | False 163 | elif ( tag.find("massgis:") == 0) : 164 | # eat it 165 | False 166 | else: 167 | ElementTree.SubElement(output_way, 'tag', 168 | { 'k':tag.decode('utf-8'),'v':values[index].decode('utf-8') }) 169 | 170 | count += 1 171 | 172 | # 2000 elements per file, not needed not that many 173 | #if ( count > 2000 ) : 174 | # filename = "stage/water-openspace-%04i.osc" % (fileCount) 175 | # output_tree.write(filename, "utf-8") 176 | # fileCount += 1; 177 | # count = 0 178 | 179 | # output_root = ElementTree.Element('osmChange', {'version':'0.3' }) 180 | # output_tree = ElementTree.ElementTree(output_root) 181 | # output_op = ElementTree.SubElement(output_root, 'modify') 182 | 183 | row = cur.fetchone() 184 | 185 | cur.close() 186 | con.close() 187 | 188 | filename = "stage/water-openspace-%04i.osc" % (fileCount) 189 | output_tree.write(filename, "utf-8") 190 | 191 | fixDEPImport() 192 | fixOpenSpaceImport() 193 | 194 | 195 | 196 | -------------------------------------------------------------------------------- /upload/osmsplit.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python 2 | # 3 | # This program is free software; you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation; either version 2 of the License. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 15 | 16 | """ 17 | Splits an Open Street Map change files (OSC) into smaller OSC files. Useful 18 | for making a large OSC file fit into the 50,000 element change set limit 19 | that Open Street Map servers have. 20 | 21 | The OSC file format is documented here. http://wiki.openstreetmap.org/wiki/OsmChange 22 | This scripts supports v0.3 and v0.6. 23 | """ 24 | 25 | import os 26 | import sys 27 | import traceback 28 | import optparse 29 | import shutil 30 | import math 31 | 32 | import xml.etree.cElementTree as ElementTree 33 | 34 | def chunkSortByLon( chunk ) : 35 | lon = 0 36 | for member in chunk['members'] : 37 | if ( member['tag'] == 'node' ) : 38 | lon = member['element'].attrib.get("lon") 39 | return lon 40 | 41 | def writeChunk( oscGraph,part_op,chunk,operation,tag) : 42 | for node in chunk['members'] : 43 | if ( node['tag'] == tag and 44 | node['operation'] == operation) : 45 | part_op.append(node['element']) 46 | 47 | def assignChunk( oscGraph, key, chunkNumber ) : 48 | if ( oscGraph[key]['chunkAssignment'] >= 0 ) : 49 | return 0 50 | 51 | oscGraph[key]['chunkAssignment'] = chunkNumber 52 | assignCount = 1 53 | 54 | for parent in oscGraph[key]['parents'] : 55 | assignCount += assignChunk( oscGraph,parent,chunkNumber) 56 | 57 | for child in oscGraph[key]['children'] : 58 | assignCount += assignChunk( oscGraph,child,chunkNumber) 59 | return assignCount 60 | 61 | def splitOSC( filename, outputDir, maxElements) : 62 | print " " + filename + " ", 63 | tree = ElementTree.parse(filename) 64 | root = tree.getroot() 65 | 66 | oscGraph = {} 67 | if root.tag != "osmChange" or (root.attrib.get("version") != "0.3" and 68 | root.attrib.get("version") != "0.6"): 69 | print >>sys.stderr, u"File %s is not a v0.3 or v0.6 osmChange file!" % (filename,) 70 | sys.exit(1) 71 | 72 | # setup nodes in connectivity graph 73 | for operation in root: 74 | for element in operation: 75 | key = element.tag + element.attrib.get("id") 76 | # chunkAssignment < 0 not assigned yet. 77 | if ( operation.tag == 'create' ) : 78 | oscGraph[key] = { 79 | 'tag' : element.tag, 80 | 'operation': operation.tag, 81 | 'children' : [] , 82 | 'parents' : [], 83 | 'chunkAssignment' : -1, 84 | 'element' : element 85 | } 86 | else : 87 | print >>sys.stderr, u"File %s: Only creating elements supported for now. Sorry no %s." % (filename,operation.tag) 88 | sys.exit(1) 89 | 90 | # make connections between nodes 91 | for operation in root: 92 | for element in operation: 93 | key = element.tag + element.attrib.get("id") 94 | 95 | if ( element.tag == 'way' ) : 96 | for child in element: 97 | if ( child.tag == 'nd') : 98 | childKey = 'node' + child.attrib.get("ref") 99 | oscGraph[key]['children'].append( childKey) 100 | oscGraph[childKey]['parents'].append(key) 101 | elif ( element.tag == 'relation' ) : 102 | for child in element: 103 | if ( child.tag == 'member') : 104 | childKey = child.attrib.get("type") + child.attrib.get("ref") 105 | oscGraph[key]['children'].append( childKey) 106 | oscGraph[childKey]['parents'].append(key) 107 | 108 | # assign chunks to each element, keep track of how many elements are in each 109 | # chunk so we know how to split them. A chunk is a set of ways, nodes, relations 110 | # that point to each other. These "chunks" can't be broken across changesets 111 | # files without creating dependencies between uploads. 112 | chunks = [] 113 | nextChunk = 0; 114 | 115 | for key in oscGraph : 116 | if ( oscGraph[key]['chunkAssignment'] < 0 ) : 117 | numberOfElementsInChunk = assignChunk( oscGraph,key,nextChunk) 118 | chunks.append( { 'count': numberOfElementsInChunk, 'members' : [] }) 119 | nextChunk += 1 120 | 121 | # build list of elements in each chunk for faster access when writing files out 122 | # it is too slow to crawl through the graph for every chuck write 123 | for key in oscGraph : 124 | chunks[oscGraph[key]['chunkAssignment']]['members'].append(oscGraph[key]) 125 | 126 | # sort the chucks so the output files are not random located 127 | chunks = sorted(chunks, key=chunkSortByLon ) 128 | 129 | totalElements = 0.0 130 | for chunk in chunks : 131 | totalElements += chunk['count'] 132 | 133 | files = int(math.ceil( totalElements / maxElements)) 134 | if ( files > 0 ) : 135 | averageElementsPerFile = int(math.ceil(totalElements/files)) 136 | else : 137 | # empty change set file. 138 | averageElementsPerFile = 1 139 | files = 1 140 | 141 | if ( files == 1 ) : 142 | shutil.copyfile(filename, outputDir + filename.split("/")[-1]) 143 | else : 144 | 145 | # figure out what chunks go in what files 146 | chunksInFiles = [ [] ] 147 | currentFileSize = 0 148 | for chunk in chunks : 149 | if ( currentFileSize > 0 and currentFileSize+chunk['count'] > maxElements ) : 150 | chunksInFiles.append([]) 151 | currentFileSize = 0 152 | 153 | chunksInFiles[-1].append( chunk) 154 | currentFileSize += chunk['count'] 155 | 156 | if ( currentFileSize > maxElements) : 157 | print >>sys.stderr, u"File %s: has a set of changes that is larger than the limit." % (filename) 158 | 159 | 160 | for fileIndex in range( 0,len(chunksInFiles)) : 161 | 162 | filename_base = filename[:-4].split("/")[-1]; 163 | outputFilename = "%s%s-part%04i.osc" % (outputDir,filename_base, fileIndex) 164 | 165 | # if input file is v0.3, then the output file will be v0.3 166 | part_root = ElementTree.Element('osmChange', {'version':root.attrib.get("version")}) 167 | part_tree = ElementTree.ElementTree(part_root) 168 | 169 | part_op = ElementTree.SubElement(part_root, 'create') 170 | for chunk in chunksInFiles[fileIndex] : 171 | writeChunk( oscGraph,part_op,chunk,'create','node') 172 | writeChunk( oscGraph,part_op,chunk,'create','way') 173 | writeChunk( oscGraph,part_op,chunk,'create','relation') 174 | 175 | part_op = ElementTree.SubElement(part_root, 'modify') 176 | for chunk in chunksInFiles[fileIndex] : 177 | writeChunk( oscGraph,part_op,chunk,'modify','node') 178 | writeChunk( oscGraph,part_op,chunk,'modify','way') 179 | writeChunk( oscGraph,part_op,chunk,'modify','relation') 180 | 181 | part_op = ElementTree.SubElement(part_root, 'delete') 182 | for chunk in chunksInFiles[fileIndex] : 183 | writeChunk( oscGraph,part_op,chunk,'delete','relation') 184 | writeChunk( oscGraph,part_op,chunk,'delete','way') 185 | writeChunk( oscGraph,part_op,chunk,'delete','node') 186 | 187 | part_tree.write(outputFilename, "utf-8") 188 | 189 | print "" 190 | 191 | try: 192 | parser = optparse.OptionParser(description='Splits an Open Street Map change set file (OSC) into smaller OSC files.',usage='osmsplit.py [options] uploadfile1.osc') 193 | 194 | parser.add_option( 195 | '--outputDir', 196 | help='output directory for split files', 197 | dest='outputDir', 198 | default='.') 199 | parser.add_option( 200 | '--maxElements', 201 | help='maximum elements in each output OSC file', 202 | dest='maxElements', 203 | type='int', 204 | default=10000) 205 | 206 | (param,filenames) = parser.parse_args() 207 | 208 | outputDir = param.outputDir 209 | if ( param.outputDir and outputDir[-1] != '/') : 210 | outputDir = outputDir + '/' 211 | 212 | for filename in filenames: 213 | splitOSC( filename, outputDir, param.maxElements) 214 | 215 | except Exception,err: 216 | print >>sys.stderr, repr(err) 217 | traceback.print_exc(file=sys.stderr) 218 | sys.exit(1) 219 | -------------------------------------------------------------------------------- /examples/massMissingBuildings/conflate.py: -------------------------------------------------------------------------------- 1 | #!usr/bin/python 2 | 3 | # Jason Remillard - This file is in the public domain. 4 | 5 | import sys, os, zipfile, subprocess 6 | 7 | # run a command, return true if it worked, false if it did not work, used 8 | # to see if something is installed. Insures nothing goes to stdout,stderr 9 | # used to make sure the environment is setup. 10 | def commandCheck( args ) : 11 | ret = 1 12 | try: 13 | subprocess.check_output(args,stderr=subprocess.STDOUT) 14 | except: 15 | ret = 0 16 | return ret 17 | 18 | 19 | def exportTown(townnumber,townname) : 20 | 21 | if ( os.path.isdir("temp") == False) : 22 | os.mkdir("temp") 23 | 24 | os.system("rm temp/*") 25 | 26 | # reproject to 900913, which is what we use inside of postGIS 27 | r = os.system("ogr2ogr -t_srs EPSG:900913 -overwrite temp/structures_poly_" + townname + ".shp externaldata/structures_poly_" + str(townnumber) + ".shp"); 28 | if ( r ) : exit(r); 29 | 30 | # import to postGIS 31 | r = os.system("shp2pgsql -D -I -s EPSG:900913 -d temp/structures_poly_" + 32 | townname + ".shp massgis_structures | psql -q gis"); 33 | if ( r ) : exit(r); 34 | 35 | # set the projection, shp2pgsl does not do this, don't know why... 36 | r = os.system("psql gis -c \"select UpdateGeometrySRID('massgis_structures','the_geom',900913)\"") 37 | if ( r ) : exit(r); 38 | 39 | r = os.system("rm temp/*") 40 | if ( r ) : exit(r); 41 | 42 | if ( os.path.isdir("stage") == False) : 43 | os.mkdir("stage") 44 | 45 | # try to find ogr2osm.py, not packaged yet by anybody. 46 | ogr2osmCmd = "" 47 | if ( commandCheck(["../../ogr2osm/ogr2osm.py","-h"])) : 48 | ogr2osmCmd = "../../ogr2osm/ogr2osm.py" 49 | elif ( commandCheck(["python","../../ogr2osm/ogr2osm.py","-h"])) : 50 | ogr2osmCmd = "python ../../ogr2osm/ogr2osm.py" 51 | elif ( commandCheck(["ogr2osm","-h"])) : 52 | ogr2osmCmd = "ogr2osm" 53 | else : 54 | print("error: ogr2osm is not installed or is not on the path. See README.md for instructions.") 55 | sys.exit(1) 56 | 57 | outBase = "staging/" + townname + "_"; 58 | 59 | # get structures missing from OSM 60 | sql = ("select ST_SimplifyPreserveTopology(the_geom,0.20),'yes' as building " 61 | "from massgis_structures " 62 | "where " 63 | " massgis_structures.town_id = " + str(townnumber) + " and " 64 | " not exists " 65 | " (select * from planet_osm_polygon as osm " 66 | " where " 67 | " osm.building != '' and " 68 | " ST_Intersects(osm.way,massgis_structures.the_geom))") 69 | 70 | r = os.system("ogr2ogr -sql \"" + sql + "\" -overwrite -f 'ESRI Shapefile' temp/structures_missing_from_osm_" + townname + ".shp PG:dbname=gis " ) 71 | if ( r ) : exit(r); 72 | 73 | r = os.system(ogr2osmCmd + " -f -o " + outBase + 74 | "buildings_missing_from_osm.osm temp/structures_missing_from_osm_" + townname + ".shp") 75 | if ( r ) : exit(r); 76 | 77 | # get structures overlapping with OSM 78 | sql = ("select ST_SimplifyPreserveTopology(the_geom,0.20),'yes' as building " 79 | "from massgis_structures " 80 | "where " 81 | " massgis_structures.town_id = " + str(townnumber) + " and " 82 | " exists " 83 | " (select * from planet_osm_polygon as osm " 84 | " where " 85 | " osm.building != '' and " 86 | " ST_Intersects(osm.way,massgis_structures.the_geom))") 87 | 88 | r = os.system("ogr2ogr -sql \"" + sql + 89 | "\" -overwrite -f 'ESRI Shapefile' temp/structures_overlap_with_osm_" + townname + ".shp " + 90 | "PG:dbname=gis ") 91 | if ( r ) : exit(r); 92 | 93 | r = os.system(ogr2osmCmd + " -f -o " + outBase + 94 | "buildings_overlap_with_osm.osm temp/structures_overlap_with_osm_" + townname + ".shp") 95 | if ( r ) : exit(r); 96 | 97 | 98 | def convertMassGISTownNumbersToName( number) : 99 | 100 | d = dict( [ ('1','ABINGTON'), 101 | ('2','ACTON'), 102 | ('3','ACUSHNET'), 103 | ('4','ADAMS'), 104 | ('5','AGAWAM'), 105 | ('6','ALFORD'), 106 | ('7','AMESBURY'), 107 | ('8','AMHERST'), 108 | ('9','ANDOVER'), 109 | ('10','ARLINGTON'), 110 | ('11','ASHBURNHAM'), 111 | ('12','ASHBY'), 112 | ('13','ASHFIELD'), 113 | ('14','ASHLAND'), 114 | ('15','ATHOL'), 115 | ('16','ATTLEBORO'), 116 | ('17','AUBURN'), 117 | ('18','AVON'), 118 | ('19','AYER'), 119 | ('20','BARNSTABLE'), 120 | ('21','BARRE'), 121 | ('22','BECKET'), 122 | ('23','BEDFORD'), 123 | ('24','BELCHERTOWN'), 124 | ('25','BELLINGHAM'), 125 | ('26','BELMONT'), 126 | ('27','BERKLEY'), 127 | ('28','BERLIN'), 128 | ('29','BERNARDSTON'), 129 | ('30','BEVERLY'), 130 | ('31','BILLERICA'), 131 | ('32','BLACKSTONE'), 132 | ('33','BLANDFORD'), 133 | ('34','BOLTON'), 134 | ('35','BOSTON'), 135 | ('36','BOURNE'), 136 | ('37','BOXBOROUGH'), 137 | ('38','BOXFORD'), 138 | ('39','BOYLSTON'), 139 | ('40','BRAINTREE'), 140 | ('41','BREWSTER'), 141 | ('42','BRIDGEWATER'), 142 | ('43','BRIMFIELD'), 143 | ('44','BROCKTON'), 144 | ('45','BROOKFIELD'), 145 | ('46','BROOKLINE'), 146 | ('47','BUCKLAND'), 147 | ('48','BURLINGTON'), 148 | ('49','CAMBRIDGE'), 149 | ('50','CANTON'), 150 | ('51','CARLISLE'), 151 | ('52','CARVER'), 152 | ('53','CHARLEMONT'), 153 | ('54','CHARLTON'), 154 | ('55','CHATHAM'), 155 | ('56','CHELMSFORD'), 156 | ('57','CHELSEA'), 157 | ('58','CHESHIRE'), 158 | ('59','CHESTER'), 159 | ('60','CHESTERFIELD'), 160 | ('61','CHICOPEE'), 161 | ('62','CHILMARK'), 162 | ('63','CLARKSBURG'), 163 | ('64','CLINTON'), 164 | ('65','COHASSET'), 165 | ('66','COLRAIN'), 166 | ('67','CONCORD'), 167 | ('68','CONWAY'), 168 | ('69','CUMMINGTON'), 169 | ('70','DALTON'), 170 | ('71','DANVERS'), 171 | ('72','DARTMOUTH'), 172 | ('73','DEDHAM'), 173 | ('74','DEERFIELD'), 174 | ('75','DENNIS'), 175 | ('76','DIGHTON'), 176 | ('77','DOUGLAS'), 177 | ('78','DOVER'), 178 | ('79','DRACUT'), 179 | ('80','DUDLEY'), 180 | ('81','DUNSTABLE'), 181 | ('82','DUXBURY'), 182 | ('83','EAST_BRIDGEWATER'), 183 | ('84','EAST_BROOKFIELD'), 184 | ('85','EAST_LONGMEADOW'), 185 | ('86','EASTHAM'), 186 | ('87','EASTHAMPTON'), 187 | ('88','EASTON'), 188 | ('89','EDGARTOWN'), 189 | ('90','EGREMONT'), 190 | ('91','ERVING'), 191 | ('92','ESSEX'), 192 | ('93','EVERETT'), 193 | ('94','FAIRHAVEN'), 194 | ('95','FALL_RIVER'), 195 | ('96','FALMOUTH'), 196 | ('97','FITCHBURG'), 197 | ('98','FLORIDA'), 198 | ('99','FOXBOROUGH'), 199 | ('100','FRAMINGHAM'), 200 | ('101','FRANKLIN'), 201 | ('102','FREETOWN'), 202 | ('103','GARDNER'), 203 | ('104','GAY_HEAD'), 204 | ('105','GEORGETOWN'), 205 | ('106','GILL'), 206 | ('107','GLOUCESTER'), 207 | ('108','GOSHEN'), 208 | ('109','GOSNOLD'), 209 | ('110','GRAFTON'), 210 | ('111','GRANBY'), 211 | ('112','GRANVILLE'), 212 | ('113','GREAT_BARRINGTON'), 213 | ('114','GREENFIELD'), 214 | ('115','GROTON'), 215 | ('116','GROVELAND'), 216 | ('117','HADLEY'), 217 | ('118','HALIFAX'), 218 | ('119','HAMILTON'), 219 | ('120','HAMPDEN'), 220 | ('121','HANCOCK'), 221 | ('122','HANOVER'), 222 | ('123','HANSON'), 223 | ('124','HARDWICK'), 224 | ('125','HARVARD'), 225 | ('126','HARWICH'), 226 | ('127','HATFIELD'), 227 | ('128','HAVERHILL'), 228 | ('129','HAWLEY'), 229 | ('130','HEATH'), 230 | ('131','HINGHAM'), 231 | ('132','HINSDALE'), 232 | ('133','HOLBROOK'), 233 | ('134','HOLDEN'), 234 | ('135','HOLLAND'), 235 | ('136','HOLLISTON'), 236 | ('137','HOLYOKE'), 237 | ('138','HOPEDALE'), 238 | ('139','HOPKINTON'), 239 | ('140','HUBBARDSTON'), 240 | ('141','HUDSON'), 241 | ('142','HULL'), 242 | ('143','HUNTINGTON'), 243 | ('144','IPSWICH'), 244 | ('145','KINGSTON'), 245 | ('146','LAKEVILLE'), 246 | ('147','LANCASTER'), 247 | ('148','LANESBOROUGH'), 248 | ('149','LAWRENCE'), 249 | ('150','LEE'), 250 | ('151','LEICESTER'), 251 | ('152','LENOX'), 252 | ('153','LEOMINSTER'), 253 | ('154','LEVERETT'), 254 | ('155','LEXINGTON'), 255 | ('156','LEYDEN'), 256 | ('157','LINCOLN'), 257 | ('158','LITTLETON'), 258 | ('159','LONGMEADOW'), 259 | ('160','LOWELL'), 260 | ('161','LUDLOW'), 261 | ('162','LUNENBURG'), 262 | ('163','LYNN'), 263 | ('164','LYNNFIELD'), 264 | ('165','MALDEN'), 265 | ('166','MANCHESTER'), 266 | ('167','MANSFIELD'), 267 | ('168','MARBLEHEAD'), 268 | ('169','MARION'), 269 | ('170','MARLBOROUGH'), 270 | ('171','MARSHFIELD'), 271 | ('172','MASHPEE'), 272 | ('173','MATTAPOISETT'), 273 | ('174','MAYNARD'), 274 | ('175','MEDFIELD'), 275 | ('176','MEDFORD'), 276 | ('177','MEDWAY'), 277 | ('178','MELROSE'), 278 | ('179','MENDON'), 279 | ('180','MERRIMAC'), 280 | ('181','METHUEN'), 281 | ('182','MIDDLEBOROUGH'), 282 | ('183','MIDDLEFIELD'), 283 | ('184','MIDDLETON'), 284 | ('185','MILFORD'), 285 | ('186','MILLBURY'), 286 | ('187','MILLIS'), 287 | ('188','MILLVILLE'), 288 | ('189','MILTON'), 289 | ('190','MONROE'), 290 | ('191','MONSON'), 291 | ('192','MONTAGUE'), 292 | ('193','MONTEREY'), 293 | ('194','MONTGOMERY'), 294 | ('195','MOUNT_WASHINGTON'), 295 | ('196','NAHANT'), 296 | ('197','NANTUCKET'), 297 | ('198','NATICK'), 298 | ('199','NEEDHAM'), 299 | ('200','NEW_ASHFORD'), 300 | ('201','NEW_BEDFORD'), 301 | ('202','NEW_BRAINTREE'), 302 | ('203','NEW_MARLBOROUGH'), 303 | ('204','NEW_SALEM'), 304 | ('205','NEWBURY'), 305 | ('206','NEWBURYPORT'), 306 | ('207','NEWTON'), 307 | ('208','NORFOLK'), 308 | ('209','NORTH_ADAMS'), 309 | ('210','NORTH_ANDOVER'), 310 | ('211','NORTH_ATTLEBOROUGH'), 311 | ('212','NORTH_BROOKFIELD'), 312 | ('213','NORTH_READING'), 313 | ('214','NORTHAMPTON'), 314 | ('215','NORTHBOROUGH'), 315 | ('216','NORTHBRIDGE'), 316 | ('217','NORTHFIELD'), 317 | ('218','NORTON'), 318 | ('219','NORWELL'), 319 | ('220','NORWOOD'), 320 | ('221','OAK_BLUFFS'), 321 | ('222','OAKHAM'), 322 | ('223','ORANGE'), 323 | ('224','ORLEANS'), 324 | ('225','OTIS'), 325 | ('226','OXFORD'), 326 | ('227','PALMER'), 327 | ('228','PAXTON'), 328 | ('229','PEABODY'), 329 | ('230','PELHAM'), 330 | ('231','PEMBROKE'), 331 | ('232','PEPPERELL'), 332 | ('233','PERU'), 333 | ('234','PETERSHAM'), 334 | ('235','PHILLIPSTON'), 335 | ('236','PITTSFIELD'), 336 | ('237','PLAINFIELD'), 337 | ('238','PLAINVILLE'), 338 | ('239','PLYMOUTH'), 339 | ('240','PLYMPTON'), 340 | ('241','PRINCETON'), 341 | ('242','PROVINCETOWN'), 342 | ('243','QUINCY'), 343 | ('244','RANDOLPH'), 344 | ('245','RAYNHAM'), 345 | ('246','READING'), 346 | ('247','REHOBOTH'), 347 | ('248','REVERE'), 348 | ('249','RICHMOND'), 349 | ('250','ROCHESTER'), 350 | ('251','ROCKLAND'), 351 | ('252','ROCKPORT'), 352 | ('253','ROWE'), 353 | ('254','ROWLEY'), 354 | ('255','ROYALSTON'), 355 | ('256','RUSSELL'), 356 | ('257','RUTLAND'), 357 | ('258','SALEM'), 358 | ('259','SALISBURY'), 359 | ('260','SANDISFIELD'), 360 | ('261','SANDWICH'), 361 | ('262','SAUGU'), 362 | ('263','SAVOY'), 363 | ('264','SCITUATE'), 364 | ('265','SEEKONK'), 365 | ('266','SHARON'), 366 | ('267','SHEFFIELD'), 367 | ('268','SHELBURNE'), 368 | ('269','SHERBORN'), 369 | ('270','SHIRLEY'), 370 | ('271','SHREWSBURY'), 371 | ('272','SHUTESBURY'), 372 | ('273','SOMERSET'), 373 | ('274','SOMERVILLE'), 374 | ('275','SOUTH_HADL'), 375 | ('276','SOUTHAMPTON'), 376 | ('277','SOUTHBOROUGH'), 377 | ('278','SOUTHBRIDGE'), 378 | ('279','SOUTHWICK'), 379 | ('280','SPENCER'), 380 | ('281','SPRINGFIELD'), 381 | ('282','STERLING'), 382 | ('283','STOCKBRIDGE'), 383 | ('284','STONEHAM'), 384 | ('285','STOUGHTON'), 385 | ('286','STOW'), 386 | ('287','STURBRIDGE'), 387 | ('288','SUDBURY'), 388 | ('289','SUNDERLAND'), 389 | ('290','SUTTON'), 390 | ('291','SWAMPSCOTT'), 391 | ('292','SWANSEA'), 392 | ('293','TAUNTON'), 393 | ('294','TEMPLETON'), 394 | ('295','TEWKSBURY'), 395 | ('296','TISBURY'), 396 | ('297','TOLLAND'), 397 | ('298','TOPSFIELD'), 398 | ('299','TOWNSEND'), 399 | ('300','TRURO'), 400 | ('301','TYNGSBOROUGH'), 401 | ('302','TYRINGHAM'), 402 | ('303','UPTON'), 403 | ('304','UXBRIDGE'), 404 | ('305','WAKEFIELD'), 405 | ('306','WALES'), 406 | ('307','WALPOLE'), 407 | ('308','WALTHAM'), 408 | ('309','WARE'), 409 | ('310','WAREHAM'), 410 | ('311','WARREN'), 411 | ('312','WARWICK'), 412 | ('313','WASHINGTON'), 413 | ('314','WATERTOWN'), 414 | ('315','WAYLAND'), 415 | ('316','WEBSTER'), 416 | ('317','WELLESLEY'), 417 | ('318','WELLFLEET'), 418 | ('319','WENDELL'), 419 | ('320','WENHAM'), 420 | ('321','WEST_BOYLSTON'), 421 | ('322','WEST_BRIDGEWATER'), 422 | ('323','WEST_BROOKFIELD'), 423 | ('324','WEST_NEWBURY'), 424 | ('325','WEST_SPRINGFIELD'), 425 | ('326','WEST_STOCKBRIDGE'), 426 | ('327','WEST_TISBURY'), 427 | ('328','WESTBOROUGH'), 428 | ('329','WESTFIELD'), 429 | ('330','WESTFORD'), 430 | ('331','WESTHAMPTON'), 431 | ('332','WESTMINSTER'), 432 | ('333','WESTON'), 433 | ('334','WESTPORT'), 434 | ('335','WESTWOOD'), 435 | ('336','WEYMOUTH'), 436 | ('337','WHATELY'), 437 | ('338','WHITMAN'), 438 | ('339','WILBRAHAM'), 439 | ('340','WILLIAMSBURG'), 440 | ('341','WILLIAMSTOWN'), 441 | ('342','WILMINGTON'), 442 | ('343','WINCHENDON'), 443 | ('344','WINCHESTER'), 444 | ('345','WINDSOR'), 445 | ('346','WINTHROP'), 446 | ('347','WOBURN'), 447 | ('348','WORCESTER'), 448 | ('349','WORTHINGTON'), 449 | ('350','WRENTHAM'), 450 | ('351','YARMOUTH')]) 451 | 452 | return d[str(number)] 453 | 454 | townnumbers = sys.argv[1:] 455 | if (len(townnumbers) == 0): 456 | townnumbers = range(1,352) 457 | 458 | for townnumber in townnumbers: 459 | exportTown( townnumber,convertMassGISTownNumbersToName(townnumber)) 460 | 461 | 462 | 463 | 464 | 465 | 466 | 467 | 468 | -------------------------------------------------------------------------------- /upload/osmupload.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3 2 | # vim: fileencoding=utf-8 encoding=utf-8 et sw=4 3 | 4 | # Copyright (C) 2009 Jacek Konieczny 5 | # Copyright (C) 2009 Andrzej Zaborowski 6 | # Copyright (C) 2013 Jason Remillard 7 | # 8 | # This program is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by 10 | # the Free Software Foundation; either version 2 of the License. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program; if not, write to the Free Software 19 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 20 | 21 | 22 | """ 23 | Uploads complete osmChange 0.3 or 0.9 files. Use your login (not email) as username. 24 | """ 25 | 26 | __version__ = "$Revision: 21 $" 27 | 28 | import os 29 | import subprocess 30 | import sys 31 | import traceback 32 | import base64 33 | import codecs 34 | import optparse 35 | import time 36 | 37 | import http.client as httplib 38 | import xml.etree.cElementTree as ElementTree 39 | import urllib.parse as urlparse 40 | 41 | class HTTPError(Exception): 42 | pass 43 | 44 | class OSM_API(object): 45 | url = 'http://api06.dev.openstreetmap.org/' 46 | def __init__(self, url, username = None, password = None): 47 | if username and password: 48 | self.username = username 49 | self.password = password 50 | else: 51 | self.username = "" 52 | self.password = "" 53 | self.changeset = None 54 | self.url = url 55 | self.debugLogFile = None 56 | 57 | def __del__(self): 58 | if ( self.debugLogFile ) : 59 | self.debugLogFile.close() 60 | pass 61 | 62 | def debugLogFilename(self,filename) : 63 | if ( self.debugLogFile ) : 64 | self.debugLogFile.close() 65 | self.debugLogFile = None 66 | self.debugLogFile = open( filename,"w") 67 | 68 | def debugMsg(self, mesg): 69 | if ( self.debugLogFile ) : 70 | self.debugLogFile.write(mesg ) 71 | self.debugLogFile.flush 72 | 73 | def request(self, conn, method, url, body, headers, progress): 74 | if progress: 75 | self.debugMsg(" making request %s\n" % (url,)) 76 | conn.putrequest(method, url) 77 | 78 | self.debugMsg(" sending headers\n") 79 | if body: 80 | conn.putheader('Content-Length', str(len(body))) 81 | for hdr, value in headers.items(): 82 | conn.putheader(hdr, value) 83 | 84 | self.debugMsg(" end of headers\n") 85 | conn.endheaders() 86 | 87 | self.debugMsg(" Sending data 0%") 88 | if body: 89 | start = 0 90 | size = len(body) 91 | chunk = size / 100 92 | if chunk < 16384: 93 | chunk = 16384 94 | while start < size: 95 | end = min(size, int(start + chunk)) 96 | conn.send(body[start:end]) 97 | start = end 98 | self.debugMsg(" %2i%%" % (int(start * 100 / size),)) 99 | self.debugMsg("\n") 100 | else: 101 | conn.request(method, url, body, headers) 102 | 103 | def _run_request(self, method, url, body = None, progress = 0, content_type = "text/xml"): 104 | url = urlparse.urljoin(self.url, url) 105 | purl = urlparse.urlparse(url) 106 | if purl.scheme != "http": 107 | raise ValueError("Unsupported url scheme: %r" % (purl.scheme,)) 108 | if ":" in purl.netloc: 109 | host, port = purl.netloc.split(":", 1) 110 | port = int(port) 111 | else: 112 | host = purl.netloc 113 | port = None 114 | url = purl.path 115 | if purl.query: 116 | url += "?" + query 117 | headers = {} 118 | if body: 119 | headers["Content-Type"] = content_type 120 | 121 | try_no_auth = 0 122 | 123 | if not try_no_auth and not self.username: 124 | raise HTTPError(0, "Need a username") 125 | 126 | try: 127 | self.debugMsg(" connecting\n") 128 | conn = httplib.HTTPConnection(host, port) 129 | # conn.set_debuglevel(10) 130 | 131 | if try_no_auth: 132 | self.request(conn, method, url, body, headers, progress) 133 | self.debugMsg(" waiting for status for %s\n" % (url,)) 134 | response = conn.getresponse() 135 | 136 | if not try_no_auth or (response.status == httplib.UNAUTHORIZED and 137 | self.username): 138 | if try_no_auth: 139 | conn.close() 140 | self.debugMsg(" re-connecting\n") 141 | conn = httplib.HTTPConnection(host, port) 142 | # conn.set_debuglevel(10) 143 | 144 | creds = self.username + ":" + self.password 145 | headers["Authorization"] = "Basic " + \ 146 | base64.b64encode(bytes(creds, "utf8")).decode("utf8") 147 | # ^ Seems to be broken in python3 (even the raw 148 | # documentation examples don't run for base64) 149 | self.request(conn, method, url, body, headers, progress) 150 | self.debugMsg(" waiting for status for %s\n" % (url,)) 151 | response = conn.getresponse() 152 | 153 | if response.status == httplib.OK: 154 | self.debugMsg(" reading response\n") 155 | response_body = response.read() 156 | else: 157 | err = response.read() 158 | raise HTTPError(response.status, "%03i: %s (%s)" % ( 159 | response.status, response.reason, err), err) 160 | finally: 161 | conn.close() 162 | return response_body 163 | 164 | def create_changeset(self, created_by, comment,source,bot): 165 | if self.changeset is not None: 166 | raise RuntimeError("Change set already opened") 167 | 168 | self.debugMsg("Creating the change set on %s\n" %( self.url,)) 169 | 170 | root = ElementTree.Element("osm") 171 | tree = ElementTree.ElementTree(root) 172 | element = ElementTree.SubElement(root, "changeset") 173 | ElementTree.SubElement(element, "tag", {"k": "created_by", "v": created_by}) 174 | 175 | if ( comment ) : 176 | ElementTree.SubElement(element, "tag", {"k": "comment", "v": comment}) 177 | 178 | if ( source ) : 179 | ElementTree.SubElement(element, "tag", {"k": "source", "v": source}) 180 | 181 | if ( bot ) : 182 | ElementTree.SubElement(element, "tag", {"k": "bot", "v": "yes"}) 183 | 184 | body = ElementTree.tostring(root, "utf-8") 185 | reply = self._run_request("PUT", "/api/0.6/changeset/create", body) 186 | changeset = int(reply.strip()) 187 | self.debugMsg(" change set id: %i\n" % (changeset)) 188 | self.changeset = changeset 189 | 190 | def upload(self, change): 191 | if self.changeset is None: 192 | raise RuntimeError("Change set not opened") 193 | self.debugMsg("Uploading OSC file\n") 194 | 195 | # add in changeset tag to all of the elements. 196 | for operation in change: 197 | if operation.tag not in ("create", "modify", "delete"): 198 | continue 199 | for element in operation: 200 | element.attrib["changeset"] = str(self.changeset) 201 | 202 | body = ElementTree.tostring(change, "utf-8") 203 | reply = self._run_request("POST", "/api/0.6/changeset/%i/upload" 204 | % (self.changeset,), body) 205 | self.debugMsg(" upload is done.\n") 206 | 207 | return reply 208 | 209 | # Sometimes the server will close a change set on us, need to this function 210 | # to figure out if we should close it when an error is thrown. 211 | def is_open_changeset(self) : 212 | if self.changeset is None: 213 | return False 214 | 215 | self.debugMsg(" checking if change set is still open.\n"); 216 | reply = self._run_request("GET", "/api/0.6/changeset/%i" % (self.changeset,)) 217 | 218 | root = ElementTree.fromstring(reply) 219 | 220 | is_open = False; 221 | 222 | for changeset in root : 223 | if ( changeset.attrib.get('open') == 'true') : 224 | self.debugMsg(" still open\n") 225 | is_open = True; 226 | else : 227 | self.debugMsg(" closed\n") 228 | 229 | self.debugMsg(" done\n") 230 | 231 | return is_open 232 | 233 | def close_changeset(self): 234 | if self.changeset is None: 235 | raise RuntimeError("Change set not opened") 236 | self.debugMsg("Closing change set\n"); 237 | reply = self._run_request("PUT", "/api/0.6/changeset/%i/close" 238 | % (self.changeset,)) 239 | self.changeset = None 240 | self.debugMsg(" done\n") 241 | 242 | def forget_changeset(self) : 243 | self.changeset = None 244 | self.debugMsg(" forgetting change set\n") 245 | 246 | try: 247 | this_dir = os.path.dirname(__file__) 248 | 249 | parser = optparse.OptionParser(description='Upload Open Street Map OSC files.',usage='upload.py [options] uploadfile1.osc [uploadfile2.osc] ...') 250 | parser.add_option('-u','--user',help='OSM Username (not email)',dest='user') 251 | parser.add_option('-p','--password',help='OSM password',dest='password') 252 | parser.add_option('--sleep',help='Sleep between upload files (s)',dest='sleep') 253 | parser.add_option('-m','--comment',help='Sets the changeset comment tag value. Can also use .comment file.',dest='comment') 254 | parser.add_option('--source',help='Sets the changeset source tag value. Can also use .source file',dest='source') 255 | parser.add_option('--server',help='URL for upload server (url,test,live), required.') 256 | parser.add_option('--bot',help='Sets the bot=yes changeset tag. Optional',action='store_true',dest='bot') 257 | (param,filenames) = parser.parse_args() 258 | 259 | if ( not param.server ) : 260 | parser.print_help() 261 | print("\n\nerror: --server flag is required.") 262 | sys.exit(1) 263 | 264 | if ( param.server == 'test') : 265 | param.server = 'http://api06.dev.openstreetmap.org' 266 | elif (param.server == 'live') : 267 | param.server = 'http://api.openstreetmap.org' 268 | 269 | if ( param.user) : 270 | login = param.user 271 | else: 272 | login = input("OSM login: ") 273 | if not login: 274 | sys.exit(1) 275 | 276 | if (param.password) : 277 | password = param.password 278 | else: 279 | password = input("OSM password: ") 280 | if not password: 281 | sys.exit(1) 282 | 283 | bot = False 284 | if (param.bot ) : 285 | bot = True 286 | 287 | api = OSM_API(param.server,login, password) 288 | 289 | changes = [] 290 | for filename in filenames: 291 | 292 | print("%-50s " % (filename ,), end='') 293 | sys.stdout.flush() 294 | 295 | if not os.path.exists(filename): 296 | print("skipped, does not exist." % (filename,)) 297 | continue 298 | 299 | tree = ElementTree.parse(filename) 300 | root = tree.getroot() 301 | if root.tag != "osmChange" or (root.attrib.get("version") != "0.3" and root.attrib.get("version") != "0.6") or not filename.endswith(".osc") : 302 | print("skipped, must be 0.3 or 0.6 OSC file") 303 | continue 304 | 305 | diff_fn = filename[:-4] + ".diff.xml" 306 | status_fn = filename[:-4] + "-status.txt" 307 | 308 | if os.path.exists(status_fn) or os.path.exists(diff_fn) : 309 | print("skipped, already uploaded.") 310 | continue 311 | 312 | comment_fn = filename[:-4] + ".comment" 313 | try: 314 | comment_file = codecs.open(comment_fn, "r", "utf-8") 315 | comment = comment_file.read().strip() 316 | comment_file.close() 317 | except IOError: 318 | comment = param.comment 319 | 320 | source_fn = filename[:-4] + ".source" 321 | try: 322 | source_file = codecs.open(source_fn, "r", "utf-8") 323 | source = source_file.read().strip() 324 | source_file.close() 325 | except IOError: 326 | source = param.source 327 | 328 | api.debugLogFilename( status_fn) 329 | api.create_changeset("osmupload.py v1.0", comment, source,bot) 330 | 331 | try: 332 | diff = api.upload(root) 333 | 334 | api.debugMsg( "Writing %s " % (diff_fn,)); 335 | diff_file = codecs.open(diff_fn, "w", "utf-8") 336 | diff_file.write(diff.decode("utf8")) 337 | diff_file.close() 338 | api.debugMsg( "OK\n"); 339 | 340 | api.close_changeset() 341 | api.debugMsg( "Upload completed successfully\n"); 342 | print("OK") 343 | 344 | except HTTPError as e: 345 | if e.args[0] in [ 404, 409, 412 ] and os.path.exists(diff_fn): 346 | # Merge conflict 347 | os.unlink(diff_fn) 348 | 349 | api.debugMsg( " error code %d\n\n" % (e.args[0],)) 350 | 351 | errstr = e.args[2].decode("utf8") 352 | api.debugMsg( errstr + "\n\n") 353 | if ( api.is_open_changeset()) : 354 | api.close_changeset() 355 | else : 356 | api.forget_changeset() 357 | 358 | print("FAIL") 359 | 360 | sys.stdout.flush() 361 | if ( param.sleep ) : 362 | time.sleep ( float(param.sleep)); 363 | 364 | except HTTPError as err: 365 | sys.stderr.write(err.args[1]) 366 | sys.exit(1) 367 | 368 | except Exception as err: 369 | sys.stderr.write(repr(err) + "\n") 370 | traceback.print_exc(file=sys.stderr) 371 | sys.exit(1) 372 | 373 | 374 | 375 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Open Street Map Importing and Mechanical Edit Toolkit 2 | ================== 3 | 4 | Please read and follow the OSM import guidelines here below. 5 | 6 | - http://wiki.openstreetmap.org/wiki/Import/Guidelines 7 | - http://wiki.openstreetmap.org/wiki/Mechanical_Edit_Policy 8 | - http://wiki.openstreetmap.org/wiki/Automated_Edits_code_of_conduct 9 | 10 | There is a lot of software around for doing mechanical 11 | edits and imports of Open Street Map data. The goal of this repository 12 | is to collect the best scripts written for doing 13 | imports and mechanical edits and to provide technical documentation 14 | on the actual process. 15 | 16 | Table of Contents 17 | ---------------------- 18 | + [The Process](#process) 19 | + [Why Automation Is So Important](#automation) 20 | + [Community](#comm) 21 | + [Technical - Breaking It Down](#tsteps) 22 | + [Obtaining/Fetching Data](#fetch) 23 | + [Loading Data into PostGIS](#loading) 24 | + [OSM Preparation](#prep) 25 | + [Conflating](#conflate) 26 | + [Ongoing Imports](#ongoing) 27 | + [Building Imports](#buildings) 28 | + [Change Set Management](#changeset) 29 | + [Upload Scripts](#code) 30 | + [Environment](#env) 31 | 32 | 33 | The Process 34 | ----------------- 35 | 36 | There are many different ways of doing imports and mechanical edits. 37 | The import documentation on the OSM wiki is necessarily prescriptive to cover 38 | the wide variety of possible imports. The OSM wiki also needs to cover the 39 | wide variety of community norms across OSM. The same import that might 40 | be accepted with open arms in one region and, then be totally unacceptable 41 | in another region. In practice, some regions of OSM will not accept any 42 | kinds of imports or mechanical edits. Also, you need to be aware that some 43 | members of OSM oppose all imports. Therefore, it is a difficult task to 44 | document "one" import policy across all of OSM. 45 | 46 | This documentation is descriptive, like a recipe, that if followed 47 | should provide good results. It is just one path, of many, to success 48 | and does not represent any kind of official OSM policy. 49 | 50 | Import and mechanical edits are hard. They are not hard, like you need 51 | to be a genius to do it, but they are hard because they require an unusual 52 | mix of knowledge. To do an import, I believe you need ... 53 | 54 | - Mastery of the OSM data model. You must know how points, ways, and 55 | relations relate to each other and how they are used in practice to 56 | model the world. You should be able to read and understand an .osm file 57 | opened in a text editor. Get yourself comfortable mapping roads, houses, address, routes, 58 | lakes with islands, point of interests, sidewalks, trees, traffic lights, 59 | benches, trails, turn restrictions, doors, stairs, fences, gates, etc. Try mapping in 60 | all of the editors: JOSM, potlatch, ID, etc. In practice, you need to 61 | do a big pile of normal mapping first. I would suggest at least 150 change sets over 6 months. 62 | 63 | - Ability to do basic programming/scripting. Small imports 64 | can be done quite well with JOSM and QGIS, but when the data gets larger, an 65 | entirely different set of tools ends up being required. Larger imports 66 | become a programming/data processing protect. If programming is not your 67 | thing, I recommend teaming up with somebody. If your data is very small, then you 68 | probably don't need this toolkit. 69 | 70 | - Patience and attention to detail. 71 | 72 | 73 | Community 74 | -------------------------- 75 | 76 | Random hints. 77 | 78 | - If you are not sure about the data license, post a link to the data to the [import mailing list][http://lists.openstreetmap.org/listinfo/imports] and ask for help. 79 | 80 | - Use the [wiki template][http://wiki.openstreetmap.org/wiki/Import/Plan_Outline] to setup your import wiki page. 81 | 82 | - Use the [Who's around me][http://resultmaps.neis-one.org/oooc] to find mappers that might be effected by your import. 83 | 84 | - Use the [Mailing Lists][http://wiki.openstreetmap.org/wiki/Mailing_lists]. 85 | 86 | 87 | Why Automation Is So Important 88 | -------------------------- 89 | 90 | You have hopefully noticed that community feedback is central 91 | to the OSM importing process. In practice this means a bunch of 92 | people asking you to make changes to your output files! If your 93 | work flow is automated, changes can be incorporated quickly, 94 | with consistency, and repeatability. If it takes 3 hours, 95 | and 1000 mouse clicks to produce your output files, it will 96 | be very difficult to openly accept community input. 97 | 98 | OSM itself is always changing. If your process is automated, fresh 99 | OSM data can be imported just before the actual upload, minimizing 100 | the chances of things getting out of sync. 101 | 102 | This is not intuitive. It is easy to think of the import as only 103 | being done once, so it seems like it might not be worth the time 104 | to automate it. However, it is really done 25 times as practice. 105 | Even the final uploading step should be practiced on the 106 | development server. 107 | 108 | The tools in this repository are python scripts, so that you can 109 | fully automate your process. The example scripts are designed to 110 | not accept any command line arguments or do any kind of user 111 | prompting. This insures that the import process is repeatable. 112 | 113 | 114 | Technical - Breaking It Down 115 | ------------------------ 116 | 117 | All imports and mechanical edits, can be broken down to the following steps. 118 | 119 | - Obtaining/fetching external data - Get your hands on some third party 120 | data source. For an edit bot, this step is skipped. 121 | 122 | - Obtaining/fetching OSM data - Most imports and mechanical edits need 123 | to consider the data that is already in the OSM database. OSM data must 124 | be downloaded. 125 | 126 | - Loading data into PostGIS - The data needs to be transformed into a 127 | format that makes it convenient to do the conflation (merging) with 128 | existing OSM data. Normally, this means loading it into a PostGIS database 129 | and getting everything reprojected into the same coordinate system. 130 | 131 | - Conflating - This is the step that merges the existing OSM data, 132 | with the external data, and outputs the changes to be made to OSM. 133 | 134 | - Conversion for uploading - The OSM servers have a limit on the 135 | upload size. The output files will need to be subdivided before uploading. 136 | If your output data is not in an .osc file, then it will need 137 | to be converted before uploading. 138 | 139 | - Uploading - Doing the actual upload to the OSM servers. 140 | 141 | 142 | Obtaining/Fetching Data 143 | ---------------------------- 144 | 145 | Downloading the source and OSM data should be automated. The OSM data will 146 | change over the coarse of the import project and will require re downloading 147 | many times. 148 | 149 | The source data might even even be updated in the middle of the import. If possible, its 150 | downloading should be automated. However, this is not always possible. The data may be 151 | delivered to you on CD, email, or the data provider might have a user 152 | friendly, web 2.0, facebook enabled web site that makes it impossible 153 | to download the data in bulk via a simple url. Just do your best here. 154 | 155 | The OSM data extract downloading can always be automated. The recommended source 156 | for regional OSM extracts is http://download.geofabrik.de/. 157 | 158 | The wget utility can be used to download OSM extracts and external data. 159 | If the data is in a zip file, the standard python environment provides code 160 | to unzip source files. 161 | 162 | In the examples directories please look at the fetchexternaldata.py, 163 | and fetchosmdata.py files for example code. 164 | 165 | 166 | Loading Data into PostGIS 167 | ------------------------------------ 168 | 169 | There are many possible ways of loading .osm files into a PostGIS 170 | database. The two main tools are osmosis and the osm2pgsql. 171 | 172 | osm2pgsql - It can only be used for purely additive imports. 173 | - It will setup the schema itself. 174 | - It will work without having postGIS setup for network access. 175 | - It is fast. 176 | - The schema is very easy to use. 177 | - Defaults to the 900913 coordinate system. 178 | - Lossy, only common tags are imported. 179 | - Can't use osm2pgsql to change existing OSM data. No edit bots and no 180 | complicated conflation logic is possible. It does not have enough 181 | information to directly write out a change file. Only additive 182 | imports can use osm2pgsql. 183 | 184 | Getting osm2pgsql requires the following steps. 185 | 1. Make a postGIS user that is a super user with the same name as the Linux user account. 186 | 2. Make a database owned by the new postGIS user called "gis". 187 | 3. Install PostGIS postgis.sql and spatial_ref_sys to gis database. 188 | 189 | osmosis - Can be used for any kind of importing/bot activity. 190 | - It supports many kind of schema that need to be setup before hand. 191 | - The best schema's for imports, snapshotdb, require an Postgresql extension. 192 | - It is slow to actually import the data. 193 | - It can only talk to PostGIS via a network connection. Even when 194 | running locally, a network loopback connection must be enabled. 195 | - The schema is harder to use. Relations and ways are not mashed together. 196 | - It is not lossy, sophisticated conflation logic is possible. 197 | - Edit bots are possible. 198 | - Uses the 4326 coordinate system. 199 | 200 | Getting osmosis ready to import an extract is more complicated. At 201 | a high level the following steps are required. 202 | 203 | 1. Make a postGIS user that is a super user with the same name as linux user account. 204 | 2. Make a database owned by the new postGIS user called "gis". 205 | 3. Install PostGIS postgis.sql and spatial_ref_sys to gis database. 206 | 4. Install hstore extension to gis database. 207 | 5. Enabled network access in PostGIS. 208 | 6. Setup schema for pgsnapshot to gis database. 209 | 7. Setup linestring support for pgsnapshot to gis database. 210 | 211 | After the OSM extract has been imported, next the external 212 | data must be imported so that you can use the PostGIS SQL operations 213 | between the OSM data and external data. They need in the same 214 | coordinate system. The osm2pgsql tool imports the OSM data as 900913, the 215 | osmosis uses 4326. Before you import the external data it must be 216 | reprojected to either 4326 or 900913. 217 | 218 | There are many formats that the external data might be provided in. But, if 219 | your external data is a .shp (shape) file, then the shp2pgsql tool can be used. 220 | 221 | In the examples directories please look at the postgisloaddata.py for 222 | example code. 223 | 224 | http://wiki.openstreetmap.org/wiki/Database_schema#Database_Schemas. 225 | 226 | 227 | OSM Preparation 228 | ------------------ 229 | 230 | Sometimes it is easier to fix errors and idiosyncrasies in the existing 231 | OSM data, rather than having very complicated conflation logic. 232 | These scripts run QA checks for some common OSM errors that could 233 | otherwise complicate the conflation processing. 234 | 235 | They all require an osm2pgsql PostGIS database, 236 | ogr2ogr, and ogr2osm. 237 | 238 | - prep/findaerowwaywithnobuildings.py - Find all aeroway=terminal areas 239 | that don't also have building=* tag. Building import conflation/merging 240 | logic that is only using building=* tag, will fail on airport terminals 241 | that are just tagged with aeroway=terminal. Use this script to fix the 242 | terminals before the import. 243 | 244 | - prep/findschoolswithnobuildings.py - Find all amenity=school areas 245 | without a building=* tag. Not necessarily an error, but could indicate 246 | that a school building was tagged with amenity=school without the 247 | building tag. Building import conflation/merging logic that is only 248 | using building=* tag will fail on school buildings that are just 249 | tagged with amenity=school. Use this script to inspect, and 250 | fix any incorrectly tagged schools before the import. 251 | 252 | - prep/findoverlappingbuildings.py - Find all overlapping buildings. 253 | After an import containing buildings, run this script to insure that 254 | data was not uploaded twice. It is suggested to run this before a 255 | building/address import to baseline the OSM data in the area. 256 | 257 | - prep/findlandusereservoir.py - Find all of the ways that use 258 | landuse=reservoir without a natural=water tag. The 259 | http://wiki.openstreetmap.org/wiki/Reservoir An image 260 | layer will need to checked for actual water. 261 | 262 | 263 | Conflating 264 | --------------------- 265 | 266 | This is the meat of the import/mechanical edit task. It is hard to talk about 267 | this generically. Each kind of data, buildings, addresses, roads, hydrology, 268 | point of interest, etc have specific issues that need to be handled. 269 | 270 | - QA the output files as much as you can stand. Even an intensive community 271 | review of your sample output files will probably only find half of the problems. 272 | You need to find the other half yourself. 273 | 274 | - Most conflation scripts will encounter ambiguous situations. It is better to be 275 | incomplete, or imprecise than wrong. If something can't be merged correctly 276 | just leave it out. Incomplete is not wrong. Put it into a separate file that 277 | to be merged by hand if you feel it is important to import all of the data. 278 | 279 | - Figuring out how to conflate will require that you have a good handle on 280 | the quality of the external data first. Often different OSM data schema 281 | imply different levels of uncertainties. For example, 282 | putting an address on a doorway node implies a high level 283 | of accuracy, whereas, address interpolation way much less so. Your 284 | output data should have the correct OSM schema that reflects the 285 | actual uncertainty of your source data. Know your external data! 286 | 287 | - The conflation output should try to look like the data was mapped by hand. 288 | In practice, this means shying away from importing data that does not have an 289 | established tagging system. If the data was important to OSM, you can be pretty 290 | sure that a tag (or two) would exist for it by now. If somebody needs it, 291 | they can always go to the same source files you used for your import and 292 | match them to the OSM data. Making the output data look like normal mapping 293 | will improve your chances of getting the import accepted by the community. 294 | 295 | - The import conflation logic should be "stateless". You should 296 | be able to re-run the conflation work flow after you have finished the final upload 297 | and get an empty output file. It also means, you should should be 298 | able to stop half way through your upload, wait 2 months, reprocess everything from scratch 299 | and have the script correctly pick up the remaining work. Sometimes 300 | conflicts with other mappers might not be detected until the actual upload. If the 301 | processing is stateless, you simply ignore the failed upload, and reprocess 302 | the data again at a later date. 303 | 304 | In the examples directories please look at the conflate.py for 305 | example code. The docs directory has documentations specific to 306 | different kinds of imports. 307 | 308 | 309 | Ongoing Imports 310 | --------------------- 311 | 312 | Ongoing imports of things like bus stops, gas stations, and all kinds 313 | of other POI's are basically an advanced version of the conflation problem. 314 | The goal is to keep OSM synchronized with an external source data 315 | and at the same time be robust and considerate of normal OSM mapping activity. 316 | 317 | All of the following will happen to the POI nodes you are trying to maintain. 318 | 319 | - POI node is deleted. 320 | - POI node is put into a relation. 321 | - POI node is deleted and all the maintained tags are merged into another node, perhaps 322 | a building node. 323 | - POI node is deleted, and all the maintained tags are merged into a way, perhaps a 324 | building way, or parking way. 325 | - POI node is deleted, and all the tags are merged into a relation. 326 | - New tags are added to your POI node. 327 | - POI node is moved a 1 meter 328 | - POI node is moved a 10 meters. 329 | - POI node is moved a 1000 meters. 330 | - POI node is moved a 10000 meters. 331 | - POI tags you are maintaining are changed in conflict with your source 332 | data. 333 | 334 | The following high level guidelines should be considered: 335 | 336 | The most important rule, is that the posture of the code 337 | should be deferential to normal mappers. Aggressive and/or 338 | rude behavior from import code (aka bots), will not be 339 | tolerated by the community. 340 | 341 | Strive to preserve edits made by others. That includes changes to the 342 | position, adding tags, merging, etc. 343 | 344 | Preserve the existing OSM node/way ID when edits are made. 345 | Don't delete then add the entity, or higher level relations and 346 | ways will be broken. Deleting and adding also makes it very 347 | difficult to understand change set history of a specific entity. 348 | 349 | The code should pay attention to who last touched the data. 350 | The code can be aggressive if nobody else has touched the data except 351 | itself. However, if non-bot user id is on the data, the code should be 352 | conservative with conflation issues. 353 | 354 | The code should skip anything that looks weird and get a human 355 | involved. There are some scenarios that will need to get resolved by 356 | sending an message to another mapper and discussing the changes. 357 | Sometimes, you will just need to fix the map manually because it was 358 | broken by accident. Supporting human intervention should be 359 | directly supported by the code. Be sure to code and test against 360 | partial updates. 361 | 362 | At this point you might be saying, no worries, all of these issues are 363 | what primary keys were invented for! Take a persistent ID from the source 364 | data and put it into OSM via a custom tag. However using these ids might 365 | make things more complicated because they are also not immune to normal 366 | edits. With the ID's you now need to code against the following 367 | additional situations in combination of the issues already listed. 368 | 369 | - The ref:yourimportname is copied to several new POI's you are not maintaining. 370 | - The ref:yourimportname is changed to a new random value. 371 | - The ref:yourimportname tag is deleted, POI node otherwise stays the same. 372 | - POI node is deleted and some tags (not your ref) are merged into another node. 373 | - POI node is deleted and some tags (not your ref) are merged into another way. 374 | 375 | You can't 100% trust your ref:yourimportname, it will be copied, deleted, and 376 | generally messed with. Don't think of it like a db primary key, it is 377 | more like a hint, and not the primary identity of the entity. 378 | 379 | The most robust code will need to treat the location and the maintained 380 | tags as the authoritative identity of the POI entity. Some people who 381 | have experience with updating OSM data via an external ID have decided 382 | that ignoring any ID's results in the most robust solution. 383 | 384 | Developing code for ongoing updates is complicated, the dev API test 385 | server should be heavily utilized to debug and test the code. 386 | 387 | This project is for doing on-going imports. 388 | 389 | https://github.com/insideout10/open-street-map-importer 390 | 391 | Please, let me know if other code exists. 392 | 393 | 394 | Building Imports 395 | ------------------------- 396 | 397 | Merging/Conflating 398 | 399 | The merging of building data into OSM is pretty straight forward in theory. 400 | Basically, don't import the building if it overlaps with a building already 401 | in OSM. 402 | 403 | The SQL to do this looks like this (this also de-nodes to 0.2 meters). 404 | 405 | select 406 | ST_SimplifyPreserveTopology(the_geom,0.20),'yes' as building 407 | from 408 | massgis_structures 409 | where 410 | not exists 411 | (select * 412 | from 413 | planet_osm_polygon as osm 414 | where 415 | osm.building != '' and ST_Intersects(osm.way,massgis_structures.the_geom)) 416 | 417 | 418 | The ST_Intersects call is not foolproof. There are at least three 419 | classes of buildings that sometimes are mapped without buildings 420 | tags: schools (amenity=school), parking garages (amenity=parking), 421 | and airport terminals (aeroway=terminal). Without the building tag, 422 | the SQL overlap check fails. The OSM data needs to be prepared by 423 | hand before the actual import is run to insure the conflation 424 | script works 100% of the time. 425 | 426 | The schools, parking lots, and airport terminals can also be found 427 | and inspected by using the JOSM mirror plugin via an overpass 428 | query, or a XAPI query, or the following SQL. 429 | 430 | select way 431 | from planet_osm_polygon as way1 432 | where 433 | way1.amenity = ''school'' and 434 | not exists 435 | (select * 436 | from 437 | planet_osm_polygon as way2 438 | where 439 | way2.building != '' and ST_Intersects(way1.way,way2.way))" 440 | 441 | 442 | The most technically difficult aspect of a building import is dealing 443 | with buildings that are touching each other or are very close together. 444 | Some data sources will merge the buildings together into one footprint/shape. 445 | This is the easy case, and if directly imported will be acceptable in 446 | OSM. However, if the buildings are represented by different shape 447 | primitives, then things get more complicated. Consider the case of two 448 | buildings that are touching each other, but are different sizes. In the 449 | source data, the two buildings are probably represented by two different 450 | shapes. However, in OSM, those buildings needs to be merged together so that 451 | the ways share nodes. 452 | 453 | This import has PostGIS code that correctly deals with the overlapping buildings. 454 | 455 | http://wiki.openstreetmap.org/wiki/Baltimore_Buildings_Import 456 | 457 | 458 | 459 | Change Set Management 460 | ----------------- 461 | 462 | There is a 50K entity limit on individual change sets. Every node, way 463 | and relation counts as an entity. JOSM is a fantastic tool, however please 464 | don't use it for uploading change sets that are larger than 50,000 elements. Why? 465 | 466 | - JOSM cuts the OSM files exactly at the limit of change set size, it 467 | makes the change set un-revertible with the JOSM revert plugin. 468 | 469 | - JOSM sorts the upload so that all nodes are uploaded first, then ways, 470 | then relations. If the upload fails before you finish, you can have 50,000 471 | untagged nodes uploaded, with no ways, leaving a big mess. 472 | 473 | Another issue to be aware of is that it is tricky to break up a large data set by 474 | hand in JOSM. Its copy and paste mechanism is very literal, unless you are very 475 | careful, your new data layer will be missing dependent relations or its dependent 476 | nodes 477 | 478 | If you have more than 50,000 entities to upload, don't use JOSM to upload them 479 | directly and don't attempt to split up the data using JOSM. Use the script 480 | in this repository, or write it directly into your conflation script. 481 | 482 | The question is sometimes asked, if 50,000 elements are too many, how many 483 | elements should a change set have? If you have 10,000 elements change sets 484 | be prepared for about 5% the uploads to fail. If you are at 2,000 elements, 485 | very few of the uploads will fail, much less than 1%. 486 | 487 | However, more important than the failure percentage, is insuring 488 | that a failure does not leave the data in OSM broken. The only way to 489 | insure that, is to make the data inside a change sets stand alone. If the 490 | import has more than one change set, they should be able to be 491 | uploaded in any order. If you are doing an import that has heavily 492 | connected data, like roads or rivers, then it would be better to have 493 | larger change sets, rather than breaking up a connected data set. 494 | 495 | Need to talk about the following.... 496 | 497 | - change set tags 498 | - development server 499 | - revert JOSM plugin 500 | - conflicts 501 | 502 | 503 | Upload Scripts 504 | ------------------ 505 | 506 | The following scripts are included in the repository. 507 | 508 | - upload/osm2osc.py - Many of the utilities that convert external data to 509 | OSM write out an .osm file. However, only .osc files can be uploaded. 510 | This simple utility converts an .osm to .osc. This is basically 511 | unmodified from the version in OSM SVN. 512 | 513 | - upload/osmsplit.py - Breaks an .osc file into several smaller .osc files. 514 | The OSM API has a 50,000 element limit for a single change set. In practice, 515 | the change sets should be kept much smaller, around 10,000 elements. Therefor, 516 | before uploading large .osc files, they need to be broken up. The smaller 517 | .osc files generated by this script are self contained without cross 518 | dependencies. The lack of dependencies means files can be uploaded in any 519 | order and errors in the uploads can be easily handled. 520 | 521 | - upload/osmupload.py - Uploads a set of .osc files into a server that 522 | supports the OSM API. This is a heavily modified version of the upload.py 523 | script that is hosted on the OSM SVN server. It keeps a log of the uploads, 524 | supports testing uploads to the development server, does not use "chunks", 525 | so that each .osc file upload is atomic, allows comment and source tags 526 | on the change set to be pulled from files, will not double upload a file, 527 | and will try hard to always close out a change set if an error occurs. 528 | The log holds the change set ids, so it is possible to review an already 529 | completed upload. 530 | 531 | 532 | Environment 533 | ----------------- 534 | 535 | This toolkit requires many other tools. Many are included standard 536 | in Debian Linux and are trivial to install. If Windows is your 537 | primary operating system, it is recommended to install Debian Linux 538 | into a virtual machine, like Oracle's busybox http://www.busybox.net/, 539 | and run the tools inside of the virtual machine. 540 | 541 | - PostGIS - http://postgis.org/ - PostGIS is an extension to 542 | PostgreSQL that adds support for common GIS operations via SQL. 543 | When the import or mechanical edit needs to make decisions based 544 | on existing OSM data the processing is done in SQL rather than 545 | processing the .osm/.xml file directly. If possible, use the default 546 | database name of gis. 547 | 548 | - osm2pgsql - Very simple way of importing OSM extract to a PostGIS 549 | database. 550 | 551 | - osmosis - Used to import OSM extracts, make custom extracts, and filter 552 | OSM files by tag. 553 | 554 | - ogr2ogr - http://www.gdal.org/ - Is part of the gdal project. 555 | It it used to re-project source data and to extract data back out 556 | from a PostGIS database. Currently gdal does not support the 557 | .osm file well. This toolkit will use ogr2osm with .shp (shape) 558 | files, then use ogr2osm to convert from a .shp file to .osm file. 559 | 560 | - ogr2osm - https://github.com/pnorman/ogr2osm - This utility 561 | converts .shp to .osm files. It is currently not packaged as 562 | part of Debian. It will need to be installed manually. You 563 | have the option of installing it anywhere and adding to your 564 | path, or installing it in ./ogr2osm. Where ./ is the root of 565 | this repository. The tools will first check if it is ./ogr2osm, 566 | then try the path. To "install" it into ./org2osm run the 567 | following git command. 568 | 569 | git clone https://github.com/pnorman/ogr2osm.git 570 | 571 | 572 | --------------------------------------------------------------------------------