├── color-slope.txt ├── .gitignore ├── rename-files-gis-friendly.sh ├── get-extent.sh ├── project-wgs84.sh ├── project-mercator.sh ├── project-web-mercator.sh ├── import-shp-to-pg.sh ├── import-shp2pgsql.sh ├── clip-raster-to-shp.sh ├── geojson-to-shp.sh ├── shp-to-geojson.sh ├── clip-to-polygon.sh ├── merge.sh ├── grep-find-replace.sh ├── make-hillshades.sh ├── gpx-to-shp.sh ├── clip-extent-project.sh ├── separate-roads-by-type-skeletron.sh ├── separate-roads-by-type.sh └── README.md /color-slope.txt: -------------------------------------------------------------------------------- 1 | 0 255 255 255 2 | 90 0 0 0 3 | 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *test.sh 3 | separate-roads-by-type-skeletron.sh 4 | tmp/ 5 | -------------------------------------------------------------------------------- /rename-files-gis-friendly.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ls | while read -r FILE 4 | do 5 | mv -v "$FILE" `echo $FILE | tr ' ' '_' | tr -d '[{}(),\!]' | tr -d "\'" | tr '[A-Z]' '[a-z]' | sed 's/_-_/_/g'` 6 | done 7 | -------------------------------------------------------------------------------- /get-extent.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SHPFILE=$1 3 | BASE=`basename $SHPFILE .shp` 4 | EXTENT=`ogrinfo -so $SHPFILE $BASE | grep Extent \ 5 | | sed 's/Extent: //g' | sed 's/(//g' | sed 's/)//g' \ 6 | | sed 's/ - /, /g'` 7 | EXTENT=`echo $EXTENT | awk -F ',' '{print $1 " " $4 " " $3 " " $2}'` 8 | echo $EXTENT -------------------------------------------------------------------------------- /project-wgs84.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # reprojects a directory of .shp files to WGS84 3 | 4 | TSRS=$1 # data source's current CRS (may improve accuracy of OGR if specified) 5 | 6 | for FILE in *.shp 7 | do 8 | echo "Transforming $FILE file..." 9 | FILENEW=`echo $FILE | sed "s/.shp/_4326.shp/"` 10 | ogr2ogr \ 11 | -f "ESRI Shapefile" -s_srs "EPSG:3785" -t_srs "EPSG:4326" \ 12 | $FILENEW $FILE 13 | 14 | done 15 | -------------------------------------------------------------------------------- /project-mercator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # reprojects a directory of .shp files to world mercator EPSG:3395 3 | 4 | TSRS=$1 # data source's current CRS (may improve accuracy of OGR if specified) 5 | 6 | for FILE in *.shp 7 | do 8 | echo "Transforming $FILE file..." 9 | FILENEW=`echo $FILE | sed "s/.shp/_3395.shp/"` 10 | ogr2ogr \ 11 | -f "ESRI Shapefile" \ 12 | -s_srs $TSRS \ 13 | -t_srs "EPSG:3395"\ 14 | $FILENEW $FILE 15 | 16 | done 17 | -------------------------------------------------------------------------------- /project-web-mercator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # reprojects a directory of .shp files to web-mercator EPSG:3857 3 | 4 | TSRS=$1 # data source's current CRS (may improve accuracy of OGR if specified) 5 | 6 | for FILE in *.shp 7 | do 8 | echo "Transforming $FILE file..." 9 | FILENEW=`echo $FILE | sed "s/.shp/_3857.shp/"` 10 | ogr2ogr \ 11 | -f "ESRI Shapefile" \ 12 | -s_srs $TSRS \ 13 | -t_srs "EPSG:3857" \ 14 | $FILENEW $FILE 15 | 16 | done 17 | -------------------------------------------------------------------------------- /import-shp-to-pg.sh: -------------------------------------------------------------------------------- 1 | USER='clhenrick' 2 | HOST='localhost' 3 | DB='grand_circle' 4 | PORT='5432' 5 | PW='' 6 | 7 | for FILE in *.shp 8 | do 9 | echo "importing $FILE to $DB..." 10 | BASE=`basename $FILE .shp` 11 | ogr2ogr -f 'PostgreSQL' \ 12 | -skipfailures \ 13 | PG:"host=$HOST \ 14 | user=$USER \ 15 | port=$PORT \ 16 | dbname=$DB \ 17 | password=$PW" \ 18 | $FILE -nln $BASE 19 | 20 | done 21 | -------------------------------------------------------------------------------- /import-shp2pgsql.sh: -------------------------------------------------------------------------------- 1 | # import ESRI shapefiles into PostgreSQL using shp2pgsql 2 | 3 | # change these variables to your liking 4 | # database name 5 | DB='grand_circle' 6 | #character encoding 7 | ENCODING='LATIN1' 8 | #target projection 9 | TSRS='4326' 10 | 11 | for FILE in *.shp 12 | do 13 | echo "importing $FILE to $DB..." 14 | BASE=`basename $FILE .shp` 15 | shp2pgsql -W $ENCODING \ 16 | -I -s $TSRS \ 17 | $FILE $BASE $DB \ 18 | | psql -d $DB 19 | 20 | done -------------------------------------------------------------------------------- /clip-raster-to-shp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | SHPFILE=$1 3 | BASE=`basename $SHPFILE .shp` 4 | EXTENT=`ogrinfo -so $SHPFILE $BASE | grep Extent \ 5 | | sed 's/Extent: //g' | sed 's/(//g' | sed 's/)//g' \ 6 | | sed 's/ - /, /g'` 7 | EXTENT=`echo $EXTENT | awk -F ',' '{print $1 " " $4 " " $3 " " $2}'` 8 | echo "Clipping to $EXTENT" 9 | RASTERFILE=$2 10 | gdal_translate -projwin $EXTENT -of GTiff $RASTERFILE /tmp/`basename $RASTERFILE`_bbclip.tif 11 | gdalwarp -co COMPRESS=DEFLATE -co TILED=YES -of GTiff \ 12 | -r lanczos -cutline $SHPFILE \ 13 | /tmp/`basename $RASTERFILE`_bbclip.tif /tmp/`basename $RASTERFILE`_shpclip.tif -------------------------------------------------------------------------------- /geojson-to-shp.sh: -------------------------------------------------------------------------------- 1 | # this script will export all .geojson files in a directory to .shp files in a specified subdirectory 2 | # ************************************************************************************************************ 3 | #!/bin/bash 4 | 5 | NEWDIR="shp/" 6 | mkdir $NEWDIR # creates new subdirectory 7 | for FILE in *.geojson # cycles through all files in directory (case-sensitive!) 8 | do 9 | echo "converting file: $FILE..." 10 | FILENEW=`echo $FILE | sed "s/.geojson/.shp/"` # replaces old filename 11 | ogr2ogr \ 12 | -f "ESRI Shapefile" \ 13 | $NEWDIR$FILENEW $FILE 14 | done 15 | exit 16 | -------------------------------------------------------------------------------- /shp-to-geojson.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # this script will export all .shp files in a directory to .geojson files in a specified subdirectory 3 | # ************************************************************************************************************ 4 | 5 | NEWDIR=${2:-"$1/geojson/"} 6 | for FILE in ${1:-}*.shp # cycles through all files in directory (case-sensitive!) 7 | do 8 | SIZE=$(ls -al "$FILE" | awk '{print $5}') 9 | echo "$SIZE bytes" 10 | if [ "$SIZE" -gt "30000000" ] 11 | then 12 | echo "WARNING: Skipping $FILE because it's pretty big and GitHub might complain!" 13 | continue 14 | fi 15 | FILENEW=`echo | basename $FILE | sed "s/.shp/.geojson/"` # replaces old filename 16 | echo "converting file: $FILE into $FILENEW..." 17 | ogr2ogr \ 18 | -f 'GeoJSON' \ 19 | -t_srs 'EPSG:4326' \ 20 | $NEWDIR$FILENEW $FILE 21 | done 22 | exit 23 | -------------------------------------------------------------------------------- /clip-to-polygon.sh: -------------------------------------------------------------------------------- 1 | # clip all data in a directory to a specified polygon 2 | # note: important! the clipping polygon and data being clipped must be in the same projection. 3 | 4 | #!/bin/bash 5 | 6 | # assign variable values for OGR parameters here: 7 | CLIP='../shp_base_vector/00_clip_bounds.shp' # the shapefile name of your clipping polygon 8 | 9 | # set path for output.shp files here 10 | DIR_OUT='/Users/chrislhenrick/Cartography/projects/rHenrick/mike_pct_map/data/shp_base_vector/' 11 | 12 | for FILE in *.shp 13 | do 14 | echo "Clipping $FILE file to $CLIP..." 15 | FILENEW=`echo $FILE | sed "s/.shp/_clip.shp/"` 16 | ogr2ogr \ 17 | -clipsrc $CLIP \ 18 | $FILENEW $FILE 19 | 20 | done 21 | 22 | # move output files to a desired location (separate from original data) 23 | for FILE in *_clip.* 24 | do 25 | echo "moving $FILE file..." 26 | mv $FILE $DIR_OUT 27 | 28 | done 29 | 30 | exit 31 | 32 | -------------------------------------------------------------------------------- /merge.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Merges a directory of shapefiles into one. 3 | # caveats: shapefiles do not support multiple geometry types, 4 | # so all shapefiles must be of the same geometry. 5 | 6 | FILE='nl_merged.shp' # name of file to be merged to 7 | LAYER='nl_merged' # must be same as above but without `.shp` extension 8 | SSRS='EPSG:2227' # source CRS 9 | TSRS='EPSG:2227' # target CRS 10 | 11 | for i in $(ls *.shp) 12 | do 13 | 14 | if [ -f "$FILE" ] 15 | then 16 | echo "transforming and merging $i..." 17 | ogr2ogr \ 18 | -f 'ESRI Shapefile' \ 19 | -s_srs $SSRS \ 20 | -t_srs $TSRS \ 21 | -update -append $FILE $i \ 22 | -nln $LAYER 23 | else 24 | echo "creating $FILE..." 25 | ogr2ogr \ 26 | -f 'ESRI Shapefile' \ 27 | -s_srs $SSRS \ 28 | -t_srs $TSRS \ 29 | $FILE $i 30 | fi 31 | done 32 | -------------------------------------------------------------------------------- /grep-find-replace.sh: -------------------------------------------------------------------------------- 1 | # ***************************************************************************************** 2 | # find_and_replace_in_files.sh 3 | # This script does a recursive, case sensitive directory search and replace of files 4 | # To make a case insensitive search replace, use the -i switch in the grep call 5 | # uses a startdirectory parameter so that you can run it outside of specified directory - else this script will modify itself! 6 | # ***************************************************************************************** 7 | 8 | #!/bin/bash 9 | # **************** Change Variables Here ************ 10 | startdirectory='/Users/chrislhenrick/Cartography/projects/eHeinze/henry-cowell-sp/data/zz_tmp' 11 | searchterm='src="place-holder.png"' 12 | replaceterm='src="images/place-holder.png"' 13 | # ********************************************************** 14 | 15 | echo "******************************************" 16 | echo "* Search and Replace in Files Version .1 *" 17 | echo "******************************************" 18 | 19 | for file in $(grep -l -R $searchterm $startdirectory) 20 | do 21 | sed -e "s/$searchterm/$replaceterm/ig" $file > /tmp/tempfile.tmp 22 | mv /tmp/tempfile.tmp $file 23 | echo "Modified: " $file 24 | done 25 | 26 | echo " *** Yay! All Done! *** " -------------------------------------------------------------------------------- /make-hillshades.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Takes a Digital Elevation Model (DEM) and generates hillshades from 4 different light angles and a slope shade 3 | # These may then be composited in QGIS, TileMill, Photoshop, etc. 4 | # Note: Process DEM prior to running this script (mosaic, clip, resample, reproject, etc) 5 | 6 | GFLT=$1 #must be a raster DEM file type supported by GDAL 7 | Z=1.3 # vertical exaggeration factor. apply greater value for smaller scale / larger areas 8 | 9 | echo "Generating hillshade from $GFLT with sunlight angle at 45˚..." 10 | gdaldem hillshade -of 'GTiff' -z $Z -az 45 $GFLT hillshade_az45.tif 11 | 12 | echo "Generating hillshade from $GFLT -z $Z with sunlight angle at 135˚..." 13 | gdaldem hillshade -of 'GTiff' -z $Z -az 135 $GFLT hillshade_az135.tif 14 | 15 | echo "Generating hillshade from $GFLT -z $Z with sunlight angle at 225˚..." 16 | gdaldem hillshade -of 'GTiff' -z $Z -az 225 $GFLT hillshade_az225.tif 17 | 18 | echo "Generating hillshade from $GFLT -z $Z with sunlight angle at 315˚..." 19 | gdaldem hillshade -of 'GTiff' -z $Z -az 315 $GFLT hillshade_az315.tif 20 | 21 | echo "Generating Slope from $GFLT..." 22 | gdaldem slope $GFLT slope.tif 23 | 24 | echo "Making color-slope.txt..." 25 | touch color-slope.txt && printf '%s\n%s\n' '0 255 255 255' '90 0 0 0' >> color-slope.txt 26 | 27 | echo "Creating slope shade..." 28 | gdaldem color-relief slope.tif color-slope.txt slopeshade.tif 29 | -------------------------------------------------------------------------------- /gpx-to-shp.sh: -------------------------------------------------------------------------------- 1 | # this script will clean up gpx data for rendering in illustrator with mapublisher. 2 | # separate gpx data into one shapefile for lines, a shapefile for waypoints, and a shapefile for track points, 3 | # and project data to a desired CRS 4 | # ************************************************************************************************************ 5 | #!/bin/bash 6 | 7 | # script variables assigned as follows: 8 | OUTDR='/Users/chrislhenrick/Cartography/projects/rHenrick/mike_pct_map/data/shp_gps_waypoint_data' # ouput directory here 9 | TSRS='EPSG:4326' # target CRS EPSG code here 10 | PRJ=`echo $TSRS | sed s/\"//g | cut -f2 -d ':'` # variable values for sed here: 11 | 12 | # extract tracks and rename file 13 | for FILE in *.gpx 14 | do 15 | echo "converting $FILE file's tracks..." 16 | FILENEW=`echo $FILE | sed "s/.gpx/_tracks_$PRJ.shp/"` 17 | ogr2ogr \ 18 | -f "ESRI Shapefile" \ 19 | -t_srs $TSRS \ 20 | $FILENEW $FILE \ 21 | tracks 22 | 23 | done 24 | 25 | # extract waypoints and rename file 26 | for FILE in *.gpx 27 | do 28 | echo "converting $FILE file's waypoints..." 29 | FILENEW=`echo $FILE | sed "s/.gpx/_waypoints_$PRJ.shp/"` 30 | ogr2ogr \ 31 | -f "ESRI Shapefile" \ 32 | -t_srs $TSRS \ 33 | $FILENEW $FILE \ 34 | waypoints 35 | 36 | done 37 | 38 | # extract track points and rename file 39 | for FILE in *.gpx 40 | do 41 | echo "converting $FILE file's waypoints..." 42 | FILENEW=`echo $FILE | sed "s/.gpx/_track-points_$PRJ.shp/"` 43 | ogr2ogr \ 44 | -f "ESRI Shapefile" \ 45 | -t_srs $TSRS \ 46 | $FILENEW $FILE \ 47 | track_points 48 | 49 | done 50 | 51 | # move files into an output directory 52 | for FILE in *$PRJ* 53 | do 54 | echo "moving $FILE to $OUTDR..." 55 | mv $FILE $OUTDR 56 | 57 | done 58 | 59 | exit -------------------------------------------------------------------------------- /clip-extent-project.sh: -------------------------------------------------------------------------------- 1 | # this script clips all data in a directory to a bounding box coordinates, projects clipped data to a desired SRS, 2 | # moves processed data to a new location, and removes unprojected clipped data. 3 | # I find this useful for cartography when working with Natural Earth Data, see http://www.naturalearthdata.com/ 4 | # ************************************************************* 5 | 6 | #!/bin/bash 7 | 8 | # set variables for bounding box coordinates here -74.701348 41.656843 -72.898654 40.295932 9 | # minimum latitude (min y value) 10 | LAT_MIN=40.295932 11 | # minimum longitude ( min x value) 12 | LON_MIN=-74.701348 13 | # maximum latitude (max y value) 14 | LAT_MAX=41.656843 15 | # maximum longitude (max x value) 16 | LON_MAX=-72.898654 17 | 18 | # set variables for -s_srs and -t_srs here 19 | # note: EPSG codes do not need to be surrounded by double quotes 20 | S_SRS='EPSG:4326' #data's source coordinate system / projection 21 | T_SRS='EPSG:32118' #data's target coordinate system / projection 22 | 23 | # set path for output .shp files here 24 | DIR_OUT='/Users/chrislhenrick/Cartography/projects/funStuff/nyc_data/data/shp/ocean' 25 | 26 | # variable values for sed here: 27 | PRJ=`echo $T_SRS | sed s/\"//g | cut -f2 -d ':'` 28 | 29 | # step 1: clip all shapefiles to specified lat lon bounding coordinates (x min y min x max y max) 30 | # note: data must be in WGS84, if in a different projection use that SRS' coordinate values 31 | # ************************************************************* 32 | 33 | for FILE in *.shp 34 | do 35 | echo "Clipping $FILE file..." 36 | FILENEW=`echo $FILE | sed "s/.shp/_clip.shp/"` 37 | ogr2ogr \ 38 | -clipsrc $LON_MIN $LAT_MIN $LON_MAX $LAT_MAX \ 39 | $FILENEW $FILE 40 | 41 | done 42 | 43 | # step2: project all clipped data to a desired SRS such as EPSG:26904 44 | # ************************************************************* 45 | # frequently used projections: 46 | # 3395 wgs84 world mercator 47 | # 900913 google web-mercator 48 | # 3857 pseudo mercator (depreciated) 49 | 50 | # project all clipped data to a desired SRS such as EPSG:3395 WGS84 World Mercator 51 | for FILE in *_clip.shp 52 | do 53 | echo "Projecting $FILE file..." 54 | FILENEW=`echo $FILE | sed "s/.shp/_$PRJ.shp/"` 55 | ogr2ogr \ 56 | -s_srs $S_SRS -t_srs $T_SRS \ 57 | $FILENEW $FILE\ 58 | 59 | done 60 | 61 | # step3: move all clipped and projected files to the project folder. 62 | # ************************************************************* 63 | 64 | for FILE in *_$PRJ.* 65 | do 66 | echo "moving $FILE file..." 67 | mv $FILE $DIR_OUT 68 | 69 | done 70 | 71 | # step 4: clean up. delete all clipped, unprojected files still in WGS84 (original coordinate system) that won't be needed. 72 | # ************************************************************* 73 | 74 | for FILE in *_clip.* 75 | do 76 | echo "deleting $FILE file..." 77 | rm $FILE 78 | 79 | done -------------------------------------------------------------------------------- /separate-roads-by-type-skeletron.sh: -------------------------------------------------------------------------------- 1 | # separate a roads GeoJSON layer into multiple shapefile layers using ogr2ogr's sql and geojson driver 2 | # takes a geojson file extracted using https://github.com/migurski/Skeletron 3 | # processes spatial data into multiple road shapefiles based on the skeletron.py output 'highway' field values 4 | # ***************************************************************************** 5 | 6 | #!/bin/bash 7 | 8 | # assign variable values for OGR parameters here: 9 | ROADS='central_az_z15_w12.json' # the name of your OSM roads layer file, must have a type field name 10 | Z='z15' # zoom level skeletron was set to 11 | TSRS='EPSG:2223' # the EPSG code you want your data projected to 12 | PRJ=`echo $TSRS | sed s/\"//g | cut -f2 -d ':'` # takes the numbers of your EPSG code and appends to the output.shp name 13 | 14 | # set path for output.shp files here 15 | DIR_OUT='/Users/chrishenrick/Desktop/DATA/Arizona/OSM/central_az_extracts/roads_generalized/' 16 | 17 | # queries out motorways 18 | for FILE in $ROADS 19 | do 20 | echo "processing $FILE file into motorways..." 21 | FILENEW=`echo $FILE | sed "s/$ROADS/motorways_$PRJ_gen_$Z.shp/"` 22 | ogr2ogr \ 23 | -f "ESRI Shapefile" \ 24 | -sql "select * from OGRGeoJSON where highway in ('motorway', 'trunk')" \ 25 | -t_srs $TSRS \ 26 | $FILENEW $FILE\ 27 | 28 | done 29 | 30 | # queries out main roads (primary, secondary, tertiary) 31 | for FILE in $ROADS 32 | do 33 | echo "processing $FILE file into main roads..." 34 | FILENEW=`echo $FILE | sed "s/$ROADS/main-rd_$PRJ_gen_$Z.shp/"` 35 | ogr2ogr \ 36 | -f "ESRI Shapefile" \ 37 | -sql "select * from OGRGeoJSON where highway in ('primary','secondary','tertiary')" \ 38 | -t_srs $TSRS \ 39 | $FILENEW $FILE\ 40 | 41 | done 42 | 43 | # queries out residential and other smaller roads 44 | for FILE in $ROADS 45 | do 46 | echo "processing $FILE file in other roads..." 47 | FILENEW=`echo $FILE | sed "s/$ROADS/other-rd_$PRJ_gen_$Z.shp/"` 48 | ogr2ogr \ 49 | -f "ESRI Shapefile" \ 50 | -sql "select * from OGRGeoJSON where highway in ('residential', 'minor', 'abandoned', 'rest_area', 'service', 'living_street', 'unclassified')" \ 51 | -t_srs $TSRS \ 52 | $FILENEW $FILE\ 53 | 54 | done 55 | 56 | # queries out dirt roads 57 | for FILE in $ROADS 58 | do 59 | echo "processing $FILE file into dirt roads..." 60 | FILENEW=`echo $FILE | sed "s/$ROADS/dirt-rd_$PRJ_gen_$Z.shp/"` 61 | ogr2ogr \ 62 | -f "ESRI Shapefile" \ 63 | -sql "select * from OGRGeoJSON where highway = 'track'" \ 64 | -t_srs $TSRS \ 65 | $FILENEW $FILE\ 66 | 67 | done 68 | 69 | # queries out paths, trails, cycling paths, etc. 70 | for FILE in $ROADS 71 | do 72 | echo "processing $FILE file into trails..." 73 | FILENEW=`echo $FILE | sed "s/$ROADS/trails_$PRJ_gen_$Z.shp/"` 74 | ogr2ogr \ 75 | -f "ESRI Shapefile" \ 76 | -sql "select * from OGRGeoJSON where highway in ('bridleway', 'cycleway', 'footway', 'path', 'platform', 'pedestrian', 'steps')" \ 77 | -t_srs $TSRS \ 78 | $FILENEW $FILE\ 79 | 80 | done 81 | 82 | # move output files to a desired location (separate from original data) 83 | for FILE in *_$PRJ_gen_$Z.* 84 | do 85 | echo "moving $FILE file..." 86 | mv $FILE $DIR_OUT 87 | 88 | done 89 | 90 | exit -------------------------------------------------------------------------------- /separate-roads-by-type.sh: -------------------------------------------------------------------------------- 1 | # separate a roads shapefile layer into multiple layers using ogr2ogr -sql 2 | # takes a roads.shp file extracted using: http://extract.bbbike.org/# 3 | # processes into multiple road shapefiles based on the OSM 'type' attributes 4 | # ***************************************************************************** 5 | 6 | #!/bin/bash 7 | 8 | # assign variable values for OGR parameters here: 9 | ROADS='roads.shp' # the name of your OSM roads layer file, must have a type field name and be named roads.shp 10 | TSRS='EPSG:2227' # the EPSG code you want your data projected to 11 | PRJ=`echo $TSRS | sed s/\"//g | cut -f2 -d ':'` # takes the numbers of your EPSG code and appends to the output file name 12 | 13 | # set path for output files here 14 | DIR_OUT='/Users/chrislhenrick/Cartography/projects/eHeinze/henry-cowell-sp/data/roads/processed/' 15 | 16 | # queries out motorways 17 | for FILE in $ROADS 18 | do 19 | echo "processing $FILE file..." 20 | FILENEW=`echo $FILE | sed "s/.shp/_motorways_$PRJ.shp/"` 21 | ogr2ogr \ 22 | -sql "select * from roads where type in ('motorway', 'trunk')" \ 23 | -t_srs $TSRS \ 24 | $FILENEW $FILE\ 25 | 26 | done 27 | 28 | # queries out motorway links (off ramps and connectors) 29 | for FILE in $ROADS 30 | do 31 | echo "processing $FILE file..." 32 | FILENEW=`echo $FILE | sed "s/.shp/_motorway-links_$PRJ.shp/"` 33 | ogr2ogr \ 34 | -sql "select * from roads where type in ('motorway_link', 'trunk_link')" \ 35 | -t_srs $TSRS \ 36 | $FILENEW $FILE\ 37 | 38 | done 39 | 40 | # queries out main roads (primary, secondary, tertiary) 41 | for FILE in $ROADS 42 | do 43 | echo "processing $FILE file..." 44 | FILENEW=`echo $FILE | sed "s/.shp/_main-rd_$PRJ.shp/"` 45 | ogr2ogr \ 46 | -sql "select * from roads where type in ('primary','secondary','tertiary')" \ 47 | -t_srs $TSRS \ 48 | $FILENEW $FILE\ 49 | 50 | done 51 | 52 | # queries out primary links (usually not many of these) 53 | for FILE in $ROADS 54 | do 55 | echo "processing $FILE file..." 56 | FILENEW=`echo $FILE | sed "s/.shp/_main-rd-links_$PRJ.shp/"` 57 | ogr2ogr \ 58 | -sql "select * from roads where type in ('primary_link','secondary_link','tertiary_link')" \ 59 | -t_srs $TSRS \ 60 | $FILENEW $FILE\ 61 | 62 | done 63 | 64 | # queries out residential and other smaller roads 65 | for FILE in $ROADS 66 | do 67 | echo "processing $FILE file..." 68 | FILENEW=`echo $FILE | sed "s/.shp/_other-rd_$PRJ.shp/"` 69 | ogr2ogr \ 70 | -sql "select * from roads where type in ('residential', 'service', 'living_street', 'unclassified')" \ 71 | -t_srs $TSRS \ 72 | $FILENEW $FILE\ 73 | 74 | done 75 | 76 | # queries out dirt roads 77 | for FILE in $ROADS 78 | do 79 | echo "processing $FILE file..." 80 | FILENEW=`echo $FILE | sed "s/.shp/_dirt-rd_$PRJ.shp/"` 81 | ogr2ogr \ 82 | -sql "select * from roads where type = 'track'" \ 83 | -t_srs $TSRS \ 84 | $FILENEW $FILE\ 85 | 86 | done 87 | 88 | # queries out paths, trails, cycling paths, etc. 89 | for FILE in $ROADS 90 | do 91 | echo "processing $FILE file..." 92 | FILENEW=`echo $FILE | sed "s/.shp/_trails_$PRJ.shp/"` 93 | ogr2ogr \ 94 | -sql "select * from roads where type in ('bridleway', 'cycleway', 'footway', 'path', 'pedestrian', 'steps')" \ 95 | -t_srs $TSRS \ 96 | $FILENEW $FILE\ 97 | 98 | done 99 | 100 | # move output files to a desired location (separate from original data) 101 | for FILE in *_$PRJ.* 102 | do 103 | echo "moving $FILE file..." 104 | mv $FILE $DIR_OUT 105 | 106 | done 107 | 108 | exit -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | shell_scripts 2 | ============= 3 | 4 | Bash [shell scripts](http://en.wikipedia.org/wiki/Shell_script) primarily for batch geoprocessing spatial data using the OGR2OGR utility, a part of the Geospatial Data Abstract Library: [GDAL.](http://www.gdal.org/ogr2ogr.html) 5 | 6 | These scripts are useful when open-source GIS applications such as [QGIS](http://qgis.org/en/site/) do not allow for batch processing directories of vector spatial data. Additionally, invoking the scripts from a shell (such as the [Terminal.App](http://en.wikipedia.org/wiki/Terminal_%28OS_X%29) in Mac OSX) allows for heavy data processing to be run in the background while freeing up a GIS software to be used simultaneously for visualization and analysis. 7 | 8 | There are also a few scripts in here for doing other useful things such as a batch find and replace or renaming files to be database and GIS friendly. 9 | 10 | ## The Scripts 11 | * [clip-extent-project.sh](#clip-extent-projectsh) 12 | * [clip-raster-to-shp.sh](#clip-raster-to-shpsh) 13 | * [clip-to-polygon.sh](#clip-to-polygonsh) 14 | * [geojson-to-shp.sh](#geojson-to-shpsh) 15 | * [get-extent.sh](#get-extentsh) 16 | * [gpx-to-shp.sh](#gpx-to-shpsh) 17 | * [grep-find-replace.sh](#grep-find-replacesh) 18 | * [make-hillshades.sh](#make-hillshadessh) 19 | * [merge.sh](#mergesh) 20 | * [project-google.sh](#project-googlesh) 21 | * [project-mercator.sh](#project-mercatorsh) 22 | * [project-wgs84.sh](#project-wgs84sh) 23 | * [rename-files-gis-friendly.sh](#rename-files-gis-friendlysh) 24 | * [separate-roads-by-type.sh](#separate-roads-by-typesh) 25 | * [shp-to-geojson.sh](#shp-to-geojsonsh) 26 | 27 | ## Requirements 28 | You must install [**GDAL**](http://gdal.org/) and be able to access its utilities from the command line. 29 | 30 | ###Mac OS X: 31 | Install GDAL via either [**King Chaos**](http://www.kyngchaos.com/software/qgis) or [**Homebrew**](http://brew.sh/). 32 | 33 | If installing through **King Chaos** be sure to follow the instructions for appending the **GDAL** utilities' absolute path to the `PATH` variable in your `.bash_profile` or `.bashrc` file in your home directory. 34 | 35 | ###Linux: 36 | 37 | Install **FWTools** (the Linux 64-bit version is [here](http://fwtools.maptools.org/linux-experimental.html)). 38 | 39 | ###Test: 40 | 41 | Make sure `ogr2ogr` is working in Bash with basic command: `ogrinfo --version` 42 | 43 | You should get back something like: 44 | 45 | GDAL 1.11.0, released 2014/04/16 46 | 47 | You can either add the shell_scripts directory to your `PATH` or run the commands with the full file path. 48 | 49 | ##FAQ 50 | The scripts I authored in this repository were written while working with Cartographic Design & GIS software that primarily uses the [ESRI Shapefile format](http://en.wikipedia.org/wiki/Shapefile) for storing vector spatial data. Thus most of these scripts assume that is your desired output (and sometimes input) data format unless otherwise specified. Use the `shp-to-geojson` and `geojson-to-shp` scripts to convert data between the ESRI Shapefile and GeoJSON data formats as needed. 51 | 52 | Many of these scripts contain variables that must be edited to in order to change their paramaters. Before running open the script of choice with a text editor and change variable assignments for parameters such as output projection (eg: `T_SRS=EPSG:4326`). 53 | 54 | **Note:** *You must set the executible permission for these scripts before you can run them.* First `cd` to the folder in which they live and do `chmod +ux *` to make them executible. 55 | 56 | ## Reference Material 57 | These resources have helped me out in learning to build the `ogr2ogr` shell scrips: 58 | 59 | * [Directory of Spatial Reference Systems (SRS)](http://spatialreference.org/ref/) 60 | 61 | * FYI, GitHub currently likes (i.e., requires) data in the [CRS](http://en.wikipedia.org/wiki/Spatial_reference_system) `urn:ogc:def:crs:OGC:1.3:CRS84` / `EPSG:4326` for displaying geoJSON. 62 | 63 | * [List of OGR-supported Vector Formats](http://www.gdal.org/ogr/ogr_formats.html)...*so many options!* 64 | 65 | * This [Unix shell scripting tutorial](https://supportweb.cs.bham.ac.uk/documentation/tutorials/docsystem/build/tutorials/unixscripting/unixscripting.html) 66 | 67 | ## Script Descriptions 68 | 69 | ### clip-extent-project.sh 70 | Clips data given bounding box coordinates and projects to an output CRS. Change variable assignments within the script for these parameters. 71 | ####Supported Types 72 | ESRI Shapefile 73 | ####Usage 74 | **variables:** 75 | 76 | ``` 77 | LAT_MIN # minimum latitude (min y value) 78 | LON_MIN # minimum longitude ( min x value) 79 | LAT_MAX # maximum latitude (max y value) 80 | LON_MAX # maximum longitude (max x value) 81 | S_SRS #data's source coordinate system / projection 82 | T_SRS #data's target coordinate system / projection 83 | DIR_OUT # set absolute path for output .shp files here 84 | PRJ # extracts EPSG code from T_SRS to append to filename 85 | ``` 86 | 87 | `cd` to the directory of your `.shp` data and do 88 | `./clip-extent-project.sh` 89 | ####Sample Output 90 | `test` 91 | 92 | ### clip-raster-to-shp.sh 93 | Clips raster datasets to a shapefile polygon. Both the raster and vector datasets must be in the same projection / CRS. 94 | ####Supported Types 95 | `.shp` and GDAL supported raster formats. 96 | ####Usage 97 | `./clip-raster-to-shp.sh polygon-to-clip-to.shp raster-data.tiff` 98 | ####Sample Output 99 | `test` 100 | 101 | ### clip-to-polygon.sh 102 | Clips all data to a shapefile polygon. 103 | ####Supported Types 104 | `.shp` 105 | ####Usage 106 | **variables:** 107 | 108 | ``` 109 | CLIP # absolute path to clipping .shp polygon 110 | DIR_OUT # absolute path for output .shp files 111 | ``` 112 | 113 | `cd` to directory containing `.shp` data and do 114 | `./clip-to-polygon.sh` 115 | ####Sample Output 116 | `test` 117 | 118 | ### geojson-to-shp.sh 119 | This script will export all `.geojson` files in a directory to `.shp` files in a specified subdirectory. 120 | ####Supported Types 121 | `.geojson` 122 | ####Usage 123 | **variables:** 124 | `NEWDIR # Path for new sub-directory for output .shp files` 125 | 126 | `cd` to the directory of `.geojson` files you want to convert 127 | `./geojson-to-shp.sh` 128 | ####Sample Output 129 | Writes new files to a subdirectory. 130 | 131 | ### get-extent.sh 132 | Returns bounding box coordinates for a dataset as: 133 | ` ` 134 | ####Supported Types 135 | `.shp` 136 | ####Usage 137 | `./get-extent.sh $DIR/$FILE` 138 | ####Sample Output 139 | `-84.391994 33.758135 -84.376599 33.754353` 140 | 141 | ### gpx-to-shp.sh 142 | Converts all `.gpx` files in a directory into separate `.shp` files for the following `.gpx` data layers: 143 | `tracks`, `way points` and `track points` assuming the `.gpx` file contains data for these layers. 144 | ####Supported Types 145 | `.gpx` 146 | ####Usage 147 | **variables:** 148 | 149 | ``` 150 | OUTDR # absolute path for .shp files here 151 | TSRS # target CRS EPSG code 152 | PRJ # extracts EPSG code from $TSRS to append to filename 153 | ``` 154 | `./gpx-to-shp.sh` 155 | ####Sample Output 156 | 157 | ``` 158 | some_gpx_file_tracks_2223.shp 159 | some_gpx_file_points_2223.shp 160 | some_gpx_file_waypoints_2223.shp 161 | ``` 162 | 163 | ### grep-find-replace.sh 164 | Search through all files in a directory and perform a find and replace on text inside those files. 165 | ####Supported Types 166 | `.txt, .html, .css, etc.` 167 | ####Usage 168 | **variables:** 169 | 170 | ``` 171 | startdirectory # directory to search in 172 | searchterm # file name to search for 173 | replaceterm # file name to move / replace 174 | ``` 175 | `./grep-find-replace.sh` 176 | ####Sample Output 177 | `test` 178 | 179 | ### make-hillshades.sh 180 | Takes a Digital Elevation Model (DEM) raster and generates hillshades from 4 different light angles and a slope shade.These 5 files may then be composited in QGIS, TileMill, Photoshop, etc. 181 | 182 | **Note:** Requires a `color-slope.txt` file containing the following: 183 | 184 | ``` 185 | 0 255 255 255 186 | 90 0 0 0 187 | ``` 188 | **Suggestion:** Process DEM data prior to running this script (mosaic, clip, resample, reproject, etc). 189 | ####Supported Types 190 | GDAL supported Raster files. 191 | ####Usage 192 | **variables:** 193 | `Z # vertical exaggeration factor. Default is 1` 194 | 195 | `cd` to the folder containing the DEM 196 | then do: `./make-hillshades.sh some-input-dem.flt` 197 | ####Sample Output 198 | 199 | ``` 200 | hillshade_az45.tif 201 | hillshade_az135.tif 202 | hillshade_az225.tif 203 | hillshade_az315.tif 204 | slope.tif 205 | slopeshade.tif 206 | ``` 207 | 208 | ### merge.sh 209 | Merges a directory of shapefiles. Copies the first `.shp` file as the target to merge to and then appends each following `.shp` file to it. 210 | **Caveats:** Files must be of same geometry. If attribute schema differs between files you will end up with a huge attribute table. 211 | ####Supported Types 212 | `.shp` 213 | ####Usage 214 | **variables:** 215 | 216 | ``` 217 | FILE # name of .shp file to be merged to (will be created if it doesn't yet exist) 218 | LAYER # must be same as above but without `.shp` extension 219 | SSRS # source CRS 220 | TSRS # target CRS 221 | ``` 222 | 223 | `cd` to the folder containing files to merge 224 | copy a shapefile to merge other shapefiles to by doing 225 | `ogr2ogr merge.shp shapefile-to-merge-everything-to.shp` 226 | then do 227 | `./merge.sh` 228 | ####Sample Output 229 | `test` 230 | 231 | ### project-web-mercator.sh 232 | Projects all data in a directory to web-mercator projection `EPSG:3857` 233 | ####Supported Types 234 | `.shp` 235 | ####Usage 236 | **variables:** 237 | `$1 # source CRS of target data` 238 | 239 | do: 240 | `./project-web-mercator.sh ` 241 | ####Sample Output 242 | `test` 243 | 244 | ### project-mercator.sh 245 | Projects all data in a directory to WGS 94 / World Mercator `EPSG:3395` 246 | ####Supported Types 247 | `.shp` 248 | ####Usage 249 | **variables:** 250 | `$1 # source CRS of target data` 251 | 252 | `cd` to the folder containing your data and do 253 | `./project-mercator.sh ` 254 | ####Sample Output 255 | `test` 256 | 257 | ### project-wgs84.sh 258 | Projects all data in a directory to WGS 84 `EPSG:4326` 259 | ####Supported Types 260 | `.shp` 261 | ####Usage 262 | **variables:** 263 | `$1 # source CRS of target data` 264 | 265 | `./project-wgs84.sh ` 266 | ####Sample Output 267 | `test` 268 | 269 | ### rename-files-gis-friendly.sh 270 | Modifies all filenames in a directory to be GIS / database friendly by 271 | replacing white spaces with underscores and making all characters lowercase. 272 | ####Supported Types 273 | any 274 | ####Usage 275 | `./rename-files-gis-friendly.sh` 276 | ####Sample Output 277 | `test` 278 | 279 | ### separate-roads-by-type.sh 280 | Takes a generic OpenStreetMap `roads.shp` file extracted from a Planet OSM source such as [bbbike.org](http://extract.bbbike.org) and parses into multiple shapefiles based on the aggregating values from the `type` field. 281 | Will reproject all files to a desired CRS and append EPSG code to the end of the file name. 282 | ####Supported Types 283 | `.shp` 284 | ####Usage 285 | **variables:** 286 | 287 | ``` 288 | ROADS # the name of your OSM roads layer file, must have a type field name and be named roads.shp 289 | TSRS # target CRS 290 | PRJ # extracts EPSG code from T_SRS to append to filename 291 | DIR_OUT # set absolute path for output .shp files here 292 | ```` 293 | `cd` to folder containing `roads.shp` and do: `./separate-roads-by-type.sh` 294 | ####Sample Output 295 | 296 | ``` 297 | roads_motorways_2227.shp 298 | roads_motorway-links_2227.shp 299 | roads_main-rd_2227.shp 300 | roads_main-rd-links_2227.shp 301 | roads_other-rd_2227.shp 302 | roads_dirt-rd_2227.shp 303 | roads_trails_2227.shp 304 | ``` 305 | 306 | ### shp-to-geojson.sh 307 | Converts all shapefiles in a directory `$DIR1` into geoJSON files in `$DIR2` and reprojects to `EPSG:4326`. 308 | 309 | If `$DIR1` is not specified, the script looks in the present working directory. If `$DIR2` is not specificied, the script will place the new geoJSON files in a new subdirectory called `geojson`. 310 | 311 | **Note:** if on Mac OSX you must specify input and output directories or script will fail due to stupid unix differences between Linux and OSX. 312 | ####Supported Types 313 | `.shp` 314 | ####Usage 315 | `./shp-to-geojson.sh $DIR1 $DIR2` 316 | ####Sample Output 317 | ``` 318 | 91764 bytes 319 | converting file: ./roads_2277.shp into roads_2277.geojson... 320 | ``` 321 | --------------------------------------------------------------------------------