├── README.mkd ├── latlon2merc.py ├── shadesrtm.sh ├── shpmerge.sh ├── tif2geo.sh └── tms-prune.py /README.mkd: -------------------------------------------------------------------------------- 1 | # GIS Scripts 2 | 3 | A place to keep scripts I've written to make various GIS & map-design tasks 4 | simpler and more efficient. 5 | 6 | ## shadesrtm.sh 7 | 8 | A demonstration of my hillshade creation process in script form. Makes use of gdaldem utilities from newer versions of GDAL. 9 | 10 | ## shpmerge.sh 11 | 12 | Uses ogr2ogr to merge multiple shapefiles into one. 13 | 14 | ## tif2geo.sh 15 | 16 | This script reads the geographic coordinates from one TIF file (using gdalinfo) and applies them to another TIF (using gdal_translate). Useful for restoring this data after making edits in a non-geo-aware program, such as GIMP or ImageMagick. 17 | 18 | ## tms-prune.py 19 | 20 | Searches a TMS tileset for files that fall outside of the world bounding box (which can get created by metatiling, bounding box errors, etc). It assumes a web-mercator projection (default for OpenStreetMap, Google Maps, MapBox tilesets). 21 | -------------------------------------------------------------------------------- /latlon2merc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Converts a latlon coordinate or bounding box to web mercator, moving/cropping 5 | # it to within the web mercator 'square world' if necessary. 6 | # 7 | # Example usage: latlon2merc.py -180 -90 180 90 8 | 9 | import sys 10 | from mapnik import Box2d, Coord, Projection, ProjTransform 11 | 12 | latlon = Projection('+proj=latlong +datum=WGS84') 13 | merc = Projection('+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs') 14 | 15 | max_4326 = 85.0511287798066 16 | max_3785 = 20037508.3427892 17 | 18 | transform = ProjTransform(latlon, merc) 19 | 20 | if len(sys.argv) == 3: 21 | ll = transform.forward(Coord(float(sys.argv[1]), float(sys.argv[2]))) 22 | if (ll.y > max_3785): 23 | print(' '.join([str(ll.x), str(max_3785)])) 24 | elif (ll.y < -max_3785): 25 | print(' '.join([str(ll.x), str(-max_3785)])) 26 | else: 27 | print(' '.join([str(ll.x), str(ll.y)])) 28 | 29 | elif len(sys.argv) == 5: 30 | minx, miny, maxx, maxy = ( 31 | float(sys.argv[1]), 32 | float(sys.argv[2]), 33 | float(sys.argv[3]), 34 | float(sys.argv[4]) 35 | ) 36 | 37 | if (miny < -max_4326): 38 | miny = -max_4326 39 | 40 | if (maxy > max_4326): 41 | maxy = max_4326 42 | 43 | bbox = transform.forward(Box2d(minx, miny, maxx, maxy)) 44 | print(' '.join([ 45 | str(bbox.minx), 46 | str(bbox.miny), 47 | str(bbox.maxx), 48 | str(bbox.maxy) 49 | ])) 50 | 51 | else: 52 | print("Error: Incorrect number of arguments") 53 | -------------------------------------------------------------------------------- /shadesrtm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e -u 3 | 4 | ## This script is intended more as documentation of the SRTM -> Hillshade tiles 5 | ## process rather than something that useful out-of-the-box. You should 6 | ## READ THROUGH THIS COMPLETELY BEFORE USING THIS SCRIPT to make sure you 7 | ## understand what it is going to do - there are practically no safeguards in 8 | ## place and many assumptions are made. USE AT YOUR OWN RISK. 9 | 10 | ## NOTES: 11 | ## * It is safe to ignore 'unknown field' warnings from convert & mogrify - 12 | ## this happens because these programs don't understand spatial data. 13 | 14 | ## Required programs: 15 | ## gdal >=1.7.0 16 | ## imagemagick (tested with 6.5.x) 17 | ## tif2geo.sh - download at 18 | ## 19 | 20 | ## See for more on what 21 | ## a color ramp file should look like. 22 | COLOR_RAMP="/home/aj/devseed/maps/_raster/afcount_winter.ramp" 23 | 24 | ## The ramp I use for slope is: 25 | ## 90 0 0 0 26 | ## 0 255 255 255 27 | SLOPE_RAMP="/home/aj/devseed/maps/_raster/slope.ramp" 28 | 29 | TIF2GEO="$HOME/bin/tif2geo.sh" 30 | 31 | ## prepare subdirectories for output files 32 | mkdir -p {slope,slope_render,hillshade,color,merged} 33 | 34 | for SRTM in $@; do 35 | ## '-s 111120' is the proper scale for data in metres. 36 | echo -n "Calculating slope [$SRTM]: " 37 | gdaldem slope -s 111120 $SRTM slope/$SRTM 38 | ## the output of 'gdaldem slope' is not directly useful - we need to apply 39 | ## colors to the values. A 0° slope will be white, a 90° slope will be black. 40 | echo -n "Rendering slope [$SRTM]: " 41 | gdaldem color-relief slope/$SRTM $SLOPE_RAMP slope_render/$SRTM 42 | 43 | echo -n "Generating hillshade [$SRTM]: " 44 | gdaldem hillshade -s 111120 $SRTM hillshade/$SRTM 45 | mogrify -fill "#b5b5b5" -opaque "#000" hillshade/$SRTM 46 | 47 | echo -n "Generating color-relief [$SRTM]: " 48 | gdaldem color-relief $SRTM $COLOR_RAMP color/$SRTM 49 | 50 | ## merge everything with imagemagick 51 | echo -n "Merging $SRTM..." 52 | convert color/$SRTM \ 53 | -compose soft-light hillshade/$SRTM -composite \ 54 | -compose multiply slope_render/$SRTM -composite \ 55 | -crop 5999x5999+1+1 merged/$SRTM 56 | 57 | ## For just slope & hillshade: 58 | # convert slope_render/$SRTM \ 59 | # -compose overlay hillshade/$SRTM -composite \ 60 | # -crop 5999x5999+1+1 merged/$SRTM 61 | 62 | ## IM destroys geo data - restore with this script I wrote. See: 63 | ## 64 | echo -n "Restoring spatial data [$SRTM]: " 65 | $TIF2GEO -s 4326 -r $SRTM -f merged/$SRTM 66 | done 67 | 68 | ## Make one big tiff 69 | echo -n "Stitching single output file..." 70 | gdal_merge.py -o terrain-merged-4326.tif merged/srtm_*.tif 71 | 72 | ## Reproject for gdal2tiles. '-wm' is memory cache in MB 73 | echo -n "Reprojecting output file..." 74 | gdalwarp -wm 512 -t_srs "EPSG:3785" -r lanczos terrain-merged-4326.tif terrain-merged-3785.tif 75 | 76 | ## render tiles, eg: 77 | #gdal2tiles.py -z 5-11 -r lanczos terrain-merged-3785.tif 78 | -------------------------------------------------------------------------------- /shpmerge.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e -u 3 | 4 | # shpmerge.sh ... 5 | 6 | OUTFILE="$1" 7 | shift 8 | 9 | for INFILE in "$@"; do 10 | if test -e "$OUTFILE"; then 11 | echo -n "Merging $INFILE ... " 12 | ogr2ogr -f "ESRI Shapefile" -update -append \ 13 | "$OUTFILE" "$INFILE" -nln `basename $OUTFILE .shp` && \ 14 | echo "OK" || exit 1 15 | else 16 | echo -n "Creating $OUTFILE from $INFILE ... " 17 | ogr2ogr -f "ESRI Shapefile" "$OUTFILE" "$INFILE" && \ 18 | echo "OK" || exit 1 19 | fi 20 | done 21 | 22 | echo "DONE!" 23 | -------------------------------------------------------------------------------- /tif2geo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e -u 3 | 4 | # tif2geo.sh 5 | # A wrapper for gdal_translate that will save you some typing/copying/pasting. 6 | # It converts a plain tif to a geotif using the geographic extent of a 7 | # specified reference file. Useful after editing geotifs with applications 8 | # such as GIMP, ImageMagick, etc. 9 | 10 | # TODO: 11 | # Implement custom output file? 12 | 13 | # Specify locations of executables here if they are not in $PATH 14 | GDALINFO=gdalinfo 15 | GDAL_TRANSLATE=gdal_translate 16 | 17 | # Default values: 18 | COMPRESS=1 19 | 20 | function usage() { 21 | echo "$0" 22 | echo "Uses gdalinfo and gdal_translate to copy geographic information" 23 | echo "into a plain TIFF file." 24 | echo "" 25 | echo "Usage: $0 -r reference file -f input_file -s srs_string [-u]" 26 | echo "-u creates an uncompress output file. LZW compression is used by default." 27 | exit 28 | } 29 | 30 | function set_srs() { 31 | # TODO: We really could autodetect this... 32 | case $1 in 33 | osm|goog|google|900913) INPUT_SRS="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs";; 34 | wgs84|4326) INPUT_SRS="+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs";; 35 | ?) INPUT_SRS="$1";; 36 | esac 37 | } 38 | 39 | while getopts "f:h:o:r:s:u" ARG; do 40 | case $ARG in 41 | f) INPUT_FILE="$OPTARG";; 42 | h) usage; exit;; 43 | r) REFERENCE_FILE="$OPTARG";; 44 | s) set_srs "$OPTARG";; 45 | u) COMPRESS=0;; 46 | [?]) usage; exit;; 47 | esac 48 | done 49 | 50 | TEMP_FILE="${INPUT_FILE}.TIF2GEO_TMP" 51 | 52 | # Extract Upper Left and Lower Right coordinates from gdalinfo output 53 | UL="`$GDALINFO $REFERENCE_FILE | grep -e "^Upper Left" | cut -c 14-38`" 54 | LR="`$GDALINFO $REFERENCE_FILE | grep -e "^Lower Right" | cut -c 14-38`" 55 | 56 | if [ $COMPRESS == 1 ]; then 57 | COMPRESS_OPT="-co compress=lzw" 58 | fi 59 | 60 | mv "$INPUT_FILE" "$TEMP_FILE" 61 | $GDAL_TRANSLATE $COMPRESS_OPT -a_ullr $UL $LR -a_srs "$INPUT_SRS" "$TEMP_FILE" "$INPUT_FILE" 62 | rm "$TEMP_FILE" -------------------------------------------------------------------------------- /tms-prune.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # WARNING: 5 | # You should definitely test this in a sandbox first to make sure it does what 6 | # you think it should. Use at your own risk. 7 | 8 | ''' 9 | tms-prune.py: removes extra tiles from a TMS tileset 10 | 11 | Notes: 12 | 13 | For now we assume that "extra" means outside of the web-mercator bounding-box 14 | for the whole earth. Given a directory structure of /z/x/y.ext, files/folders 15 | that do not fit the following rule are unnecessary: 16 | 17 | z >= 0 and z <= 18 and x < 2**z and y < 2**z 18 | 19 | For now, we are ignoring files and folders whose name is not a number. In the 20 | future we may want an option to delete these as well. 21 | 22 | This script is also pretty slow. As it is, it checks nearly every file in a 23 | layer to see if it's needed or not. This could be sped up by only searching 24 | for extra files that are likely to be present, with a full search being 25 | optional. 26 | ''' 27 | 28 | import os, sys 29 | from shutil import rmtree 30 | from optparse import OptionParser 31 | 32 | def is_tms_zlevel(z): 33 | ''' 34 | Returns the directory as an integer if it is a valid TMS z-value. 35 | Returns -1 if the directory is a number, but not a valid TMS z-value. 36 | Returns -2 if the directory is not a number. 37 | ''' 38 | try: 39 | int(z) 40 | except ValueError: 41 | return -2 42 | z = int(z) 43 | if z >= 0 and z <= 18: 44 | return z 45 | else: 46 | return -1 47 | 48 | def delete(path): 49 | ''' Delete a file or directory, or just print the path if --list ''' 50 | if os.path.isdir(path): 51 | path_type = "directory" 52 | elif os.path.isfile(path): 53 | path_type = "file" 54 | else: 55 | # then what is it? 56 | return 0 57 | if options.list_only: 58 | print path 59 | else: 60 | if path_type == "directory": 61 | rmtree(path) 62 | if path_type == "file": 63 | os.remove(path) 64 | 65 | def prune_layer(layer): 66 | print "Pruning %s" % (layer) 67 | count = 0 68 | msg = "" 69 | for z in os.listdir(layer): 70 | if is_tms_zlevel(z) < 0: 71 | continue 72 | z_dir = os.path.join(layer, z) 73 | max_xy = 2**int(z)-1 74 | for x in os.listdir(z_dir): 75 | x_dir = os.path.join(z_dir, x) 76 | if not os.path.isdir(x_dir): 77 | continue 78 | try: 79 | if int(x) > max_xy: 80 | count = count + len([name for name in os.listdir(x_dir) if os.path.isfile(name)]) 81 | delete(x_dir) 82 | for c in msg: sys.stdout.write(chr(8)) 83 | msg = "%s out-of-bounds tiles deleted" % (count) 84 | sys.stdout.write(msg) 85 | else: 86 | for y in os.listdir(x_dir): 87 | y_path = os.path.join(x_dir, y) 88 | # Drop the file extension 89 | y_val = os.path.splitext(y)[0] 90 | try: 91 | if int(y_val) > max_xy: 92 | count = count + 1 93 | delete(y_path) 94 | for c in msg: sys.stdout.write(chr(8)) 95 | msg = "%s out-of-bounds tiles deleted" % (count) 96 | sys.stdout.write(msg) 97 | except ValueError: 98 | pass 99 | except ValueError: 100 | pass 101 | sys.stdout.write(chr(10)) 102 | 103 | def main(layers): 104 | for layer in layers: 105 | if not os.path.isdir(layer): 106 | # @TODO: Proper error handling? 107 | print "Warning: %s is not a directory." % (layer) 108 | else: 109 | prune_layer(layer) 110 | 111 | if __name__ == '__main__': 112 | usage = "usage: %prog LAYERS..." 113 | 114 | parser = OptionParser(usage=usage) 115 | 116 | parser.add_option("-l", "--list", 117 | action="store_true", dest="list_only", default=False, 118 | help="Don't remove any files, just print a list") 119 | 120 | (options, args) = parser.parse_args() 121 | 122 | if len(args) < 1: 123 | # We need at least one layer to prune 124 | parser.error('No TMS layer specified') 125 | main(args) 126 | --------------------------------------------------------------------------------