├── checkstatus.pl ├── README.md ├── clean_bitiles.pl ├── gpx2bitiles.pl ├── filter-points.pl ├── filter-gpx.pl └── bitiles2png.pl /checkstatus.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # Generate a status image of existing tiles (in any format). 4 | # Written by Ilya Zverev, licensed WTFPL. 5 | 6 | use strict; 7 | use GD; 8 | use File::Basename; 9 | use Getopt::Long; 10 | 11 | my $source_dir; 12 | my $dest; 13 | my $zoom; 14 | my $thread_str; 15 | my $help; 16 | 17 | GetOptions('h|help' => \$help, 18 | 'i|input=s' => \$source_dir, 19 | 'o|output=s' => \$dest, 20 | 'z|zoom=i' => \$zoom, 21 | 't|threads=s' => \$thread_str, 22 | ) || usage(); 23 | 24 | if( $help ) { 25 | usage(); 26 | } 27 | 28 | usage("Please specify input directory with -i") unless defined($source_dir); 29 | die "Source directory not found" unless -d $source_dir; 30 | usage("Please specify target image name with -o") unless defined($dest); 31 | 32 | $zoom = find_zoom($source_dir) unless defined($zoom); 33 | my $size = 2**$zoom; 34 | 35 | my $bxmin = $size + 1; 36 | my $bxmax; 37 | my $width; 38 | if( defined($thread_str) ) { 39 | die 'Threads string should contain three numbers: minlon, maxlon, thread count.' unless $thread_str =~ /^(-?[\d.]+),(-?[\d.]+),(\d{1,2})$/; 40 | my ($minlon, $maxlon, $threads) = ($1, $2, $3); 41 | 42 | my $eps = 10**-8; 43 | $bxmin = int(($minlon+$eps+180)/360 * $size); 44 | $bxmax = int(($maxlon-$eps+180)/360 * $size); 45 | my $bxwidth = $bxmax - $bxmin + 1; 46 | die("Total number of threads is higher than horizontal number of tiles") if $threads > $bxwidth; 47 | $width = int(($bxwidth + $threads - 1) / $threads); 48 | } 49 | 50 | my $img = GD::Image->new($size, $size); 51 | my $black = $img->colorAllocate(0, 0, 0); 52 | my $white = $img->colorAllocate(255, 255, 255); 53 | my $gray = $img->colorAllocate(224, 224, 224); 54 | 55 | for my $x (0 .. $size) { 56 | my %ys = {}; 57 | if( opendir(my $dh, "$source_dir/$zoom/$x") ) { 58 | %ys = map { if( /^(\d+)\./ ) { $1 => 1 } } readdir($dh); 59 | closedir($dh); 60 | } 61 | for my $y (0 .. $size) { 62 | if( exists $ys{$y} ) { 63 | $img->setPixel($x, $y, $black) 64 | } else { 65 | $img->setPixel($x, $y, $x < $bxmin ? $white : $x > $bxmax ? (int($bxmax / $width)%2 ? $gray : $white) : int(($x - $bxmin) / $width)%2 ? $white : $gray); 66 | } 67 | } 68 | } 69 | 70 | open PIC, ">$dest"; 71 | binmode PIC; 72 | print PIC $img->png(); 73 | close PIC; 74 | 75 | sub find_zoom { 76 | my $dir = shift; 77 | my $dh; 78 | opendir($dh, $dir) or die "Cannot open $dir"; 79 | my @zlist = sort grep { /^\d+$/ && -d "$dir/$_" } readdir($dh); 80 | closedir($dh); 81 | die "No tiles found in $dir" if $#zlist < 0; 82 | return $zlist[-1]; 83 | } 84 | 85 | sub usage { 86 | my ($msg) = @_; 87 | print STDERR "$msg\n\n" if defined($msg); 88 | 89 | my $prog = basename($0); 90 | print STDERR << "EOF"; 91 | Generate a PNG image with all existing tiles drawn as dots. 92 | 93 | usage: $prog [-h] -i source -o target [-z zoom] [-t minlon,maxlon,threads] 94 | 95 | -h : print ths help message and exit. 96 | -i source : directory with tiles. 97 | -o target : output file name. 98 | -z zoom : tiles zoom level (detected automatically). 99 | -t minlon,laxlon,threads : bounds and number of threads in bitiles generation. 100 | 101 | EOF 102 | exit; 103 | } 104 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GPX Planet Tools 2 | 3 | These tools are for processing planet.gpx, an enormous CSV file 4 | consisting of point coordinates. [Get it here](http://planet.osm.org/gps/). 5 | 6 | * `filter-points.pl` 7 | 8 | Extract points inside a polygon specified by a osmosis' poly file 9 | or a number of them (or just a bbox). Can be used to make a regional 10 | extracts of planet.gpx. 11 | 12 | * `filter-gpx.pl` 13 | 14 | Make extracts for GPX planet dump. Accepts tar file on STDIN and 15 | produces tar file and a metadata header for each filtering polygon. 16 | 17 | * `gpx2bitiles.pl` 18 | 19 | Renders planet.gpx or its part into "bitiles": bit arrays that can 20 | later be rendered into regular png images. 21 | 22 | * `clean_bitiles.pl` 23 | 24 | Clean bitiles of salt-and-pepper type noise (there's a lot of it 25 | between latitudes -28.65 and 28.65). 26 | 27 | * `bitiles2png.pl` 28 | 29 | Convert bitiles to png images and generate low zoom tiles from them. 30 | 31 | * `checkstatus.pl` 32 | 33 | Draws an image of all existing tiles for a given zoom. Useful for checking 34 | rendering progress of `gpx2bitiles.pl` -- but only for CSV input. 35 | 36 | You can create antialiased low zoom tiles using [this tool](https://github.com/AMDmi3/tiletool). 37 | 38 | ## EXAMPLES 39 | 40 | 1. Make a regional extract for Austria. 41 | 42 | xz -cdv gps-points.csv.xz | perl filter-points.pl -p austria.poly | xz > austria.csv.xz 43 | 44 | Result: `austria.csv.xz` with every point that lies inside austria.poly polygon. 45 | 46 | 47 | 2. Make regional extracts for Austria and Germany simultaneously. 48 | 49 | Create a file regions.lst with the following two lines: 50 | 51 | poly/europe/austria.poly 52 | poly/europe/germany.poly 53 | 54 | Run a script: 55 | 56 | xz -cdv gps-points.csv.xz | perl filter-points.pl -l regions.lst -o extracts -z 57 | 58 | Result: two files in `extracts` directory, `austria.csv.gz` and `germany.csv.gz` 59 | (gzipped because of `-z` switch). 60 | 61 | 62 | 3. Make regional extracts from GPX dump for Austria and Germany. 63 | 64 | xz -cdv gpx-planet-dump.tar.xz | perl filter-gpx.pl -l regions.lst -o extracts -z 65 | 66 | for f in extracts/*.gz ; do echo $f ; gzip -dc $f | cat ${f%.tar.gz}.metadata.tarc - \ 67 | | xz > ${f%.gz}.xz ; rm $f ; rm ${f%.tar.gz}.metadata.tarc ; done 68 | 69 | 70 | 4. Create zoom 12 bitiles for Australia. 71 | 72 | xz -cdv gps-points.csv.xz | perl gpx2bitiles.pl -b 112,-44,154,-10 -o ausbitiles -z 12 73 | 74 | The process doesn't differ for GPX dump, except you have to untar it: 75 | 76 | tar -xJOf gpx-planet-dump.tar.xz | perl gpx2bitiles.pl -b 112,-44,154,-10 -o ausbitiles -z 12 77 | 78 | 79 | 5. The same, but in three processes. 80 | 81 | Run the following commands simultaneously (in different windows or processes): 82 | 83 | xz -cdv gps-points.csv.xz | perl gpx2bitiles.pl -b 112,-44,154,-10 -o ausbitiles -z 12 -t 1,3 84 | xz -cdv gps-points.csv.xz | perl gpx2bitiles.pl -b 112,-44,154,-10 -o ausbitiles -z 12 -t 2,3 85 | xz -cdv gps-points.csv.xz | perl gpx2bitiles.pl -b 112,-44,154,-10 -o ausbitiles -z 12 -t 3,3 86 | 87 | During its run, gpx2bitiles script creates temporary files `gpx2bitiles.state*`. 88 | Those files are needed in case something happens: processing a CSV file 89 | can take several days, and this is a mean to resume processing. If you have 90 | stopped the script and do not intend on resuming, do not forget to delete 91 | state files. 92 | 93 | 94 | 6. Remove noise from generated bitiles. 95 | 96 | perl clean_bitiles.pl -i ausbitiles -o acbitiles 97 | 98 | 99 | 7. Create PNG tiles for Australia, zoom levels 0 to 12. 100 | 101 | perl bitiles2png.pl -i acbitiles -o austiles -0 -v 102 | 103 | 104 | 8. Create PNG tiles for Sydney with light green dots, zooms 5 to 12. 105 | Write empty tiles where there are no points (JOSM requires them). 106 | 107 | perl bitiles2png.pl -i acbitiles -o sydneytiles -b 150.8,-34.1,151.4,-33.6 -z 12 -u 5 -c 0,255,0 -e -v 108 | 109 | 110 | ## AUTHOR 111 | 112 | Written by Ilya Zverev. All scripts are under WTFPL license: you can 113 | do whatever you want with them. Please do not make regional extracts 114 | or tiles of your cat. Thanks. 115 | -------------------------------------------------------------------------------- /clean_bitiles.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # Remove salt-n-pepper noise from bitiles. 4 | # Written by Ilya Zverev, licensed WTFPL. 5 | 6 | use strict; 7 | use Bit::Vector; 8 | use File::Path 2.07 qw(make_path); 9 | use File::Basename; 10 | use Getopt::Long; 11 | use Fcntl qw( O_RDONLY O_RDWR O_CREAT O_BINARY ); 12 | 13 | my $source_dir; 14 | my $dest_dir; 15 | my $help; 16 | my $verbose; 17 | my $maxpixels = 64; 18 | my $threshold; 19 | my $thresholdpx; 20 | my $threshold_m = 200; 21 | 22 | GetOptions('h|help' => \$help, 23 | 'v|verbose' => \$verbose, 24 | 'i|input=s' => \$source_dir, 25 | 'o|output=s' => \$dest_dir, 26 | 'p|pixels=i' => \$maxpixels, 27 | 't|threshold=i' => \$threshold_m, 28 | 'x|thresholdpx=i' => \$thresholdpx, 29 | ) || usage(); 30 | 31 | if( $help ) { 32 | usage(); 33 | } 34 | 35 | usage("Please specify input directory with -i") unless defined($source_dir); 36 | die "Source directory not found" unless -d $source_dir; 37 | # no default because this operation is irreversible 38 | usage("Please specify output directory with -o") unless defined($dest_dir); 39 | 40 | my $dh; 41 | opendir($dh, "$source_dir") or die "Fail: cannot open $source_dir"; 42 | my @zlist = grep { /^\d+$/ && -d "$source_dir/$_" } readdir($dh); 43 | closedir($dh); 44 | die "No zoom level directories found in $source_dir" if $#zlist < 0; 45 | 46 | for my $z (@zlist) { 47 | opendir($dh, "$source_dir/$z") or next; 48 | my @xlist = grep { /^\d+$/ && -d "$source_dir/$z/$_" } readdir($dh); 49 | closedir($dh); 50 | next if $#xlist < 0; 51 | 52 | if( $thresholdpx ) { 53 | $threshold = $thresholdpx; 54 | } else { 55 | my $resolution = 156543.04 * 0.5 / (2**$z); # meters/pixel at 60 degrees lat 56 | $threshold = int($threshold_m / $resolution + 0.5); 57 | } 58 | $threshold = 64 if $threshold > 64; 59 | $threshold = 2 if $threshold < 2; 60 | 61 | print STDERR "Processing zoom $z, threshold is $threshold pixels" if $verbose; 62 | 63 | for my $x (@xlist) { 64 | my $folder = "$source_dir/$z/$x"; 65 | opendir($dh, $folder) or next; 66 | my @ylist = grep { /^\d+\.bitile$/ && -r "$folder/$_" } readdir($dh); 67 | closedir($dh); 68 | print STDERR '.' if $verbose; 69 | 70 | for my $filename (@ylist) { 71 | $filename =~ /^(\d+)/; 72 | clean_tile($z, $x, $1); 73 | } 74 | } 75 | print STDERR "Done\n" if $verbose; 76 | } 77 | 78 | sub clean_tile { 79 | my ($z, $x, $y) = @_; 80 | my $vec = read_bit_vector("$source_dir/$z/$x/$y.bitile"); 81 | if( $vec->Norm() <= $maxpixels ) { 82 | my $vec_top = read_bit_vector("$source_dir/$z/$x/".($y-1).".bitile"); 83 | my $vec_bottom = read_bit_vector("$source_dir/$z/$x/".($y+1).".bitile"); 84 | my $vec_left = read_bit_vector("$source_dir/$z/".($x-1)."/$y.bitile"); 85 | my $vec_right = read_bit_vector("$source_dir/$z/".($x+1)."/$y.bitile"); 86 | my @pixels_to_remove = (); 87 | for my $pixel ($vec->Index_List_Read()) { 88 | my $px = $pixel % 256; 89 | my $py = int($pixel / 256); 90 | my $value = 0; 91 | TEST: for my $tx ($px-$threshold..$px+$threshold) { 92 | for my $ty ($py-$threshold..$py+$threshold) { 93 | next if $tx == $px && $ty == $py; 94 | if( $tx < 0 && $ty >= 0 && $ty < 256 ) { 95 | $value = $vec_left->bit_test($ty * 256 + $tx + 256); 96 | } elsif( $tx >= 0 && $tx < 256 ) { 97 | if( $ty < 0 ) { 98 | $value = $vec_top->bit_test(($ty + 256) * 256 + $tx); 99 | } elsif( $ty < 256 ) { 100 | $value = $vec->bit_test($ty * 256 + $tx); 101 | } else { 102 | $value = $vec_bottom->bit_test(($ty - 256) * 256 + $tx); 103 | } 104 | } elsif( $tx > 255 && $ty >= 0 && $ty < 256 ) { 105 | $value = $vec_right->bit_test($ty * 256 + $tx - 256); 106 | } 107 | last TEST if $value; 108 | } 109 | } 110 | push @pixels_to_remove, $pixel if !$value; 111 | } 112 | $vec->Index_List_Remove(@pixels_to_remove); 113 | } 114 | return if !$vec->Norm(); 115 | make_path("$dest_dir/$z/$x"); 116 | sysopen(BITILE, "$dest_dir/$z/$x/$y.bitile", O_RDWR | O_CREAT | O_BINARY); 117 | syswrite(BITILE, $vec->Block_Read()); 118 | close BITILE; 119 | } 120 | 121 | sub read_bit_vector { 122 | my $filename = shift; 123 | my $vec = Bit::Vector->new(65536); 124 | if( sysopen(BITILE, $filename, O_RDONLY | O_BINARY) ) { 125 | if( sysread(BITILE, my $read, 8192) == 8192 ) { 126 | $vec->Block_Store($read); 127 | } 128 | close BITILE; 129 | } 130 | return $vec; 131 | } 132 | 133 | sub usage { 134 | my ($msg) = @_; 135 | print STDERR "$msg\n\n" if defined($msg); 136 | 137 | my $prog = basename($0); 138 | print STDERR << "EOF"; 139 | This script traverses input directory and cleans all bitiles in it 140 | from salt-and-pepper noise. 141 | 142 | usage: $prog [-h] [-v] -i source -o target [-p pixels] [-t threshold] 143 | 144 | -h : print ths help message and exit. 145 | -i source : directory with bitiles. 146 | -o target : directory to store processed bitiles. 147 | -p pixels : maximal number of pixels to start cleaning procedure ($maxpixels). 148 | -t threshold : radius of an empty area around a point to remove it 149 | (in meters at 60 degrees latitude, default is $threshold_m). 150 | -x threshold : the same radius, but in pixels. 151 | -v : display progress. 152 | 153 | EOF 154 | exit; 155 | } 156 | -------------------------------------------------------------------------------- /gpx2bitiles.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # This script receives CSV file of "lat,lon" and makes a lot of bitiles. 4 | # A bitile is a 8k bit array. Those should be converted to PNG images with bitiles2png.pl 5 | # Written by Ilya Zverev, licensed WTFPL. 6 | 7 | use strict; 8 | use Math::Trig; 9 | use File::Path 2.07 qw(make_path); 10 | use File::Basename; 11 | use Bit::Vector; 12 | use Fcntl qw( SEEK_SET O_RDWR O_RDONLY O_BINARY O_CREAT ); 13 | use Getopt::Long; 14 | 15 | my $factor = 10**7; # number by which all coordinates in source CSV are multiplied 16 | my $notify_points = 50000; # how often to print dots and save state file 17 | 18 | my $tile_folder; 19 | my $zoom; 20 | my $bbox_str = '-180,-85,180,85'; 21 | 22 | my $help; 23 | my $verbose; 24 | my $infile = '-'; 25 | my $thread_str; 26 | my $state_file = 'gpx2bitiles.state'; 27 | my $lines; 28 | 29 | GetOptions('h|help' => \$help, 30 | 'o|output=s' => \$tile_folder, 31 | 'z|zoom=i' => \$zoom, 32 | 'v|verbose' => \$verbose, 33 | 'i|input=s' => \$infile, 34 | 'b|bbox=s' => \$bbox_str, 35 | 't|thread=s' => \$thread_str, 36 | 'l|lines' => \$lines, 37 | ) || usage(); 38 | 39 | if( $help ) { 40 | usage(); 41 | } 42 | 43 | usage("Please specify output directory with -o") unless defined($tile_folder); 44 | usage("Please specify zoom level with -z") unless defined($zoom); 45 | 46 | die('Drawing lines is not implemented yet.') if $lines; 47 | 48 | my @bbox = split(",", $bbox_str); 49 | die ("badly formed bounding box - use four comma-separated values for left longitude, ". 50 | "bottom latitude, right longitude, top latitude") unless $#bbox == 3; 51 | die("max longitude is less than min longitude") if $bbox[2] < $bbox[0]; 52 | die("max latitude is less than min latitude") if $bbox[3] < $bbox[1]; 53 | 54 | my $zoom2 = 2**$zoom; 55 | my $xmintile = 0; 56 | my $xmaxtile = 2**18; 57 | my $thread = ''; 58 | my @cache = ('', undef, 0); 59 | my $maxdist = $zoom < 11 ? 1 : 2**($zoom - 11); 60 | $maxdist = 255 if $maxdist > 255; 61 | 62 | if( defined($thread_str) ) { 63 | die("Badly formed thread string: it must be THREAD,TOTAL") unless $thread_str =~ /^(\d+),(\d+)$/; 64 | $thread = $1; 65 | my $total = $2; 66 | die("Total number of threads must be no more than 16") if $total <1 || $total > 16; 67 | die("Thread number is higher than number of threads") if $thread > $total || $thread < 1; 68 | 69 | my $eps = 10**-8; 70 | my $bxmin = int(($bbox[0]+$eps+180)/360 * $zoom2); 71 | my $bxmax = int(($bbox[2]-$eps+180)/360 * $zoom2); 72 | my $bxwidth = $bxmax - $bxmin + 1; 73 | die("Total number of threads is higher than horizontal number of tiles") if $total > $bxwidth; 74 | my $width = int(($bxwidth + $total - 1) / $total); 75 | $xmintile = $bxmin + ($thread-1) * $width; 76 | $xmaxtile = $xmintile + $width - 1; 77 | } 78 | 79 | my $count = 0; 80 | my $ctarget = 0; 81 | if( open(STATE, "<$state_file$thread") ) { 82 | my $line = ; 83 | close STATE; 84 | $ctarget = $1 if $line =~ /^(\d+)/; 85 | flush_tile(); 86 | print STDERR "Continuing from point $ctarget\n" if $verbose; 87 | } 88 | 89 | my $filemode = 0; # 1=csv, 2=gpx 90 | my @lastpoint; 91 | 92 | open CSV, "<$infile" or die "Cannot open $infile: $!\n"; 93 | while() { 94 | if( !$filemode ) { 95 | if( /^-?\d+(?:\.\d+)?,\s*-?\d+(?:\.\d+)?/ ) { 96 | $filemode = 1; 97 | } elsif( /\/; 112 | next if !/\= $bbox[3] || $lon <= $bbox[0] || $lon >= $bbox[2]; 120 | if( $count % $notify_points == 0 && open(STATE, ">$state_file$thread") ) { 121 | print STDERR '.' if $verbose; 122 | print STATE $count; 123 | close STATE; 124 | } 125 | 126 | my $x = ($lon+180)/360 * $zoom2; 127 | my $y = (1 - log(tan(deg2rad($lat)) + sec(deg2rad($lat)))/pi)/2 * $zoom2; 128 | my $xtile = int($x); 129 | my $ytile = int($y); 130 | next if $xtile < $xmintile || $xtile > $xmaxtile; 131 | my $xpix = int(256 * ($x - $xtile)); 132 | my $ypix = int(256 * ($y - $ytile)); 133 | 134 | my $dist = @lastpoint ? abs($lastpoint[0] - $x) + abs($lastpoint[1] - $y) : 0; 135 | if( $filemode != 2 || !$lines || $dist <= 1 || $dist > $maxdist ) { 136 | my $vec = read_tile("$tile_folder/$zoom/$xtile/$ytile.bitile"); 137 | $vec->Bit_On($ypix * 256 + $xpix); 138 | } else { 139 | draw_line("$tile_folder/$zoom", $lastpoint[0], $lastpoint[1], $x, $y); 140 | } 141 | @lastpoint = ($x, $y); 142 | } 143 | close CSV; 144 | flush_tile(); 145 | unlink "$state_file$thread"; 146 | print STDERR "\n" if $verbose; 147 | 148 | sub read_tile { 149 | my $filename = shift; 150 | return $cache[1] if $filename eq $cache[0]; 151 | flush_tile(); 152 | 153 | my $vec = Bit::Vector->new(65536); 154 | if( sysopen(BITILE, $filename, O_RDONLY | O_BINARY) ) { 155 | if( sysread(BITILE, my $read, 8192) == 8192 ) { 156 | $vec->Block_Store($read); 157 | } 158 | close BITILE; 159 | } 160 | $cache[0] = $filename; 161 | $cache[1] = $vec; 162 | $cache[2] = $vec->Norm(); 163 | return $vec; 164 | } 165 | 166 | sub flush_tile { 167 | return if !$cache[0] || $cache[1]->Norm() == $cache[2]; 168 | make_path(dirname($cache[0])); 169 | print STDERR "Error opening tile $cache[0]\n" unless sysopen(BITILE, $cache[0], O_RDWR | O_CREAT | O_BINARY); 170 | syswrite(BITILE, $cache[1]->Block_Read()); 171 | close BITILE; 172 | } 173 | 174 | sub draw_line { 175 | # draw a line from @lastpoint (excluding it) to ($x, $y) 176 | # todo 177 | my ($base, $x0, $y0, $x, $y) = @_; 178 | my $xtile = int($x); 179 | my $ytile = int($y); 180 | my $xpix = int(256 * ($x - $xtile)); 181 | my $ypix = int(256 * ($y - $ytile)); 182 | } 183 | 184 | sub usage { 185 | my ($msg) = @_; 186 | print STDERR "$msg\n\n" if defined($msg); 187 | 188 | my $prog = basename($0); 189 | print STDERR << "EOF"; 190 | This script receives CSV file of "lat,lon" (or GPX) and makes a lot of bitiles. 191 | A bitile is a 8k bit array, which can be converted to regular PNG tile 192 | with bitiles2png.pl 193 | 194 | usage: $prog [-h] [-v] [-i file] -o target -z zoom [-b bbox] 195 | 196 | -h : print ths help message and exit. 197 | -i file : CSV/GPX points file to process (default is STDIN). 198 | -o target : directory to store bitiles. 199 | -z zoom : generated tiles zoom level. 200 | -b bbox : limit points by bounding box (four comma-separated 201 | numbers: minlon,minlat,maxlon,maxlat). 202 | -t n,t : process Nth part of T jobs. 203 | -l : connect GPX trace points. 204 | -v : print a dot every $notify_points points. 205 | 206 | All coordinates in source CSV file should be multiplied by $factor 207 | (you can change this number in the code). 208 | 209 | EOF 210 | exit; 211 | } 212 | -------------------------------------------------------------------------------- /filter-points.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # This script filters gps-points csv file by a set of polygons. 4 | # Written by Ilya Zverev, licensed under WTFPL. 5 | # Poly reading sub is from extract_polygon.pl by Frederik Ramm. 6 | 7 | use strict; 8 | use Math::Polygon::Tree; 9 | use Getopt::Long; 10 | use File::Basename; 11 | use File::Path 2.07 qw(make_path); 12 | use IO::Compress::Gzip; 13 | use IO::File; 14 | 15 | my $factor = 10000000; 16 | 17 | my $help; 18 | my $verbose; 19 | my $infile = '-'; 20 | my $target; 21 | my $suffix = ''; 22 | my $polylist; 23 | my $polyfile; 24 | my $bbox; 25 | my $zip; 26 | my $nodupes; 27 | 28 | GetOptions('h|help' => \$help, 29 | 'v|verbose' => \$verbose, 30 | 'i|input=s' => \$infile, 31 | 'o|target=s' => \$target, 32 | 's|suffix=s' => \$suffix, 33 | 'l|list=s' => \$polylist, 34 | 'p|polyfile=s' => \$polyfile, 35 | 'b|bbox=s' => \$bbox, 36 | 'z|gzip' => \$zip, 37 | 'd|nodupes' => \$nodupes, 38 | ) || usage(); 39 | 40 | if( $help ) { 41 | usage(); 42 | } 43 | 44 | my $minlon = 999; 45 | my $minlat = 999; 46 | my $maxlat = -999; 47 | my $maxlon = -999; 48 | my $bboxset = 0; 49 | my @polygons = (); 50 | 51 | if( $bbox ) { 52 | # simply set minlat, maxlat etc from bbox parameter - no polygons 53 | ($minlon, $minlat, $maxlon, $maxlat) = split(",", $bbox); 54 | die ("badly formed bounding box - use four comma-separated values for left longitude, ". 55 | "bottom latitude, right longitude, top latitude") unless defined($maxlat); 56 | die ("max longitude is less than min longitude") if ($maxlon < $minlon); 57 | die ("max latitude is less than min latitude") if ($maxlat < $minlat); 58 | $bboxset = 1; 59 | } 60 | 61 | if( $polylist ) { 62 | open LIST, "<$polylist" or die "Cannot open $polylist: $!"; 63 | $target = '.' if !$target; 64 | make_path($target); 65 | while() { 66 | chomp; 67 | add_polygon_file($_, 1); 68 | } 69 | close LIST; 70 | } elsif( $polyfile ) { 71 | add_polygon_file($polyfile, 0); 72 | } elsif( $bbox ) { 73 | printf STDERR "bbox -> %s\n", $target || 'STDOUT' if $verbose; 74 | my $fh = $target ? ($zip ? new IO::Compress::Gzip $target : IO::File->new($target, 'w')) 75 | : IO::File->new_from_fd(*STDOUT, '>'); 76 | die "Cannot open file: $!" if !$fh; 77 | push @polygons, [$fh, poly_from_bbox()]; 78 | } else { 79 | usage("Please specify either bbox, polygon file or a list of them."); 80 | } 81 | 82 | $minlat *= $factor; 83 | $minlon *= $factor; 84 | $maxlat *= $factor; 85 | $maxlon *= $factor; 86 | 87 | open CSV, "<$infile" or die "Cannot open $infile: $!\n"; 88 | my $lastline = ''; 89 | while() { 90 | if (/^(-?\d+),(-?\d+)/) 91 | { 92 | next if $nodupes && ($lastline eq $_); 93 | next if $1 < $minlat || $1 > $maxlat || $2 < $minlon || $2 > $maxlon; 94 | for my $poly (@polygons) { 95 | $poly->[0]->print($_) if is_in_poly($poly->[1], $1, $2); 96 | } 97 | $lastline = $_; 98 | } 99 | } 100 | close CSV; 101 | 102 | close $_->[0] for @polygons; 103 | 104 | sub add_polygon_file { 105 | my ($p, $multi) = @_; 106 | my $filename = $multi ? ($target ? $target.'/'.basename($p) : basename($p)) : $target; 107 | $filename =~ s/\.poly$/$suffix\.csv/; 108 | $filename .= '.gz' if $zip && $filename !~ /\.gz$/; 109 | print STDERR "$p -> $filename\n" if $verbose; 110 | my $bp = read_poly($p); 111 | my $fh = $target ? ($zip ? new IO::Compress::Gzip $filename : IO::File->new($filename, 'w')) 112 | : IO::File->new_from_fd(*STDOUT, '>'); 113 | die "Cannot open file: $!" if !$fh; 114 | push @polygons, [$fh, $bp]; 115 | } 116 | 117 | sub is_in_poly { 118 | my ($borderpolys, $lat, $lon) = @_; 119 | my $ll = [$lat, $lon]; 120 | my $rv = 0; 121 | foreach my $p (@{$borderpolys}) 122 | { 123 | my($poly,$invert,$bbox) = @$p; 124 | next if ($ll->[0] < $bbox->[0]) or ($ll->[0] > $bbox->[2]); 125 | next if ($ll->[1] < $bbox->[1]) or ($ll->[1] > $bbox->[3]); 126 | 127 | if ($poly->contains($ll)) 128 | { 129 | # If this polygon is for exclusion, we immediately bail and go for the next point 130 | if($invert) 131 | { 132 | return 0; 133 | } 134 | $rv = 1; 135 | # do not exit here as an exclusion poly may still be there 136 | } 137 | } 138 | return $rv; 139 | } 140 | 141 | sub read_poly { 142 | my $polyfile = shift; 143 | my $borderpolys = []; 144 | my $currentpoints; 145 | open (PF, "<$polyfile") || die "Could not open $polyfile: $!"; 146 | 147 | my $invert; 148 | # initialize border polygon. 149 | while() 150 | { 151 | if (/^(!?)\d/) 152 | { 153 | $invert = ($1 eq "!") ? 1 : 0; 154 | $currentpoints = []; 155 | } 156 | elsif (/^END/) 157 | { 158 | if( $#{$currentpoints} > 0 ) { 159 | # close polygon if it isn't 160 | push(@{$currentpoints}, [$currentpoints->[0][0], $currentpoints->[0][1]]) 161 | if $currentpoints->[0][0] != $currentpoints->[-1][0] 162 | || $currentpoints->[0][1] != $currentpoints->[-1][1]; 163 | my $pol = Math::Polygon::Tree->new($currentpoints); 164 | push(@{$borderpolys}, [$pol,$invert,[$pol->bbox]]); 165 | printf STDERR "Added polygon: %d points (%d,%d)-(%d,%d) %s\n", 166 | scalar(@{$currentpoints}), @{$borderpolys->[-1][2]}, 167 | ($borderpolys->[-1][1] ? "exclude" : "include") if $verbose; 168 | } 169 | undef $currentpoints; 170 | } 171 | elsif (defined($currentpoints)) 172 | { 173 | /^\s+([0-9.E+-]+)\s+([0-9.E+-]+)/ or die "Incorrent line in poly: $_"; 174 | push(@{$currentpoints}, [int($2*$factor), int($1*$factor)]); 175 | if( !$bboxset ) { 176 | $minlat = $2 if ($2 < $minlat); 177 | $maxlat = $2 if ($2 > $maxlat); 178 | $minlon = $1 if ($1 < $minlon); 179 | $maxlon = $1 if ($1 > $maxlon); 180 | } 181 | } 182 | } 183 | close (PF); 184 | return $borderpolys; 185 | } 186 | 187 | sub poly_from_bbox { 188 | my @b = (int($minlat*$factor), int($minlon*$factor), int($maxlat*$factor), int($maxlon*$factor)); 189 | my $currentpoints = [[$b[0], $b[1]], [$b[0], $b[3]], [$b[2], $b[3]], [$b[2], $b[1]], [$b[0], $b[1]]]; 190 | my $pol = Math::Polygon::Tree->new($currentpoints); 191 | my $bp = []; 192 | push(@{$bp}, [$pol,0,[$pol->bbox]]); 193 | return $bp; 194 | } 195 | 196 | sub usage { 197 | my ($msg) = @_; 198 | print STDERR "$msg\n\n" if defined($msg); 199 | 200 | my $prog = basename($0); 201 | print STDERR << "EOF"; 202 | This script receives CSV file of "lat,lon" and filters it by a bounding 203 | box, an osmosis' polygon filter file or a number of them. 204 | 205 | usage: $prog [-h] [-i infile] [-o target] [-z] 206 | [-b bbox] [ -p poly | -l list [-s suffix] ] 207 | 208 | -h : print ths help message and exit. 209 | -i infile : CSV points file to process (default is STDIN). 210 | -o target : a file or a directory to put resulting files. 211 | -z : compress output file with gzip. 212 | -b bbox : limit points by bounding box (four comma-separated 213 | numbers: minlon,minlat,maxlon,maxlat). 214 | -p poly : a polygon filter file. 215 | -l list : file with names of poly files. 216 | -s suffix : a suffix to add to all output files. 217 | -d : remove duplicate consecutive points. 218 | -v : print debug messages. 219 | 220 | All coordinates in source CSV file should be multiplied by $factor 221 | (you can change this number in the code). Please keep the number 222 | of polygons below 200, or unexpected problems may occur. 223 | 224 | EOF 225 | exit; 226 | } 227 | -------------------------------------------------------------------------------- /filter-gpx.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # This script filters gpx tar archive by a set of polygons. 4 | # Written by Ilya Zverev, licensed under WTFPL. 5 | # Poly reading sub is from extract_polygon.pl by Frederik Ramm. 6 | 7 | use strict; 8 | use Math::Polygon::Tree; 9 | use Getopt::Long; 10 | use File::Basename; 11 | use File::Path 2.07 qw(make_path); 12 | use IO::Compress::Gzip; 13 | use IO::File; 14 | use Archive::Tar; 15 | 16 | my $help; 17 | my $verbose; 18 | my $target; 19 | my $suffix = ''; 20 | my $polylist; 21 | my $polyfile; 22 | my $bbox; 23 | my $zip; 24 | my $min_points = 10; 25 | 26 | GetOptions('h|help' => \$help, 27 | 'v|verbose' => \$verbose, 28 | 'o|target=s' => \$target, 29 | 's|suffix=s' => \$suffix, 30 | 'l|list=s' => \$polylist, 31 | 'p|polyfile=s' => \$polyfile, 32 | 'b|bbox=s' => \$bbox, 33 | 'z|gzip' => \$zip, 34 | ) || usage(); 35 | 36 | if( $help ) { 37 | usage(); 38 | } 39 | 40 | my $minlon = 999; 41 | my $minlat = 999; 42 | my $maxlat = -999; 43 | my $maxlon = -999; 44 | my $bboxset = 0; 45 | my @polygons = (); 46 | 47 | if( $bbox ) { 48 | # simply set minlat, maxlat etc from bbox parameter - no polygons 49 | ($minlon, $minlat, $maxlon, $maxlat) = split(",", $bbox); 50 | die ("badly formed bounding box - use four comma-separated values for left longitude, ". 51 | "bottom latitude, right longitude, top latitude") unless defined($maxlat); 52 | die ("max longitude is less than min longitude") if ($maxlon < $minlon); 53 | die ("max latitude is less than min latitude") if ($maxlat < $minlat); 54 | $bboxset = 1; 55 | } 56 | 57 | if( $polylist ) { 58 | open LIST, "<$polylist" or die "Cannot open $polylist: $!"; 59 | $target = '.' if !$target; 60 | make_path($target); 61 | while() { 62 | chomp; 63 | add_polygon_file($_, 1); 64 | } 65 | close LIST; 66 | } elsif( $polyfile ) { 67 | add_polygon_file($polyfile, 0); 68 | } elsif( $bbox ) { 69 | printf STDERR "bbox -> %s\n", $target || 'STDOUT' if $verbose; 70 | push_poly($target, poly_from_bbox()); 71 | } else { 72 | usage("Please specify either bbox, polygon file or a list of them."); 73 | } 74 | 75 | my $tariter = Archive::Tar->iter(IO::File->new_from_fd(*STDIN, '<')); 76 | my $metadata_name = 'metadata.xml'; 77 | my $metadata_header = ''; 78 | my %metadata; 79 | my @keysmetadata; 80 | while(my $f = $tariter->()) { 81 | my $cref = $f->get_content_by_ref; 82 | my $name = $f->full_path; 83 | if( $name =~ /metadata\.xml$/ ) { 84 | $metadata_name = $name; 85 | $metadata_header = $1 if $$cref =~ /^(.+?)(?=\ xml fragment } 87 | %metadata = map { /filename="([^"]+\.gpx)"/ ? ($1 => $_) : () } $$cref =~ /\\s+/gs; 88 | @keysmetadata = keys %metadata; 89 | } elsif( $name =~ /\.gpx$/ ) { 90 | my @check_polygons = @polygons; 91 | my @poly_cnt = ($min_points) x scalar(@check_polygons); 92 | # find relevant metadata entry (NB: filenames in tar and metadata are different) 93 | my $metadata_fragment = ''; 94 | foreach (@keysmetadata) { 95 | if( index($name, $_) != -1 ) { 96 | $metadata_fragment = $metadata{$_}; 97 | last; 98 | } 99 | } 100 | for( split /\n/, $$cref ) { 101 | next if !/\ $maxlat || $lon < $minlon || $lon > $maxlon; 105 | for( my $pid = $#check_polygons; $pid >= 0; $pid-- ) { 106 | my $poly = $check_polygons[$pid]; 107 | if( is_in_poly($poly->[1], $lat, $lon) && --$poly_cnt[$pid] <= 0 ) { 108 | # append GPX file to the resulting tar archive 109 | my $arch = Archive::Tar->new; 110 | push @{$arch->{_data}}, $f; # dirty hack from Archive::Tar source 111 | my $tardata = $arch->write; 112 | $poly->[0]->syswrite($tardata, length($tardata) - (Archive::Tar::Constant::BLOCK * 2)); 113 | $arch->clear(); 114 | # append metadata fragment to a resulting metadata.xml 115 | $poly->[2] .= $metadata_fragment; 116 | # remove polygon from array 117 | splice @check_polygons, $pid, 1; 118 | splice @poly_cnt, $pid, 1; 119 | } 120 | } 121 | } 122 | } 123 | } 124 | 125 | for (@polygons) { 126 | # finalize GPX traces tar archive 127 | $_->[0]->syswrite(Archive::Tar::Constant::TAR_END x 2); 128 | close $_->[0]; 129 | # write metadata.xml.tar header (not a proper tar file, intended for concatenation) 130 | if( $_->[2] && $_->[3] && open my $fh, '>', $_->[3] ) { 131 | my $arch = Archive::Tar->new; 132 | $arch->add_data($metadata_name, $metadata_header.$_->[2]."\n"); 133 | my $tardata = $arch->write; 134 | $arch->clear(); 135 | syswrite $fh, $tardata, length($tardata) - (Archive::Tar::Constant::BLOCK * 2); 136 | close $fh; 137 | } 138 | } 139 | 140 | sub add_polygon_file { 141 | my ($p, $multi) = @_; 142 | my $filename = $multi ? ($target ? $target.'/'.basename($p) : basename($p)) : $target; 143 | $filename =~ s/\.poly$/$suffix\.tar/; 144 | $filename .= '.gz' if $zip && $filename !~ /\.gz$/; 145 | print STDERR "$p -> $filename\n" if $verbose; 146 | my $bp = read_poly($p); 147 | push_poly($target ? $filename : '-', $bp); 148 | } 149 | 150 | sub push_poly { 151 | my ($filename, $poly) = @_; 152 | my $fh = $filename && $filename ne '-' 153 | ? ($zip ? new IO::Compress::Gzip $filename : IO::File->new($filename, 'w')) 154 | : IO::File->new_from_fd(*STDOUT, '>'); 155 | die "Cannot open file: $!" if !$fh; 156 | my $metaname = $filename && $filename =~ /^(.+)\.tar(?:\.gz)?$/ ? "$1.metadata.tarc" : 'metadata.tarc'; 157 | # file handle, polygon, metadata.xml contents, metadata.xml archive filename 158 | push @polygons, [$fh, $poly, '', $metaname]; 159 | } 160 | 161 | sub is_in_poly { 162 | my ($borderpolys, $lat, $lon) = @_; 163 | my $ll = [$lat, $lon]; 164 | my $rv = 0; 165 | foreach my $p (@{$borderpolys}) 166 | { 167 | my($poly,$invert,$bbox) = @$p; 168 | next if ($ll->[0] < $bbox->[0]) or ($ll->[0] > $bbox->[2]); 169 | next if ($ll->[1] < $bbox->[1]) or ($ll->[1] > $bbox->[3]); 170 | 171 | if ($poly->contains($ll)) 172 | { 173 | # If this polygon is for exclusion, we immediately bail and go for the next point 174 | if($invert) 175 | { 176 | return 0; 177 | } 178 | $rv = 1; 179 | # do not exit here as an exclusion poly may still be there 180 | } 181 | } 182 | return $rv; 183 | } 184 | 185 | sub read_poly { 186 | my $polyfile = shift; 187 | my $borderpolys = []; 188 | my $currentpoints; 189 | open (PF, "<$polyfile") || die "Could not open $polyfile: $!"; 190 | 191 | my $invert; 192 | # initialize border polygon. 193 | while() 194 | { 195 | if (/^(!?)\d/) 196 | { 197 | $invert = ($1 eq "!") ? 1 : 0; 198 | $currentpoints = []; 199 | } 200 | elsif (/^END/) 201 | { 202 | if( $#{$currentpoints} > 0 ) { 203 | # close polygon if it isn't 204 | push(@{$currentpoints}, [$currentpoints->[0][0], $currentpoints->[0][1]]) 205 | if $currentpoints->[0][0] != $currentpoints->[-1][0] 206 | || $currentpoints->[0][1] != $currentpoints->[-1][1]; 207 | my $pol = Math::Polygon::Tree->new($currentpoints); 208 | push(@{$borderpolys}, [$pol,$invert,[$pol->bbox]]); 209 | printf STDERR "Added polygon: %d points (%d,%d)-(%d,%d) %s\n", 210 | scalar(@{$currentpoints}), @{$borderpolys->[-1][2]}, 211 | ($borderpolys->[-1][1] ? "exclude" : "include") if $verbose; 212 | } 213 | undef $currentpoints; 214 | } 215 | elsif (defined($currentpoints)) 216 | { 217 | /^\s+([0-9.E+-]+)\s+([0-9.E+-]+)/ or die "Incorrent line in poly: $_"; 218 | push(@{$currentpoints}, [$2, $1]); 219 | if( !$bboxset ) { 220 | $minlat = $2 if ($2 < $minlat); 221 | $maxlat = $2 if ($2 > $maxlat); 222 | $minlon = $1 if ($1 < $minlon); 223 | $maxlon = $1 if ($1 > $maxlon); 224 | } 225 | } 226 | } 227 | close (PF); 228 | return $borderpolys; 229 | } 230 | 231 | sub poly_from_bbox { 232 | my @b = ($minlat, $minlon, $maxlat, $maxlon); 233 | my $currentpoints = [[$b[0], $b[1]], [$b[0], $b[3]], [$b[2], $b[3]], [$b[2], $b[1]], [$b[0], $b[1]]]; 234 | my $pol = Math::Polygon::Tree->new($currentpoints); 235 | my $bp = []; 236 | push(@{$bp}, [$pol,0,[$pol->bbox]]); 237 | return $bp; 238 | } 239 | 240 | sub usage { 241 | my ($msg) = @_; 242 | print STDERR "$msg\n\n" if defined($msg); 243 | 244 | my $prog = basename($0); 245 | print STDERR << "EOF"; 246 | This script receives a tar archive of GPX files (from STDIN) 247 | and filters it by a bounding box, an osmosis' polygon filter file 248 | or a number of them. 249 | 250 | usage: $prog [-h] [-o target] [-z] [-b bbox] [ -p poly | -l list [-s suffix] ] 251 | 252 | -h : print ths help message and exit. 253 | -o target : a file or a directory to put resulting files. 254 | -z : compress output files with gzip. 255 | -b bbox : limit points by bounding box (four comma-separated 256 | numbers: minlon,minlat,maxlon,maxlat). 257 | -p poly : a polygon filter file. 258 | -l list : file with names of poly files. 259 | -s suffix : a suffix to add to all output files. 260 | -v : print debug messages. 261 | 262 | Resulting files will have .metadata.tarc counterparts if input 263 | file had metadata.xml file. Those should be concatenated with 264 | their respective tar files, so they are first in resulting archives. 265 | 266 | Please keep the number of polygons below 100, or unexpected problems 267 | may occur. 268 | 269 | EOF 270 | exit; 271 | } 272 | -------------------------------------------------------------------------------- /bitiles2png.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # Create tiles out of processed bitiles. 4 | # Written by Ilya Zverev, licensed WTFPL. 5 | 6 | use strict; 7 | use GD; 8 | use Bit::Vector; 9 | use File::Path 2.07 qw(make_path); 10 | use File::Basename; 11 | use Getopt::Long; 12 | use Math::Trig; 13 | use Fcntl qw( O_RDONLY O_RDWR O_CREAT O_BINARY ); 14 | 15 | my $source_dir; 16 | my $dest_dir; 17 | my $delete_original; 18 | my $keep; 19 | my $zoom; 20 | my $colour_str = '0,0,0'; 21 | my $uptozoom; 22 | my $zoom0; 23 | my $make_empty_tiles; 24 | my $help; 25 | my $verbose; 26 | my $bbox_str = '-180,-85,180,85'; 27 | my @tile_bounds = (0, 0, 2**18, 2**18); 28 | 29 | GetOptions('h|help' => \$help, 30 | 'v|verbose' => \$verbose, 31 | 'i|input=s' => \$source_dir, 32 | 'o|output=s' => \$dest_dir, 33 | 'z|zoom=i' => \$zoom, 34 | 'u|uptozoom=i' => \$uptozoom, 35 | '0|zoom0' => \$zoom0, 36 | 'd|delorig' => \$delete_original, 37 | 'k|keep' => \$keep, 38 | 'e|empty' => \$make_empty_tiles, 39 | 'c|colour=s' => \$colour_str, 40 | 'b|bbox=s' => \$bbox_str, 41 | ) || usage(); 42 | 43 | if( $help ) { 44 | usage(); 45 | } 46 | 47 | usage("Please specify input directory with -i") unless defined($source_dir); 48 | die "Source directory not found" unless -d $source_dir; 49 | $dest_dir = $source_dir unless defined($dest_dir); 50 | 51 | $zoom = find_zoom($source_dir) unless defined($zoom); 52 | $uptozoom = $zoom unless defined($uptozoom) && $uptozoom <= $zoom && $uptozoom >= 0; 53 | $uptozoom = 0 if $zoom0; 54 | 55 | die "Colour should be three comma-separated numbers: r,g,b" unless $colour_str =~ /^\d{1,3},\d{1,3},\d{1,3}$/; 56 | my @colour = split(",", $colour_str); 57 | 58 | set_tile_bounds($bbox_str) if $bbox_str; 59 | 60 | my $xmin; 61 | my $ymin; 62 | my $xmax; 63 | my $ymax; 64 | 65 | process_zoom($zoom, 1); 66 | 67 | if( $uptozoom < $zoom && $xmin <= $xmax && $ymin <= $ymax ) { 68 | for my $zoomd (1 .. $zoom-$uptozoom) { 69 | my $newzoom = $zoom - $zoomd; 70 | generate_lowzoom($newzoom); 71 | clean_bitiles($newzoom + 1) if !$keep && ($delete_original || $newzoom + 1 < $zoom); 72 | process_zoom($newzoom, 0); 73 | } 74 | } 75 | clean_bitiles($uptozoom) if $delete_original || $uptozoom < $zoom; 76 | 77 | sub generate_lowzoom { 78 | my $zoom = shift; 79 | my $z = $zoom + 1; 80 | 81 | print STDERR "Generating zoom $zoom bitiles" if $verbose; 82 | for( my $x = $xmin - $xmin%2; $x <= $xmax; $x += 2 ) { 83 | next if !-d "$source_dir/$z/$x" && !-d "$source_dir/$z/".($x+1); 84 | print STDERR '.' if $verbose; 85 | make_path("$source_dir/$zoom/".($x/2)); 86 | for( my $y = $ymin - $ymin%2; $y <= $ymax; $y += 2 ) { 87 | quadtile( 88 | "$source_dir/$z/$x/$y.bitile", 89 | "$source_dir/$z/".($x+1)."/$y.bitile", 90 | "$source_dir/$z/$x/".($y+1).".bitile", 91 | "$source_dir/$z/".($x+1)."/".($y+1).".bitile", 92 | "$source_dir/$zoom/".($x/2)."/".($y/2).".bitile" 93 | ); 94 | } 95 | } 96 | print STDERR "Done\n" if $verbose; 97 | } 98 | 99 | sub quadtile { 100 | # (x,y) (x+1,y) (x,y+1) (x+1,y+1) (x/2,y/2) 101 | my ($b1file, $b2file, $b3file, $b4file, $result) = @_; 102 | return if !-r $b1file && !-r $b2file && !-r $b3file && !-r $b4file; 103 | 104 | my @btiles = (read_bit_vector($b1file), read_bit_vector($b2file), 105 | read_bit_vector($b3file), read_bit_vector($b4file)); 106 | my $res = Bit::Vector->new(65536); 107 | 108 | for my $y (0..255) { 109 | for my $x (0..255) { 110 | my $b = ($y >> 7) * 2 + ($x >> 7); 111 | my $bx = ($x * 2) % 256; 112 | my $by = ($y * 2) % 256; 113 | $res->Bit_On($y * 256 + $x ) 114 | if $btiles[$b]->bit_test($by * 256 + $bx) 115 | || $btiles[$b]->bit_test($by * 256 + $bx + 1) 116 | || $btiles[$b]->bit_test(($by + 1) * 256 + $bx) 117 | || $btiles[$b]->bit_test(($by + 1) * 256 + $bx + 1); 118 | } 119 | } 120 | 121 | sysopen(BITILE, $result, O_RDWR | O_CREAT | O_BINARY); 122 | syswrite(BITILE, $res->Block_Read()); 123 | close BITILE; 124 | } 125 | 126 | sub process_zoom { 127 | my ($zoom, $skip) = @_; 128 | print STDERR "Generating PNG images for zoom $zoom" if $verbose; 129 | 130 | $xmin = 10**6; 131 | $ymin = 10**6; 132 | $xmax = -10**6; 133 | $ymax = -10**6; 134 | 135 | my $dh; 136 | if( !opendir($dh, "$source_dir/$zoom") ) { 137 | print STDERR "Fail: cannot open $source_dir/$zoom\n"; 138 | return; 139 | } 140 | my @xlist = grep { /^\d+$/ && -d "$source_dir/$zoom/$_" } readdir($dh); 141 | closedir($dh); 142 | if( $#xlist < 0 ) { 143 | print STDERR "No tiles there\n"; 144 | return; 145 | } 146 | 147 | for my $x (@xlist) { 148 | next if $skip && ($x < $tile_bounds[0] || $x > $tile_bounds[2]); 149 | my $folder = "$source_dir/$zoom/$x"; 150 | opendir($dh, $folder) || next; 151 | my @ylist = grep { /^\d+\.bitile$/ && -r "$folder/$_" } readdir($dh); 152 | closedir($dh); 153 | 154 | print STDERR "." if $verbose; 155 | make_path("$dest_dir/$zoom/$x"); 156 | 157 | for my $y (@ylist) { 158 | $y =~ /^(\d+)/; 159 | my $yt = $1; 160 | next if $skip && ($yt < $tile_bounds[1] || $yt > $tile_bounds[3]); 161 | 162 | my $vec = read_bit_vector("$folder/$y"); 163 | my $tile = vector2png($vec); 164 | open PIC, ">$dest_dir/$zoom/$x/$yt.png"; 165 | binmode PIC; 166 | print PIC $tile->png(); 167 | close PIC; 168 | 169 | $xmin = $x if $x < $xmin; 170 | $xmax = $x if $x > $xmax; 171 | $ymin = $yt if $yt < $ymin; 172 | $ymax = $yt if $yt > $ymax; 173 | } 174 | } 175 | if( $make_empty_tiles ) { 176 | print STDERR "Empty tiles..." if $verbose; 177 | generate_empty_tiles($zoom); 178 | } 179 | print STDERR "Done\n" if $verbose; 180 | } 181 | 182 | sub read_bit_vector { 183 | my $filename = shift; 184 | my $vec = Bit::Vector->new(65536); 185 | if( sysopen(BITILE, $filename, O_RDONLY | O_BINARY) ) { 186 | if( sysread(BITILE, my $read, 8192) == 8192 ) { 187 | $vec->Block_Store($read); 188 | } 189 | close BITILE; 190 | } 191 | return $vec; 192 | } 193 | 194 | sub vector2png { 195 | my $vec = shift; 196 | my $tile = GD::Image->new(256, 256); 197 | my $transp = $tile->colorAllocate(253,254,255); 198 | $tile->transparent($transp); 199 | $tile->filledRectangle(0,0,255,255,$transp); 200 | my $col = $tile->colorAllocate($colour[0], $colour[1], $colour[2]); 201 | 202 | for my $yy (0..255) { 203 | for my $xx (0..255) { 204 | if( $vec->bit_test($yy * 256 + $xx) ) { 205 | $tile->setPixel($xx, $yy, $col); 206 | } 207 | } 208 | } 209 | return $tile; 210 | } 211 | 212 | sub generate_empty_tiles { 213 | my $z = shift; 214 | 215 | my $img = GD::Image->new(256, 256); 216 | my $white = $img->colorAllocate(255, 255, 255); 217 | $img->transparent($white); 218 | $img->filledRectangle(0,0,255,255,$white); 219 | my $empty_tile = $img->png(); 220 | 221 | for my $x ($xmin .. $xmax) { 222 | for my $y ($ymin .. $ymax) { 223 | my $filename = "$dest_dir/$z/$x/$y.png"; 224 | if( !-f $filename ) { 225 | open PIC, ">$filename"; 226 | binmode PIC; 227 | print PIC $empty_tile; 228 | close PIC; 229 | } 230 | } 231 | } 232 | } 233 | 234 | sub clean_bitiles { 235 | my $zoom = shift; 236 | 237 | print STDERR "Removing zoom $zoom bitiles\n" if $verbose; 238 | opendir(my $dh, "$source_dir/$zoom") || return; 239 | my @xlist = grep { /^\d+$/ && -d "$source_dir/$zoom/$_" } readdir($dh); 240 | closedir($dh); 241 | return if $#xlist < 0; 242 | 243 | for my $x (@xlist) { 244 | my $folder = "$source_dir/$zoom/$x"; 245 | opendir($dh, $folder) || next; 246 | my @ylist = grep { /^\d+\.bitile$/ && -r "$folder/$_" } readdir($dh); 247 | closedir($dh); 248 | 249 | for my $y (@ylist) { 250 | $y =~ /^(\d+)/; 251 | my $yt = $1; 252 | unlink "$folder/$y" if -f "$dest_dir/$zoom/$x/$yt.png"; 253 | } 254 | rmdir $folder; 255 | } 256 | rmdir "$source_dir/$zoom"; 257 | } 258 | 259 | sub set_tile_bounds { 260 | my $bbox_str = shift; 261 | my @bbox = split(",", $bbox_str); 262 | die ("badly formed bounding box - use four comma-separated values for left longitude, ". 263 | "bottom latitude, right longitude, top latitude") unless $#bbox == 3; 264 | die("max longitude is less than min longitude") if ($bbox[2] < $bbox[0]); 265 | die("max latitude is less than min latitude") if ($bbox[3] < $bbox[1]); 266 | 267 | my $zoom2 = 2**$zoom; 268 | my $eps = 10**-8; 269 | $tile_bounds[0] = int(($bbox[0]+$eps+180)/360 * $zoom2); 270 | $tile_bounds[2] = int(($bbox[2]-$eps+180)/360 * $zoom2); 271 | $tile_bounds[3] = int((1 - log(tan(deg2rad($bbox[1])) + sec(deg2rad($bbox[1])))/pi)/2 * $zoom2); 272 | $tile_bounds[1] = int((1 - log(tan(deg2rad($bbox[3])) + sec(deg2rad($bbox[3])))/pi)/2 * $zoom2); 273 | } 274 | 275 | sub find_zoom { 276 | my $dir = shift; 277 | my $dh; 278 | opendir($dh, $dir) or die "Cannot open $dir"; 279 | my @zlist = sort grep { /^\d+$/ && -d "$dir/$_" } readdir($dh); 280 | closedir($dh); 281 | die "No tiles found in $dir" if $#zlist < 0; 282 | return $zlist[-1]; 283 | } 284 | 285 | sub usage { 286 | my ($msg) = @_; 287 | print STDERR "$msg\n\n" if defined($msg); 288 | 289 | my $prog = basename($0); 290 | print STDERR << "EOF"; 291 | This script traverses input directory and creates PNG tiles from all 292 | bitiles found (created with gpx2bitiles.pl). 293 | 294 | usage: $prog [-h] [-v] -i source [-o target] [-z zoom] [-u zoom] [-0] [-d] [-e] [-c colour] 295 | 296 | -h : print ths help message and exit. 297 | -i source : directory with bitiles. 298 | -o target : directory to store PNG tiles (by default equal to source). 299 | -z zoom : bitiles zoom level. 300 | -u zoom : generate tiles up to that zoom level (e.g. 0). 301 | -0 : equivalent to -u 0. 302 | -b bbox : limit tiles by a bounding box (four comma-separated 303 | numbers: minlon,minlat,maxlon,maxlat). 304 | -d : delete processed bitiles. 305 | -k : keep all generated bitiles. 306 | -e : generate empty tiles where there are no bitiles. 307 | -c colour : colour of GPS points (three comma-separated numbers: r,g,b 308 | default is black). 309 | -v : display progress. 310 | 311 | EOF 312 | exit; 313 | } --------------------------------------------------------------------------------