├── .eslintignore ├── .eslintrc ├── .gitignore ├── LICENSE ├── README.md ├── bench ├── aggregations │ └── osm-roads.js ├── bench.js ├── data │ └── united_states_of_america.tiles.txt ├── osm-roads.js ├── read-tiles.js ├── results │ └── osm-roads.ndjson ├── run.sh ├── summarize └── summary.js ├── bin └── vt-grid ├── index.js ├── lib ├── aggregate-cells.js ├── aggregate.js ├── degenerate.js ├── geojson-wrapper.js └── tile-util.js ├── package.json └── test ├── fixture ├── aggregate-cells.input.geojson ├── dc.geojson ├── dc.mbtiles ├── dc.z12-grid-quadkeys.txt ├── degenerate-features-2.geojson └── degenerate-features.geojson ├── index.js └── lib ├── aggregate-cells.js └── degenerate.js /.eslintignore: -------------------------------------------------------------------------------- 1 | lib/geojson-wrapper.js 2 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["standard"], 3 | "env": { 4 | "node": true, 5 | "es6": true, 6 | "browser": true 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | coverage 13 | .nyc_output 14 | 15 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 16 | .grunt 17 | 18 | # node-waf configuration 19 | .lock-wscript 20 | 21 | # Compiled binary addons (http://nodejs.org/api/addons.html) 22 | build/Release 23 | 24 | # Dependency directory 25 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 26 | node_modules 27 | 28 | # Data 29 | temp 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Anand Thakker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- 24 | 25 | Contains the file 'geojson-wrapper.js' from mapbox-gl-js 26 | 27 | Copyright (c) 2014, Mapbox 28 | 29 | All rights reserved. 30 | 31 | Redistribution and use in source and binary forms, with or without modification, 32 | are permitted provided that the following conditions are met: 33 | 34 | * Redistributions of source code must retain the above copyright notice, 35 | this list of conditions and the following disclaimer. 36 | * Redistributions in binary form must reproduce the above copyright notice, 37 | this list of conditions and the following disclaimer in the documentation 38 | and/or other materials provided with the distribution. 39 | * Neither the name of Mapbox GL JS nor the names of its contributors 40 | may be used to endorse or promote products derived from this software 41 | without specific prior written permission. 42 | 43 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 44 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 45 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 46 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 47 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 48 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 49 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 50 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 51 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 52 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 53 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 54 | 55 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # vt-grid 2 | 3 | Build up a pyramid of [Mapbox vector 4 | tiles](https://github.com/mapbox/vector-tile-spec) by aggregating quantitative 5 | data into grids at lower zooms. 6 | 7 | ## Motivation 8 | 9 | Say you have a dataset of polygons that have some kind of density data 10 | (population, vegetation, ...), and you want to build an interactive map with 11 | it. Vector tiles are great for this--especially with 12 | [mapbox-gl](https://github.com/mapbox/mapbox-gl-js) steadily maturing. 13 | 14 | But if your data is at a fine resolution and you don't want to be limited to 15 | very high zoom levels, you're stuck using standard simplification techniques. 16 | (Or, much better, the rather badass and blazingly fast simplification and point 17 | dropping techniques offered by 18 | [tippecanoe](https://github.com/mapbox/tippecanoe)). For many cases, this works 19 | great, but it's not ideal here: for instance, in simplification many small, 20 | high-density polygons get dropped, even though these are often important 21 | features. 22 | 23 | This tool is an alternative to simplification: using a grid whose resolution 24 | varies with zoom level, aggregate the quantitative features of interest, so 25 | that you can visualize the spatial distribution of your data at any scale. 26 | 27 | ## Installation 28 | 29 | Install [tippecanoe](https://github.com/mapbox/tippecanoe), and then: 30 | 31 | ```sh 32 | npm install -g vt-grid 33 | ``` 34 | 35 | ## Usage 36 | 37 | To start, you'll need an `mbtiles` file containing the original feature data at 38 | some (high) zoom level. If you've got the data in, say, a shapefile or 39 | PostGIS, you can use Mapbox Studio to create a source and then export to 40 | MBTiles -- just set the min and max zoom to something high enough. 41 | 42 | ### CLI 43 | 44 | Let's say you've got the data in `data.mbtiles`, at zoom 12 in a layer called 45 | `'foo'`, and each polygon in this layer has a field called `density`. Then, you 46 | can build the grid pyramid above this base layer with: 47 | 48 | ```sh 49 | vt-grid input.mbtiles -o output.mbtiles --basezoom 12 --minzoom 1 --gridsize 16 \ 50 | --aggregations 'foo:areaWeightedMean(density)' 51 | ``` 52 | 53 | Starting at zoom 11 and going down to zoom 1, this will build a 16x16 grid in 54 | each tile, aggregating the data from the zoom level above. The aggregations 55 | are defined by the `--aggregations` parameters. Each one is of the form: 56 | `layer:aggregationFunction(field)`, where `aggregationFunction` can 57 | be any of the built-in aggregations available in 58 | [`geojson-polygon-aggregate`](https://github.com/anandthakker/geojson-polygon-aggregate). 59 | So, in this case, we'll end up with a grid where each box has a `density` 60 | property, which is the (correctly weighted) mean of the densities of the 61 | polygons from the previous (higher) zoom level that fall within that box. 62 | 63 | With other aggregations, other stats. For instance, we could have done: 64 | 65 | ```sh 66 | # first use count() to find out the number of polygons from the original 67 | # dataset being aggregated into each grid box at z11 68 | vt-grid input.mbtiles output.mbtiles --basezoom 12 --minzoom 11 --gridsize 16 \ 69 | --aggregations 'foo:areaWeightedMean(density)' 'foo:count(numzones)' 70 | 71 | # now, for z10 and below, sum the counts 72 | vt-grid input.mbtiles output.mbtiles --basezoom 12 --minzoom 11 --gridsize 16 \ 73 | --aggregations 'foo:areaWeightedMean(density)' 'foo:sum(numzones)' 74 | ``` 75 | 76 | ### Node 77 | 78 | You can have a little more flexibility with aggregations (and post-aggregation 79 | functions) by using vt-grid programmatically: 80 | 81 | ```javascript 82 | var path = require('path') 83 | var vtGrid = require('vt-grid') 84 | var reducers = require('geojson-polygon-aggregate/reducers') 85 | 86 | if (require.main === module) { 87 | vtGrid('/path/to/output.mbtiles', 'path/to/input.mbtiles', { 88 | minzoom: 1, 89 | basezoom: 10, 90 | aggregations: __filename, // this can be any file that exports an `aggregations` object like the one below 91 | postAggregations: __filename // same for this 92 | }, function (err) { 93 | if (err) { throw err } 94 | console.log('Finished!') 95 | }) 96 | } 97 | 98 | module.exports = { 99 | aggregations: { 100 | footprints: { 101 | FID: reducers.union('FID'), 102 | someField: function myCustomAggregator (memo, feature) { 103 | var newMemo = -1 104 | // do stuff, works like an Array.reduce() function 105 | return newMemo 106 | } 107 | } 108 | }, 109 | postAggregations: { 110 | footprints: { 111 | // called on each grid square feature after all aggregations are run, with 112 | // the result added to its properties under the given key (unique_count) 113 | unique_count: function (feature) { 114 | return feature.properties.FID ? JSON.parse(feature.properties.FID).length : 0 115 | } 116 | } 117 | } 118 | } 119 | ``` 120 | 121 | This yields features that look like: 122 | 123 | ```json 124 | { 125 | "type": "Feature", 126 | "geometry": { 127 | "type": "Polygon", 128 | "coordinates": [ 129 | [ 130 | [ 131 | -111.09375, 132 | 40.97989806962016 133 | ], 134 | [ 135 | -111.09375, 136 | 40.9964840143779 137 | ], 138 | [ 139 | -111.07177734375, 140 | 40.9964840143779 141 | ], 142 | [ 143 | -111.07177734375, 144 | 40.97989806962016 145 | ], 146 | [ 147 | -111.09375, 148 | 40.97989806962016 149 | ] 150 | ] 151 | ] 152 | }, 153 | "properties": { 154 | "FID": "[59, 707, 1002]", 155 | "unique_count": 3, 156 | "someField": -1 157 | } 158 | } 159 | ``` 160 | 161 | ## API 162 | 163 | ### vtGrid 164 | 165 | Build a pyramid of aggregated square-grid features. 166 | 167 | **Parameters** 168 | 169 | - `output` **[string](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String)** Path to output aggregated mbtiles data 170 | - `input` **[string](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String)** Path to the input mbtiles data 171 | - `opts` **([Object](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object)\|[Array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array))** Options OR an array of options objects to allow different aggregations/settings for different zoom levels 172 | - `opts.basezoom` **[number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number)** The zoom level at which to find the initial data 173 | - `opts.inputTiles` **[Array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array)=** An array of [z, x, y] tile coordinates to start with 174 | - `opts.gridsize` **[number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number)** Number of grid squares per tile 175 | - `opts.aggregations` **([Object](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object)\|[string](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String))** If an object, then it maps layer names to aggregation objects, which themselves map field names to geojson-polygon-aggregate aggregation function names. Each worker will construct the actual aggregation function from geojson-polygon-aggregate by passing it the field name as an argument. If a string, then it's the path of a module that exports a layer to aggregation object map (see `#grid` for details). 176 | - `opts.postAggregations` **[string](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String)=** Path to a module mapping layer names to postAggregations objects. See `#grid` for details. 177 | - `opts.jobs` **[number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number)** The number of jobs to run in parallel. 178 | - `done` **[function](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/function)** called with (err) when done 179 | 180 | ## Built With 181 | 182 | - [Turf.js](http://turfjs.org), 183 | [geojson-vt](https://github.org/mapbox/geojson-vt), and several other super 184 | fly modules by [Mapbox](https://github.com/mapbox) 185 | - Also, several conversations with @morganherlocker (the author of many of the 186 | aforementioned modules, including Turf.) 187 | -------------------------------------------------------------------------------- /bench/aggregations/osm-roads.js: -------------------------------------------------------------------------------- 1 | var flatten = require('lodash.flatten') 2 | var cheapRuler = require('cheap-ruler') 3 | 4 | module.exports = { 5 | aggregations: { 6 | osm: { 7 | roads_km: function (memo, feature, _, tile) { 8 | if (!feature.properties.highway) { return memo } 9 | 10 | memo = memo || 0 11 | var ruler = getRuler(tile) 12 | return memo + totalLineDistance(ruler, feature.geometry) 13 | } 14 | } 15 | } 16 | } 17 | 18 | function totalLineDistance (ruler, geometry) { 19 | var lines 20 | if (geometry.type === 'MultiPolygon') { 21 | // polygons -> rings -> coordinates 22 | // [ [ [ [x1, y1], [x2, y2], ... ], [...] ] ] 23 | lines = flatten(geometry.coordinates) 24 | } else if (geometry.type === 'Polygon' || geometry.type === 'MultiLineString') { 25 | // this is what we want 26 | // [ [ [x1, y1], [x2, y2], ... ], [...] ] 27 | lines = geometry.coordinates 28 | } else if (geometry.type === 'LineString') { 29 | // wrap in an array 30 | lines = [geometry.coordinates] 31 | } else { 32 | return 0 33 | } 34 | 35 | var sum = 0 36 | for (var i = 0; i < lines.length; i++) { 37 | sum += ruler.lineDistance(lines[i]) 38 | } 39 | return sum 40 | } 41 | 42 | var cache = {} 43 | function getRuler (tile) { 44 | var key = tile[1] 45 | if (!cache[key]) { 46 | cache[key] = cheapRuler.fromTile(tile[1], tile[0]) 47 | } 48 | return cache[key] 49 | } 50 | 51 | -------------------------------------------------------------------------------- /bench/bench.js: -------------------------------------------------------------------------------- 1 | var flat = require('flat') 2 | var queue = require('queue-async') 3 | 4 | var q 5 | 6 | module.exports = function bench (label, fn) { 7 | if (!q) { 8 | q = queue(1) 9 | setImmediate(function () { 10 | q.awaitAll(function (err) { if (err) { console.error(err) } }) 11 | q = null 12 | }) 13 | } 14 | 15 | q.defer(function (done) { 16 | var start 17 | var b = { 18 | start: function () { 19 | start = Date.now() 20 | }, 21 | result: function (err, data) { 22 | report({ 23 | label: label, 24 | elapsed: Date.now() - start, 25 | error: err, 26 | data: data 27 | }) 28 | start = Date.now() 29 | }, 30 | end: function (err, data) { 31 | b.result(err, data) 32 | done() 33 | } 34 | } 35 | 36 | try { 37 | b.start() 38 | fn(b, b.end.bind(b)) 39 | } catch (e) { 40 | b.end(e.message || e) 41 | done() 42 | } 43 | }) 44 | } 45 | 46 | function report (result, data) { 47 | console.log(JSON.stringify(flat(result))) 48 | } 49 | -------------------------------------------------------------------------------- /bench/osm-roads.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var path = require('path') 3 | var bench = require('./bench') 4 | var aggregate = require('../lib/aggregate') 5 | var readTiles = require('./read-tiles') 6 | 7 | var input = 'mbtiles://' + path.join(__dirname, 'data/united_states_of_america.mbtiles') 8 | var tiles = fs.readFileSync(path.join(__dirname, 'data/united_states_of_america.tiles.txt'), 'utf-8') 9 | .split('\n') 10 | .slice(0, 10000) 11 | 12 | var tileStream = readTiles(input).on('data', runBenchmark) 13 | tiles.forEach(function (tile) { 14 | tile = tile.split(' ').slice(1) 15 | tile = [tile[1], tile[2], tile[0]].map(Number) 16 | tileStream.write(tile) 17 | }) 18 | 19 | function runBenchmark (tileData) { 20 | var tile = tileData.tile 21 | var features = tileData.features 22 | var data = { data: { layer: { type: 'FeatureCollection', features: features } } } 23 | var writeData = function () {} 24 | 25 | bench('osm/1024-grid/no-aggregations', function (b, done) { 26 | aggregate._setup({ 27 | aggregations: { layer: {} }, 28 | gridsize: 1024 29 | }) 30 | b.start() 31 | aggregate(data, tile, writeData, done) 32 | }) 33 | 34 | bench('osm/1024-grid/road-length', function (b, done) { 35 | aggregate._setup({ 36 | aggregations: path.join(__dirname, 'aggregations/osm-roads.js'), 37 | gridsize: 1024 38 | }) 39 | b.start() 40 | aggregate(data, tile, writeData, done) 41 | }) 42 | } 43 | 44 | -------------------------------------------------------------------------------- /bench/read-tiles.js: -------------------------------------------------------------------------------- 1 | var zlib = require('zlib') 2 | var Pbf = require('pbf') 3 | var VectorTile = require('vector-tile').VectorTile 4 | var MBTiles = require('mbtiles') 5 | var through = require('through2') 6 | 7 | module.exports = function (mbtiles) { 8 | var db 9 | 10 | return through.obj(write) 11 | 12 | function write (tile, _, next) { 13 | var self = this 14 | if (!db) { 15 | db = new MBTiles(mbtiles, function (err) { 16 | if (err) { return next(err) } 17 | writeTile.call(self, tile, next) 18 | }) 19 | } else { 20 | writeTile.call(self, tile, next) 21 | } 22 | } 23 | 24 | function writeTile (tile, next) { 25 | var self = this 26 | var x = tile[0] 27 | var y = tile[1] 28 | var z = tile[2] 29 | db.getTile(z, x, y, function (err, tiledata) { 30 | if (err) { return next(err) } 31 | zlib.gunzip(tiledata, function (err, pbfdata) { 32 | if (err) { return next(err) } 33 | var vt = new VectorTile(new Pbf(pbfdata)) 34 | var features = [] 35 | for (var l in vt.layers) { 36 | var layer = vt.layers[l] 37 | for (var j = 0; j < layer.length; j++) { 38 | features.push(layer.feature(j).toGeoJSON(x, y, z)) 39 | } 40 | } 41 | self.push({ tile: tile, features: features }) 42 | next() 43 | }) 44 | }) 45 | } 46 | } 47 | 48 | -------------------------------------------------------------------------------- /bench/results/osm-roads.ndjson: -------------------------------------------------------------------------------- 1 | {"label":"osm/1024-grid/no-aggregations","n":10000,"elapsed:mean":1.3253999999999997,"elapsed:standard_deviation":1.3253999999999997,"data.layers.layer:mean":9.04780000000001} 2 | {"label":"osm/1024-grid/road-length","n":10000,"elapsed:mean":1.298699999999996,"elapsed:standard_deviation":1.298699999999996,"data.layers.layer:mean":9.04780000000001} 3 | -------------------------------------------------------------------------------- /bench/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | node osm-roads.js | ./summarize elapsed:mean elapsed:standard_deviation data.layers.layer:mean > results/osm-roads.ndjson 4 | -------------------------------------------------------------------------------- /bench/summarize: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict' 3 | 4 | const split = require('split') 5 | const through = require('through2') 6 | const streamStatistics = require('stream-statistics') 7 | 8 | module.exports = summaryStream 9 | function summaryStream (columns) { 10 | let summaries = {} 11 | return through.obj(write, end) 12 | 13 | function write (data, _, next) { 14 | let self = this 15 | let summary = summaries[data.label] 16 | if (!summary) { 17 | // set up summary streams 18 | summary = summaries[data.label] = { 19 | streams: {}, 20 | results: { label: data.label, n: 0 }, 21 | pending: 0 22 | } 23 | columns.forEach(function (column) { 24 | let parsed = column.split(':') 25 | let k = parsed[0] 26 | let stat = parsed[1] 27 | if (summary.streams[k]) { return } 28 | 29 | summary.pending++ 30 | summary.streams[k] = streamStatistics() 31 | .on('data', function (summarized) { 32 | columns 33 | .filter((c) => c.startsWith(k + ':')) 34 | .forEach((column) => { summary.results[column] = summarized[stat] }) 35 | }) 36 | .on('end', function () { 37 | summary.pending-- 38 | if (summary.pending === 0) { 39 | self.push(summary.results) 40 | for (let l in summaries) { if (summaries[l].pending) { return } } 41 | self.push(null) 42 | } 43 | }) 44 | }) 45 | } 46 | 47 | summary.results.n++ 48 | for (let k in summary.streams) { summary.streams[k].write(data[k]) } 49 | next() 50 | } 51 | 52 | function end () { 53 | for (let label in summaries) { 54 | let summary = summaries[label] 55 | for (let k in summary.streams) { 56 | summary.streams[k].end() 57 | } 58 | } 59 | } 60 | } 61 | 62 | if (require.main === module) { 63 | process.stdin.pipe(split()) 64 | .pipe(through.obj(function (line, _, next) { 65 | if (line && line.length) { 66 | next(null, JSON.parse(line)) 67 | } else { 68 | next() 69 | } 70 | })) 71 | .pipe(summaryStream(process.argv.slice(2))) 72 | .pipe(through.obj(function (data, _, next) { 73 | next(null, JSON.stringify(data) + '\n') 74 | })) 75 | .pipe(process.stdout) 76 | } 77 | -------------------------------------------------------------------------------- /bench/summary.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict' 3 | 4 | const split = require('split') 5 | const through = require('through2') 6 | const streamStatistics = require('stream-statistics') 7 | 8 | module.exports = summaryStream 9 | function summaryStream (columns) { 10 | let summaries = {} 11 | return through.obj(write, end) 12 | 13 | function write (data, _, next) { 14 | let self = this 15 | let summary = summaries[data.label] 16 | if (!summary) { 17 | // set up summary streams 18 | summary = summaries[data.label] = { 19 | streams: {}, 20 | results: { label: data.label }, 21 | pending: 0 22 | } 23 | columns.forEach(function (column) { 24 | let parsed = column.split(':') 25 | let k = parsed[0] 26 | let stat = parsed[1] 27 | if (summary.streams[k]) { return } 28 | 29 | summary.pending++ 30 | summary.streams[k] = streamStatistics() 31 | .on('data', function (summarized) { 32 | columns 33 | .filter((c) => c.startsWith(k + ':')) 34 | .forEach((column) => { summary.results[column] = summarized[stat] }) 35 | }) 36 | .on('end', function () { 37 | summary.pending-- 38 | if (summary.pending === 0) { 39 | self.push(summary.results) 40 | for (let l in summaries) { if (summaries[l].pending) { return } } 41 | self.push(null) 42 | } 43 | }) 44 | }) 45 | } 46 | 47 | for (let k in summary.streams) { summary.streams[k].write(data[k]) } 48 | next() 49 | } 50 | 51 | function end () { 52 | for (let label in summaries) { 53 | let summary = summaries[label] 54 | for (let k in summary.streams) { 55 | summary.streams[k].end() 56 | } 57 | } 58 | } 59 | } 60 | 61 | if (require.main === module) { 62 | process.stdin.pipe(split()) 63 | .pipe(through.obj(function (line, _, next) { 64 | if (line && line.length) { 65 | next(null, JSON.parse(line)) 66 | } else { 67 | next() 68 | } 69 | })) 70 | .pipe(summaryStream(process.argv.slice(2))) 71 | .pipe(through.obj(function (data, _, next) { 72 | next(null, JSON.stringify(data) + '\n') 73 | })) 74 | .pipe(process.stdout) 75 | } 76 | -------------------------------------------------------------------------------- /bin/vt-grid: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var os = require('os') 3 | var vtGrid = require('../') 4 | 5 | var validAggregations = Object.keys(require('geojson-polygon-aggregate/reducers')) 6 | 7 | var argv = require('yargs') 8 | .usage('$0 input.mbtiles -o output.mbtiles [--minzoom 7] [--basezoom 12] [--gridsize 1024] --aggregations \'layerName:areaWeightedMean(fieldName)\' \'layerName:count()\'') 9 | .demand(1) 10 | .alias('output', 'o') 11 | .array('aggregations') 12 | .demand('aggregations') 13 | .describe('aggregations', 'The aggregations to perform, either as a js module (see docs), or in the form \'layerName:aggregationFunction(fieldName)\', aggregationFunction is one of: ' + validAggregations.join(', ')) 14 | .describe('postAggregations', 'Module exporting post-aggregation functions to apply (see docs for details).') 15 | .default('minzoom', 1) 16 | .describe('minzoom', 'The lowest zoom level at which to build the grid.') 17 | .default('gridsize', 1024) 18 | .describe('gridsize', 'The number of grid squares per tile. Must be a power of 4.') 19 | .describe('basezoom', 'The zoom level at which to start building (initial data should exist at z-basezoom in input.mbtiles).') 20 | .default('basezoom', Infinity) 21 | .default('jobs', os.cpus().length) 22 | .describe('jobs', 'The number of concurrent processes to run') 23 | .describe('quiet', 'Suppress log output') 24 | .help('h') 25 | .argv 26 | 27 | if (argv.aggregations.length === 1 && /\.js/.test(argv.aggregations[0])) { 28 | argv.aggregations = argv.aggregations[0] 29 | } else { 30 | var aggregations = {} 31 | argv.aggregations.forEach(function (field) { 32 | // layer:func(inField) 33 | var match = /([^:]+):([^\(]+)\((.*)\)/.exec(field) 34 | var layer = match[1] 35 | var fn = match[2] 36 | var fieldName = match[3] 37 | if (!aggregations[layer]) { aggregations[layer] = {} } 38 | aggregations[layer][fieldName] = fn 39 | if (validAggregations.indexOf(fn) < 0) { 40 | throw new Error('Unknown aggregation function: ' + fn) 41 | } 42 | }) 43 | 44 | argv.aggregations = aggregations 45 | } 46 | 47 | vtGrid(argv.output, argv._[0], argv, function (err) { 48 | if (err) { console.error(err) } 49 | }) 50 | 51 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var fs = require('fs') 4 | var os = require('os') 5 | var spawn = require('child_process').spawn 6 | var path = require('path') 7 | var tmp = require('tmp') 8 | var MBTiles = require('mbtiles') 9 | var tileReduce = require('tile-reduce') 10 | var log = require('single-line-log').stderr 11 | var prettyMs = require('pretty-ms') 12 | 13 | var debug = require('debug') 14 | var debugLog = debug('vt-grid:main') 15 | var debugEnabled = debug.enabled('vt-grid:main') 16 | 17 | if (!debugEnabled) { 18 | tmp.setGracefulCleanup() 19 | } 20 | 21 | module.exports = vtGrid 22 | 23 | /** 24 | * Build a pyramid of aggregated square-grid features. 25 | * 26 | * @param {string} output Path to output aggregated mbtiles data 27 | * @param {string} input Path to the input mbtiles data 28 | * @param {Object|Array} options Options OR an array of options objects to allow different aggregations/settings for different zoom levels 29 | * @param {number} options.basezoom The zoom level at which to find the initial data 30 | * @param {Array} [options.tiles] An array of [z, x, y] tile coordinates to start with 31 | * @param {Array} [options.bbox] A [w, s, e, n] bbox defining the area to start with 32 | * @param {number} options.gridsize Number of grid squares per tile 33 | * @param {Object|string} options.aggregations If an object, then it maps layer names to aggregation objects, which themselves map field names to geojson-polygon-aggregate aggregation function names. Each worker will construct the actual aggregation function from geojson-polygon-aggregate by passing it the field name as an argument. If a string, then it's the path of a module that exports a layer to aggregation object map (see {@link #grid} for details). 34 | * @param {string} [options.postAggregations] - Path to a module mapping layer names to postAggregations objects. See {@link #grid} for details. 35 | * @param {boolean} [options.includeBaseData=true] Set false to exclude the base-level data from the merged output. 36 | * @param {number} options.jobs The number of jobs to run in parallel. 37 | * @param {boolean} [options.quiet=false] Disable log output 38 | * @param {function} callback called with (err) when done 39 | */ 40 | 41 | function vtGrid (output, input, options, callback) { 42 | // allow an array of options, each defining different parts of the pyramid, 43 | // to allow different aggregations at different parts (often needed for 44 | // setting up the first aggregation layer) 45 | var optionStack = Array.isArray(options) ? options : [options] 46 | optionStack = optionStack.map(function (o) { 47 | return Object.assign({ 48 | jobs: os.cpus().length, 49 | basezoom: Infinity, 50 | includeBaseData: true 51 | }, o) 52 | }) 53 | .sort(function (a, b) { return b.basezoom - a.basezoom }) 54 | // check that the zoom levels covered by each set of options make sense 55 | optionStack.forEach(function (o, i) { 56 | if (i > 0 && o.basezoom !== optionStack[i - 1].minzoom) { 57 | throw new Error('Basezoom of each option set must match minzoom of previous set.') 58 | } 59 | }) 60 | 61 | var stats = { tiles: 0, zoomLevels: {}, layers: {}, start: Date.now() } 62 | var currentOptions = optionStack.shift() 63 | var currentState = '' // aggregating | tiling 64 | var currentZoom 65 | var zoomLevelFiles = currentOptions.includeBaseData ? [input] : [] 66 | 67 | var timer = setInterval(logProgress, 100) 68 | 69 | function done (err, data) { 70 | logProgress(true) 71 | clearInterval(timer) 72 | if (callback) { callback(err, data) } 73 | } 74 | 75 | getInfo(input, function (err, info) { 76 | if (err) { return done(err) } 77 | if (isNaN(currentOptions.basezoom) || currentOptions.basezoom === Infinity) { 78 | currentOptions.basezoom = info.minzoom 79 | } 80 | currentZoom = currentOptions.basezoom - 1 81 | 82 | // Hack: just use the first layer name from the source data 83 | // Upstream issue in tippecanoe will allow removing this hack 84 | // https://github.com/mapbox/tippecanoe/issues/188 85 | if (!currentOptions.layer) { currentOptions.layer = info.vector_layers[0].id } 86 | optionStack.forEach(function (o) { o.layer = o.layer || currentOptions.layer }) 87 | 88 | tmp.dir({unsafeCleanup: !debugEnabled}, function (err, tmpdir) { 89 | if (err) { return done(err) } 90 | buildZoomLevel(tmpdir, input) 91 | }) 92 | }) 93 | 94 | function buildZoomLevel (tmpdir, input) { 95 | var outputTiles = path.join(tmpdir, 'z' + currentZoom + '.mbtiles') 96 | var outputGeojson = path.join(tmpdir, 'z' + currentZoom + '.json') 97 | var outputStream = output 98 | ? fs.createWriteStream(outputGeojson) 99 | : process.stdout 100 | 101 | debugLog('build zoom level', outputGeojson, outputTiles) 102 | 103 | var tileReduceOptions = { 104 | map: path.join(__dirname, 'lib/aggregate.js'), 105 | sources: [{ name: 'data', mbtiles: input }], 106 | zoom: currentZoom + 1, 107 | maxWorkers: currentOptions.jobs, 108 | mapOptions: currentOptions, 109 | output: outputStream, 110 | log: false 111 | } 112 | 113 | if (currentOptions.tiles) { 114 | tileReduceOptions.tiles = currentOptions.tiles 115 | } else if (currentOptions.bbox) { 116 | tileReduceOptions.bbox = currentOptions.bbox 117 | if (typeof tileReduceOptions.bbox === 'string') { 118 | tileReduceOptions.bbox = tileReduceOptions.bbox.split(',').map(Number) 119 | } 120 | } else { 121 | tileReduceOptions.sourceCover = 'data' 122 | } 123 | 124 | stats.zoomLevels[currentZoom] = { features: 0, tiles: 0, start: Date.now() } 125 | currentState = 'aggregating' 126 | 127 | tileReduce(tileReduceOptions) 128 | .on('reduce', function (data) { 129 | stats.tiles++ 130 | stats.zoomLevels[currentZoom].tiles++ 131 | for (var k in data.layers) { 132 | stats.zoomLevels[currentZoom].features += data.layers[k] 133 | stats.layers[k] = (stats.layers[k] || 0) + data.layers[k] 134 | } 135 | }) 136 | .on('end', function () { 137 | if (!output) { return done() } 138 | outputStream.end() 139 | 140 | logNext() 141 | stats.zoomLevels[currentZoom].tilingStart = Date.now() 142 | currentState = 'tiling' 143 | 144 | tippecanoe(outputTiles, currentOptions.layer, outputGeojson, currentZoom) 145 | .on('exit', function (code) { 146 | if (code > 0) { 147 | return done(new Error('Tippecanoe exited nonzero: ' + code)) 148 | } 149 | logNext() 150 | 151 | zoomLevelFiles.push(outputTiles) 152 | if (--currentZoom < currentOptions.minzoom) { 153 | currentOptions = optionStack.shift() 154 | } 155 | if (currentOptions) { 156 | buildZoomLevel(tmpdir, outputTiles) 157 | } else { 158 | mergeZoomLevels() 159 | } 160 | }) 161 | }) 162 | } 163 | 164 | function mergeZoomLevels () { 165 | var args = [ '-o', output ].concat(zoomLevelFiles) 166 | debugLog('tile-join', args) 167 | spawn('tile-join', args, { stdio: 'inherit' }) 168 | .on('exit', function (code) { 169 | if (code) { 170 | return done(new Error('tile-join exited nonzero: ' + code)) 171 | } else { 172 | return done(null, stats) 173 | } 174 | }) 175 | } 176 | 177 | function logProgress (finished) { 178 | if (!currentOptions || currentOptions.quiet) { return } 179 | var currentStats = stats.zoomLevels[currentZoom] 180 | if (currentState === 'aggregating') { 181 | log('z' + currentZoom + ': aggregated ' + 182 | currentStats.features + ' features / ' + 183 | currentStats.tiles + ' tiles in ' + 184 | prettyMs(Date.now() - currentStats.start)) 185 | } else if (currentState === 'tiling' && !debugEnabled) { 186 | // if debug output is enabled, then let tippecanoe log its own stuff 187 | log('Writing tiles ' + prettyMs(Date.now() - currentStats.tilingStart)) 188 | } 189 | if (finished) { log.clear() } 190 | } 191 | 192 | function logNext () { 193 | if (currentOptions && !currentOptions.quiet) { 194 | logProgress() 195 | process.stderr.write('\n') 196 | } 197 | } 198 | } 199 | 200 | function tippecanoe (tiles, layerName, data, zoom) { 201 | var args = [ 202 | '-f', 203 | '-l', layerName, 204 | '-o', tiles, 205 | '-z', zoom, 206 | '-Z', zoom, 207 | '--read-parallel', 208 | '-b', 0, 209 | data 210 | ] 211 | if (!debugEnabled) { args.unshift('--quiet') } 212 | debugLog('tippecanoe ', args) 213 | return spawn('tippecanoe', args, { stdio: 'inherit' }) 214 | } 215 | 216 | function getInfo (input, cb) { 217 | var db = new MBTiles(input, function (err) { 218 | if (err) { return cb(err) } 219 | db.getInfo(cb) 220 | }) 221 | } 222 | 223 | -------------------------------------------------------------------------------- /lib/aggregate-cells.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var geojsonvt = require('geojson-vt') 4 | var filterDegenerate = require('./degenerate') 5 | var aggregate = require('geojson-polygon-aggregate').all 6 | var tilebelt = require('tilebelt') 7 | var tileUtil = require('./tile-util') 8 | var GeoJSONWrapper = require('./geojson-wrapper') 9 | 10 | module.exports = aggregateCells 11 | 12 | function aggregateCells (features, tile, gridZoom, aggregations, postAggregations) { 13 | var boxes 14 | tile = tileUtil.toZXY(tile) 15 | if (features && features[0] && features[0].properties._quadKey) { 16 | boxes = aggregateFromGrid(features, tile, gridZoom, aggregations, postAggregations) 17 | } else { 18 | boxes = aggregateFromRaw(features, tile, gridZoom, aggregations, postAggregations) 19 | } 20 | return boxes 21 | } 22 | 23 | // Aggregate *grid* input data--that is, features that are already themselves 24 | // grid squares from a higher zoom. Much faster than aggregating raw data, becase 25 | // we don't have to do any clipping. 26 | function aggregateFromGrid (inputFeatures, currentTile, gridZoom, aggregations, postAggregations) { 27 | var children = {} 28 | var numfeatures = inputFeatures.length 29 | for (var i = 0; i < numfeatures; i++) { 30 | var f = inputFeatures[i] 31 | var parentcell = tilebelt.getParent(tilebelt.quadkeyToTile(f.properties._quadKey)) 32 | var parentkey = tilebelt.tileToQuadkey(parentcell) 33 | if (!children[parentkey]) { 34 | children[parentkey] = [] 35 | } 36 | children[parentkey].push(f) 37 | } 38 | 39 | var cells = tileUtil.getProgeny(currentTile, gridZoom) 40 | var numcells = cells.length 41 | var boxes = [] 42 | for (var c = 0; c < numcells; c++) { 43 | var cell = cells[c] 44 | var cellkey = tilebelt.tileToQuadkey([cell[1], cell[2], cell[0]]) 45 | var features = children[cellkey] || [] 46 | boxes.push(makeCell(cell, features, aggregations, postAggregations, currentTile)) 47 | } 48 | 49 | return boxes 50 | } 51 | 52 | // Aggregate "raw" input data, using geojson-vt to slice it up into grid cells 53 | // first. 54 | function aggregateFromRaw (inputFeatures, currentTile, gridZoom, aggregations, postAggregations) { 55 | var tileIndex = geojsonvt({ 56 | type: 'FeatureCollection', 57 | features: inputFeatures 58 | }, { 59 | maxZoom: gridZoom, 60 | tolerance: 0, 61 | buffer: 0, 62 | indexMaxZoom: gridZoom 63 | }) 64 | 65 | var cells = tileUtil.getProgeny(currentTile, gridZoom) 66 | var numcells = cells.length 67 | var boxes = [] 68 | 69 | for (var i = 0; i < numcells; i++) { 70 | var t = tileIndex.getTile.apply(tileIndex, cells[i]) 71 | if (!t) { continue } 72 | var vt = new GeoJSONWrapper(t.features) 73 | var features = [] 74 | for (var j = 0; j < vt.length; j++) { 75 | var feat = vt.feature(j) 76 | .toGeoJSON(cells[i][1], cells[i][2], cells[i][0]) 77 | features.push(feat) 78 | } 79 | boxes.push(makeCell(cells[i], features, aggregations, postAggregations, currentTile)) 80 | } 81 | 82 | return boxes 83 | } 84 | 85 | function makeCell (cell, features, aggregations, postAggregations, tile) { 86 | // filter out features that are exactly on the tile boundary and not 87 | // properly within the tile 88 | features = features.filter(filterDegenerate(cell)) 89 | 90 | var box = { 91 | type: 'Feature', 92 | properties: aggregate(features, aggregations, null, [tile]), 93 | geometry: tilebelt.tileToGeoJSON(tileUtil.toXYZ(cell)) 94 | } 95 | 96 | if (postAggregations) { 97 | for (var field in postAggregations) { 98 | var fn = postAggregations[field] 99 | box.properties[field] = fn(box, tile) 100 | } 101 | } 102 | box.properties._quadKey = tilebelt.tileToQuadkey(tileUtil.toXYZ(cell)) 103 | return box 104 | } 105 | -------------------------------------------------------------------------------- /lib/aggregate.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var path = require('path') 4 | var reducers = require('geojson-polygon-aggregate/reducers') 5 | var aggregateCells = require('./aggregate-cells') 6 | 7 | var options 8 | setup(global.mapOptions) 9 | 10 | module.exports = aggregateTile 11 | // for testing/benchmarking use 12 | module.exports._setup = setup 13 | 14 | function setup (opts) { 15 | options = opts 16 | if (!opts) { return } 17 | // aggregation functions were passed in as names. look up the actual functions. 18 | if (typeof options.aggregations !== 'string') { 19 | for (var layer in options.aggregations) { 20 | for (var field in options.aggregations[layer]) { 21 | var fn = reducers[options.aggregations[layer][field]] 22 | options.aggregations[layer][field] = fn(field) 23 | } 24 | } 25 | } 26 | if (typeof options.aggregations === 'string') { 27 | var mod = path.resolve(process.cwd(), options.aggregations) 28 | options.aggregations = require(mod).aggregations 29 | } 30 | 31 | if (typeof options.postAggregations === 'string') { 32 | mod = path.resolve(process.cwd(), options.postAggregations) 33 | options.postAggregations = require(mod).postAggregations 34 | } else if (!options.postAggregations) { 35 | options.postAggregations = {} 36 | } 37 | 38 | options._depth = Math.log2(options.gridsize) / 2 - 1 39 | if (options._depth !== (options._depth | 0)) { 40 | throw new Error('Gridsize must be a power of 4') 41 | } 42 | 43 | return options 44 | } 45 | 46 | function aggregateTile (data, tile, writeData, done) { 47 | var counts = {} 48 | for (var layer in data.data) { 49 | counts[layer] = data.data[layer].features.length 50 | var gridFeatures = aggregateCells( 51 | data.data[layer].features, 52 | tile, 53 | tile[2] + options._depth, 54 | options.aggregations[layer], 55 | options.postAggregations[layer]) 56 | 57 | gridFeatures.forEach(function (feature) { 58 | feature.properties.layer = layer 59 | writeData(JSON.stringify(feature) + '\n') 60 | }) 61 | } 62 | done(null, { tile: tile, layers: counts }) 63 | } 64 | 65 | -------------------------------------------------------------------------------- /lib/degenerate.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var tilebelt = require('tilebelt') 4 | var tileUtil = require('./tile-util.js') 5 | 6 | /** 7 | * Return a filter that only passes features that have at least one point in 8 | * the strict interior of the given tile. 9 | */ 10 | module.exports = function (tile) { 11 | var bbox = tilebelt.tileToBBOX(tileUtil.toXYZ(tile)) 12 | // z0: 360 / 4096 = 0.087 degrees / 'pixel' ~ 2 decimal places 13 | // divide by an additional 4 for each zoom level 14 | var precision = 0.087 / Math.pow(4, tile[0]) 15 | 16 | function lte (left, right) { 17 | return left - right <= precision 18 | } 19 | 20 | return function filterDegenerate (feature) { 21 | var geom = feature.geometry 22 | var coords 23 | if (geom.type === 'Polygon') { 24 | coords = geom.coordinates[0] 25 | } else if (geom.type === 'LineString') { 26 | coords = geom.coordinates 27 | } else if (geom.type === 'Point') { 28 | coords = [geom.coordinates] 29 | } else if (geom.type === 'MultiLineString') { 30 | coords = [].concat.apply([], geom.coordinates) 31 | } else if (geom.type === 'MultiPolygon') { 32 | return geom.coordinates.every(function (rings) { 33 | return filterDegenerate({ geometry: { type: 'Polygon', coordinates: rings } }) 34 | }) 35 | } else { 36 | throw new Error('Unknown geometry type: ' + geom.type) 37 | } 38 | 39 | var left = !coords.every(function (point) { return lte(point[0], bbox[0]) }) 40 | var right = !coords.every(function (point) { return lte(bbox[2], point[0]) }) 41 | var top = !coords.every(function (point) { return lte(point[1], bbox[1]) }) 42 | var bottom = !coords.every(function (point) { return lte(bbox[3], point[1]) }) 43 | var okay = left && right && top && bottom 44 | 45 | return okay 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /lib/geojson-wrapper.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* 3 | * Copied directly from: 4 | * https://github.com/mapbox/mapbox-gl-js/blob/e523db31c1a5d3355a5d97d6bbada2cd64b6711a/js/source/geojson_wrapper.js 5 | * 6 | Copyright (c) 2014, Mapbox 7 | 8 | All rights reserved. 9 | 10 | Redistribution and use in source and binary forms, with or without modification, 11 | are permitted provided that the following conditions are met: 12 | 13 | * Redistributions of source code must retain the above copyright notice, 14 | this list of conditions and the following disclaimer. 15 | * Redistributions in binary form must reproduce the above copyright notice, 16 | this list of conditions and the following disclaimer in the documentation 17 | and/or other materials provided with the distribution. 18 | * Neither the name of Mapbox GL JS nor the names of its contributors 19 | may be used to endorse or promote products derived from this software 20 | without specific prior written permission. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 26 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 27 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 28 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 29 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 30 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 31 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | 35 | var Point = require('point-geometry'); 36 | var VectorTileFeature = require('vector-tile').VectorTileFeature; 37 | 38 | module.exports = GeoJSONWrapper; 39 | 40 | // conform to vectortile api 41 | function GeoJSONWrapper(features) { 42 | this.features = features; 43 | this.length = features.length; 44 | } 45 | 46 | GeoJSONWrapper.prototype.feature = function(i) { 47 | return new FeatureWrapper(this.features[i]); 48 | }; 49 | 50 | function FeatureWrapper(feature) { 51 | this.type = feature.type; 52 | this.rawGeometry = feature.type === 1 ? [feature.geometry] : feature.geometry; 53 | this.properties = feature.tags; 54 | this.extent = 4096; 55 | } 56 | 57 | FeatureWrapper.prototype.loadGeometry = function() { 58 | var rings = this.rawGeometry; 59 | this.geometry = []; 60 | 61 | for (var i = 0; i < rings.length; i++) { 62 | var ring = rings[i], 63 | newRing = []; 64 | for (var j = 0; j < ring.length; j++) { 65 | newRing.push(new Point(ring[j][0], ring[j][1])); 66 | } 67 | this.geometry.push(newRing); 68 | } 69 | return this.geometry; 70 | }; 71 | 72 | FeatureWrapper.prototype.bbox = function() { 73 | if (!this.geometry) this.loadGeometry(); 74 | 75 | var rings = this.geometry, 76 | x1 = Infinity, 77 | x2 = -Infinity, 78 | y1 = Infinity, 79 | y2 = -Infinity; 80 | 81 | for (var i = 0; i < rings.length; i++) { 82 | var ring = rings[i]; 83 | 84 | for (var j = 0; j < ring.length; j++) { 85 | var coord = ring[j]; 86 | 87 | x1 = Math.min(x1, coord.x); 88 | x2 = Math.max(x2, coord.x); 89 | y1 = Math.min(y1, coord.y); 90 | y2 = Math.max(y2, coord.y); 91 | } 92 | } 93 | 94 | return [x1, y1, x2, y2]; 95 | }; 96 | 97 | FeatureWrapper.prototype.toGeoJSON = VectorTileFeature.prototype.toGeoJSON; 98 | -------------------------------------------------------------------------------- /lib/tile-util.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var tilebelt = require('tilebelt') 4 | var uniq = require('uniq') 5 | 6 | module.exports = { 7 | getChildren: getChildren, 8 | getProgeny: getProgeny, 9 | getAncestors: getAncestors, 10 | hasProgeny: hasProgeny, 11 | toXYZ: toXYZ, 12 | toZXY: toZXY 13 | } 14 | 15 | function getAncestors (tiles, minzoom) { 16 | var ancestors = [] 17 | tiles = tiles.map(toXYZ) 18 | minzoom = minzoom || 0 19 | 20 | while (tiles.length > 0) { 21 | tiles = tiles 22 | .map(tilebelt.getParent.bind(tilebelt)) 23 | .filter(function (tile) { return tile[2] >= minzoom }) 24 | // remove duplicates 25 | tiles = uniq(tiles.map(join).sort()).map(split) 26 | if (tiles.length > 0) { ancestors.push(tiles.map(toZXY)) } 27 | } 28 | 29 | return ancestors 30 | 31 | function join (t) { return t.join('/') } 32 | function split (t) { return t.split('/').map(Number) } 33 | } 34 | 35 | function getChildren (tile) { 36 | return tilebelt.getChildren(toXYZ(tile)).map(toZXY) 37 | } 38 | 39 | function getProgeny (tile, zoom) { 40 | var z = tile[0] 41 | var tiles = [toXYZ(tile)] 42 | while (z < zoom) { 43 | var c = 0 44 | var nextTiles = new Array(tiles.length * 4) 45 | for (var i = 0; i < tiles.length; i++) { 46 | var children = tilebelt.getChildren(tiles[i]) 47 | for (var j = 0; j < 4; j++) { 48 | nextTiles[c++] = children[j] 49 | } 50 | } 51 | tiles = nextTiles 52 | z++ 53 | } 54 | return tiles.map(toZXY) 55 | } 56 | 57 | // assumes all ancestors are the same zoom level 58 | function hasProgeny (ancestors) { 59 | if (!ancestors[0] || !ancestors[0][0]) { 60 | return function () { return false } 61 | } 62 | var zoom = ancestors[0][0] 63 | var map = {} 64 | ancestors.forEach(function (a) { 65 | map[toXYZ(a).join('/')] = true 66 | }) 67 | 68 | return function (child) { 69 | child = toXYZ(child) 70 | while (child[2] > zoom) { 71 | child = tilebelt.getParent(child) 72 | } 73 | return map[child.join('/')] 74 | } 75 | } 76 | 77 | function toZXY (tile) { 78 | return [tile[2], tile[0], tile[1]] 79 | } 80 | 81 | function toXYZ (tile) { 82 | return [tile[1], tile[2], tile[0]] 83 | } 84 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vt-grid", 3 | "version": "4.1.1", 4 | "description": "Build up a pyramid of vector tiles by aggregating quantitative data into grids at lower zooms.", 5 | "bin": "bin/vt-grid", 6 | "scripts": { 7 | "test": "eslint . && tap --coverage test/*.js test/lib", 8 | "docs": "documentation readme -s API" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git+https://github.com/anandthakker/vt-grid.git" 13 | }, 14 | "author": "Anand Thakker (http://anandthakker.net/)", 15 | "license": "ISC", 16 | "dependencies": { 17 | "debug": "^2.2.0", 18 | "geojson-polygon-aggregate": "^2.0.0", 19 | "geojson-vt": "^2.1.7", 20 | "mbtiles": "^0.9.0", 21 | "point-geometry": "0.0.0", 22 | "pretty-ms": "^2.1.0", 23 | "simple-statistics": "^2.0.0-beta1", 24 | "single-line-log": "^1.1.1", 25 | "tile-reduce": "^3.1.1", 26 | "tilebelt": "^0.7.1", 27 | "tmp": "0.0.28", 28 | "uniq": "^1.0.1", 29 | "vector-tile": "^1.1.3", 30 | "yargs": "^3.15.0" 31 | }, 32 | "devDependencies": { 33 | "cheap-ruler": "^2.3.0", 34 | "documentation": "^4.0.0-beta2", 35 | "eslint": "^2.8.0", 36 | "eslint-config-standard": "^5.2.0", 37 | "eslint-plugin-promise": "^1.1.0", 38 | "eslint-plugin-standard": "^1.3.2", 39 | "flat": "^2.0.0", 40 | "fs-extra": "^0.24.0", 41 | "lodash.flatten": "^4.2.0", 42 | "oam-browser-filters": "^2.0.0", 43 | "split": "^1.0.0", 44 | "stream-statistics": "^0.4.0", 45 | "tap": "^2.0.0", 46 | "through2": "^2.0.1", 47 | "vt-geojson": "^2.0.0" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /test/fixture/aggregate-cells.input.geojson: -------------------------------------------------------------------------------- 1 | { "type": "FeatureCollection", "features": [ {"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[31.61865234375,1.1644706071806183],[31.624022126197815,1.1644706071806183],[31.624022126197815,1.161375975522148],[31.62312626838684,1.161375975522148],[31.62312626838684,1.1586621363047414],[31.622230410575867,1.1586621363047414],[31.622230410575867,1.1577557348799274],[31.62043333053589,1.1577557348799274],[31.62043333053589,1.1568493331653542],[31.61865234375,1.1568493331653542],[31.61865234375,1.1644706071806183]]]},"properties":{"densitypph":2}} 2 | , 3 | {"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[31.639299988746643,1.1644706071806183],[31.639299988746643,1.164095175444558],[31.63390874862671,1.164095175444558],[31.63390874862671,1.1644706071806183],[31.639299988746643,1.1644706071806183]]]},"properties":{"densitypph":0}} 4 | , 5 | {"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[31.61865234375,1.1644706071806183],[31.640625,1.1644706071806183],[31.640625,1.1568600598145622],[31.639299988746643,1.1568600598145622],[31.639299988746643,1.1577664615257106],[31.63840413093567,1.1577664615257106],[31.63840413093567,1.1586674996259063],[31.637502908706665,1.1586674996259063],[31.637502908706665,1.1595739007589856],[31.635705828666687,1.1595739007589856],[31.635705828666687,1.1586674996259063],[31.634809970855713,1.1586674996259063],[31.634809970855713,1.1568600598145622],[31.63390874862671,1.1568600598145622],[31.63391411304474,1.1550472555237974],[31.633012890815735,1.1550472555237974],[31.633012890815735,1.1559536578138392],[31.63211166858673,1.1559536578138392],[31.63211166858673,1.1568546964899582],[31.631215810775757,1.1568546964899582],[31.631215810775757,1.1550472555237974],[31.63211703300476,1.1550472555237974],[31.63211703300476,1.1541462162743557],[31.63391411304474,1.1541462162743557],[31.63391411304474,1.1532398134080353],[31.633012890815735,1.1532398134080353],[31.633012890815735,1.152333410253064],[31.63211703300476,1.152333410253064],[31.63211703300476,1.1514323701444908],[31.62941873073578,1.151427006809655],[31.62942409515381,1.1487131580911125],[31.628522872924805,1.1487131580911125],[31.628522872924805,1.1478121168396171],[31.62762701511383,1.1478121168396171],[31.62762701511383,1.1469057119608692],[31.625829935073853,1.1469057119608692],[31.625829935073853,1.1441918589516007],[31.62492871284485,1.1441918589516007],[31.62493407726288,1.1432854529276568],[31.624032855033875,1.1432854529276568],[31.624032855033875,1.1425024037061462],[31.61865234375,1.1425024037061462],[31.61865234375,1.1644706071806183]],[[31.622230410575867,1.1586621363047414],[31.62312626838684,1.1586621363047414],[31.62312626838684,1.161375975522148],[31.624022126197815,1.161375975522148],[31.624022126197815,1.1644706071806183],[31.61865234375,1.1644706071806183],[31.61865234375,1.1568493331653542],[31.62043333053589,1.1568493331653542],[31.62043333053589,1.1577557348799274],[31.622230410575867,1.1577557348799274],[31.622230410575867,1.1586621363047414]],[[31.63390874862671,1.1644706071806183],[31.63390874862671,1.164095175444558],[31.639299988746643,1.164095175444558],[31.639299988746643,1.1644706071806183],[31.63390874862671,1.1644706071806183]],[[31.639299988746643,1.164095175444558],[31.639299988746643,1.163194139074065],[31.640195846557617,1.163194139074065],[31.640195846557617,1.164095175444558],[31.639299988746643,1.164095175444558],[31.639299988746643,1.164095175444558]]]},"properties":{"densitypph":1}} 6 | , 7 | {"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[31.639299988746643,1.164095175444558],[31.640195846557617,1.164095175444558],[31.640195846557617,1.163194139074065],[31.639299988746643,1.163194139074065],[31.639299988746643,1.164095175444558],[31.639299988746643,1.164095175444558]]]},"properties":{"densitypph":0}} 8 | , 9 | {"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[31.61865234375,1.1644706071806183],[31.640625,1.1644706071806183],[31.640625,1.1425024037061462],[31.61865234375,1.1425024037061462],[31.61865234375,1.1644706071806183]],[[31.635705828666687,1.1595739007589856],[31.637502908706665,1.1595739007589856],[31.637502908706665,1.1586674996259063],[31.63840413093567,1.1586674996259063],[31.63840413093567,1.1577664615257106],[31.639299988746643,1.1577664615257106],[31.639299988746643,1.1568600598145622],[31.640625,1.1568600598145622],[31.640625,1.1644706071806183],[31.61865234375,1.1644706071806183],[31.61865234375,1.1425024037061462],[31.624032855033875,1.1425024037061462],[31.624032855033875,1.1432854529276568],[31.62493407726288,1.1432854529276568],[31.62492871284485,1.1441918589516007],[31.625829935073853,1.1441918589516007],[31.625829935073853,1.1469057119608692],[31.62762701511383,1.1469057119608692],[31.62762701511383,1.1478121168396171],[31.628522872924805,1.1478121168396171],[31.628522872924805,1.1487131580911125],[31.62942409515381,1.1487131580911125],[31.62941873073578,1.151427006809655],[31.63211703300476,1.1514323701444908],[31.63211703300476,1.152333410253064],[31.633012890815735,1.152333410253064],[31.633012890815735,1.1532398134080353],[31.63391411304474,1.1532398134080353],[31.63391411304474,1.1541462162743557],[31.63211703300476,1.1541462162743557],[31.63211703300476,1.1550472555237974],[31.631215810775757,1.1550472555237974],[31.631215810775757,1.1568546964899582],[31.63211166858673,1.1568546964899582],[31.63211166858673,1.1559536578138392],[31.633012890815735,1.1559536578138392],[31.633012890815735,1.1550472555237974],[31.63391411304474,1.1550472555237974],[31.63390874862671,1.1568600598145622],[31.634809970855713,1.1568600598145622],[31.634809970855713,1.1586674996259063],[31.635705828666687,1.1586674996259063],[31.635705828666687,1.1595739007589856],[31.635705828666687,1.1595739007589856]]]},"properties":{"densitypph":0}}] } -------------------------------------------------------------------------------- /test/fixture/dc.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": { 7 | "data": 1 8 | }, 9 | "geometry": { 10 | "type": "Polygon", 11 | "coordinates": [ 12 | [ 13 | [ 14 | -77.12127685546875, 15 | 38.9348437659246 16 | ], 17 | [ 18 | -77.0416259765625, 19 | 38.99730766542481 20 | ], 21 | [ 22 | -76.90910339355469, 23 | 38.892636142310295 24 | ], 25 | [ 26 | -77.04093933105469, 27 | 38.79048618862274 28 | ], 29 | [ 30 | -77.03887939453125, 31 | 38.87018642754998 32 | ], 33 | [ 34 | -77.12127685546875, 35 | 38.9348437659246 36 | ] 37 | ] 38 | ] 39 | } 40 | }, 41 | { 42 | "type": "Feature", 43 | "properties": { 44 | "data": 0 45 | }, 46 | "geometry": { 47 | "type": "Polygon", 48 | "coordinates": [ 49 | [ 50 | [ 51 | -77.30392456054688, 52 | 38.73212548425921 53 | ], 54 | [ 55 | -77.30392456054688, 56 | 39.08477116318522 57 | ], 58 | [ 59 | -76.75392150878906, 60 | 39.08477116318522 61 | ], 62 | [ 63 | -76.75392150878906, 64 | 38.73212548425921 65 | ], 66 | [ 67 | -77.30392456054688, 68 | 38.73212548425921 69 | ] 70 | ] 71 | ] 72 | } 73 | } 74 | ] 75 | } -------------------------------------------------------------------------------- /test/fixture/dc.mbtiles: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/developmentseed/vt-grid/02b53bdf3f3bb502134005242c3d30656adf2250/test/fixture/dc.mbtiles -------------------------------------------------------------------------------- /test/fixture/dc.z12-grid-quadkeys.txt: -------------------------------------------------------------------------------- 1 | 032010032213033 2 | 032010032213210 3 | 032010032213211 4 | 032010032213300 5 | 032010032213203 6 | 032010032213212 7 | 032010032213213 8 | 032010032213302 9 | 032010032213303 10 | 032010032213220 11 | 032010032213221 12 | 032010032213230 13 | 032010032213231 14 | 032010032213320 15 | 032010032213321 16 | 032010032213330 17 | 032010032212333 18 | 032010032213222 19 | 032010032213223 20 | 032010032213232 21 | 032010032213233 22 | 032010032213322 23 | 032010032213323 24 | 032010032213332 25 | 032010032213333 26 | 032010032230110 27 | 032010032230111 28 | 032010032231000 29 | 032010032231001 30 | 032010032231010 31 | 032010032231011 32 | 032010032231100 33 | 032010032231101 34 | 032010032231110 35 | 032010032231111 36 | 032010032320000 37 | 032010032230103 38 | 032010032230112 39 | 032010032230113 40 | 032010032231002 41 | 032010032231003 42 | 032010032231012 43 | 032010032231013 44 | 032010032231102 45 | 032010032231103 46 | 032010032231112 47 | 032010032231113 48 | 032010032320002 49 | 032010032320003 50 | 032010032230120 51 | 032010032230121 52 | 032010032230130 53 | 032010032230131 54 | 032010032231020 55 | 032010032231021 56 | 032010032231030 57 | 032010032231031 58 | 032010032231120 59 | 032010032231121 60 | 032010032231130 61 | 032010032231131 62 | 032010032320020 63 | 032010032320021 64 | 032010032320030 65 | 032010032230123 66 | 032010032230132 67 | 032010032230133 68 | 032010032231022 69 | 032010032231023 70 | 032010032231032 71 | 032010032231033 72 | 032010032231122 73 | 032010032231123 74 | 032010032231132 75 | 032010032231133 76 | 032010032320022 77 | 032010032320023 78 | 032010032320032 79 | 032010032320033 80 | 032010032230310 81 | 032010032230311 82 | 032010032231200 83 | 032010032231201 84 | 032010032231210 85 | 032010032231211 86 | 032010032231300 87 | 032010032231301 88 | 032010032231310 89 | 032010032231311 90 | 032010032320200 91 | 032010032320201 92 | 032010032320210 93 | 032010032320211 94 | 032010032320300 95 | 032010032230313 96 | 032010032231202 97 | 032010032231203 98 | 032010032231212 99 | 032010032231213 100 | 032010032231302 101 | 032010032231303 102 | 032010032231312 103 | 032010032231313 104 | 032010032320202 105 | 032010032320203 106 | 032010032320212 107 | 032010032320213 108 | 032010032320302 109 | 032010032320303 110 | 032010032231220 111 | 032010032231221 112 | 032010032231230 113 | 032010032231231 114 | 032010032231320 115 | 032010032231321 116 | 032010032231330 117 | 032010032231331 118 | 032010032320220 119 | 032010032320221 120 | 032010032320230 121 | 032010032320231 122 | 032010032320320 123 | 032010032320321 124 | 032010032320330 125 | 032010032231222 126 | 032010032231223 127 | 032010032231232 128 | 032010032231233 129 | 032010032231322 130 | 032010032231323 131 | 032010032231332 132 | 032010032231333 133 | 032010032320222 134 | 032010032320223 135 | 032010032320232 136 | 032010032320233 137 | 032010032320322 138 | 032010032320323 139 | 032010032320332 140 | 032010032320333 141 | 032010032233001 142 | 032010032233010 143 | 032010032233011 144 | 032010032233100 145 | 032010032233101 146 | 032010032233110 147 | 032010032233111 148 | 032010032322000 149 | 032010032322001 150 | 032010032322010 151 | 032010032322011 152 | 032010032322100 153 | 032010032322101 154 | 032010032322110 155 | 032010032233012 156 | 032010032233013 157 | 032010032233102 158 | 032010032233103 159 | 032010032233112 160 | 032010032233113 161 | 032010032322002 162 | 032010032322003 163 | 032010032322012 164 | 032010032322013 165 | 032010032322102 166 | 032010032322103 167 | 032010032233031 168 | 032010032233120 169 | 032010032233121 170 | 032010032233130 171 | 032010032233131 172 | 032010032322020 173 | 032010032322021 174 | 032010032322030 175 | 032010032322031 176 | 032010032322120 177 | 032010032233033 178 | 032010032233122 179 | 032010032233123 180 | 032010032233132 181 | 032010032233133 182 | 032010032322022 183 | 032010032322023 184 | 032010032322032 185 | 032010032322033 186 | 032010032233211 187 | 032010032233300 188 | 032010032233301 189 | 032010032233310 190 | 032010032233311 191 | 032010032322200 192 | 032010032322201 193 | 032010032322210 194 | 032010032233213 195 | 032010032233302 196 | 032010032233303 197 | 032010032233312 198 | 032010032233313 199 | 032010032322202 200 | 032010032322203 201 | 032010032233231 202 | 032010032233320 203 | 032010032233321 204 | 032010032233330 205 | 032010032233331 206 | 032010032322220 207 | 032010032233233 208 | 032010032233322 209 | 032010032233323 210 | 032010032233332 211 | 032010032233333 212 | 032010210011011 213 | 032010210011100 214 | 032010210011101 215 | 032010210011110 216 | 032010210011013 217 | 032010210011102 218 | 032010210011103 219 | 032010210011031 220 | 032010210011120 221 | 032010210011033 222 | 032010032322111 223 | 032010032322112 224 | 032010032322121 225 | 032010032322122 226 | 032010032322211 227 | 032010032322212 228 | 032010032322221 229 | 032010032322222 230 | 032010210011111 231 | 032010210011112 232 | 032010210011121 233 | 032010210011122 234 | 032010032230331 235 | 032010032230312 236 | 032010032230301 237 | 032010032230122 238 | -------------------------------------------------------------------------------- /test/fixture/degenerate-features-2.geojson: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type": "Feature", 4 | "properties": { 5 | "data": 1, 6 | "_quadKey": "0320100322132211" 7 | }, 8 | "geometry": { 9 | "type": "Polygon", 10 | "coordinates": [ 11 | [ 12 | [ 13 | -77.0635986328125, 14 | 38.97222194853654 15 | ], 16 | [ 17 | -77.0635986328125, 18 | 38.97649248553941 19 | ], 20 | [ 21 | -77.05810546875, 22 | 38.97649248553941 23 | ], 24 | [ 25 | -77.05810546875, 26 | 38.97222194853654 27 | ], 28 | [ 29 | -77.0635986328125, 30 | 38.97222194853654 31 | ] 32 | ] 33 | ] 34 | } 35 | }, 36 | { 37 | "type": "Feature", 38 | "properties": { 39 | "data": 1, 40 | "_quadKey": "0320100322132310" 41 | }, 42 | "geometry": { 43 | "type": "Polygon", 44 | "coordinates": [ 45 | [ 46 | [ 47 | -77.047119140625, 48 | 38.97222194853654 49 | ], 50 | [ 51 | -77.047119140625, 52 | 38.97649248553941 53 | ], 54 | [ 55 | -77.0416259765625, 56 | 38.97649248553941 57 | ], 58 | [ 59 | -77.0416259765625, 60 | 38.97222194853654 61 | ], 62 | [ 63 | -77.047119140625, 64 | 38.97222194853654 65 | ] 66 | ] 67 | ] 68 | } 69 | }, 70 | { 71 | "type": "Feature", 72 | "properties": { 73 | "data": 1, 74 | "_quadKey": "0320100322132301" 75 | }, 76 | "geometry": { 77 | "type": "Polygon", 78 | "coordinates": [ 79 | [ 80 | [ 81 | -77.0526123046875, 82 | 38.97222194853654 83 | ], 84 | [ 85 | -77.0526123046875, 86 | 38.97649248553941 87 | ], 88 | [ 89 | -77.047119140625, 90 | 38.97649248553941 91 | ], 92 | [ 93 | -77.047119140625, 94 | 38.97222194853654 95 | ], 96 | [ 97 | -77.0526123046875, 98 | 38.97222194853654 99 | ] 100 | ] 101 | ] 102 | } 103 | }, 104 | { 105 | "type": "Feature", 106 | "properties": { 107 | "data": 1, 108 | "_quadKey": "0320100322132300" 109 | }, 110 | "geometry": { 111 | "type": "Polygon", 112 | "coordinates": [ 113 | [ 114 | [ 115 | -77.05810546875, 116 | 38.97222194853654 117 | ], 118 | [ 119 | -77.05810546875, 120 | 38.97649248553941 121 | ], 122 | [ 123 | -77.0526123046875, 124 | 38.97649248553941 125 | ], 126 | [ 127 | -77.0526123046875, 128 | 38.97222194853654 129 | ], 130 | [ 131 | -77.05810546875, 132 | 38.97222194853654 133 | ] 134 | ] 135 | ] 136 | } 137 | }, 138 | { 139 | "type": "Feature", 140 | "properties": { 141 | "data": 1, 142 | "_quadKey": "0320100322132122" 143 | }, 144 | "geometry": { 145 | "type": "Polygon", 146 | "coordinates": [ 147 | [ 148 | [ 149 | -77.05810546875, 150 | 38.97649248553941 151 | ], 152 | [ 153 | -77.05810546875, 154 | 38.98076276501632 155 | ], 156 | [ 157 | -77.0526123046875, 158 | 38.98076276501632 159 | ], 160 | [ 161 | -77.0526123046875, 162 | 38.97649248553941 163 | ], 164 | [ 165 | -77.05810546875, 166 | 38.97649248553941 167 | ] 168 | ] 169 | ] 170 | } 171 | }, 172 | { 173 | "type": "Feature", 174 | "properties": { 175 | "data": 1, 176 | "_quadKey": "0320100322132123" 177 | }, 178 | "geometry": { 179 | "type": "Polygon", 180 | "coordinates": [ 181 | [ 182 | [ 183 | -77.0526123046875, 184 | 38.97649248553941 185 | ], 186 | [ 187 | -77.0526123046875, 188 | 38.98076276501632 189 | ], 190 | [ 191 | -77.047119140625, 192 | 38.98076276501632 193 | ], 194 | [ 195 | -77.047119140625, 196 | 38.97649248553941 197 | ], 198 | [ 199 | -77.0526123046875, 200 | 38.97649248553941 201 | ] 202 | ] 203 | ] 204 | } 205 | }, 206 | { 207 | "type": "Feature", 208 | "properties": { 209 | "data": 1, 210 | "_quadKey": "0320100322132121" 211 | }, 212 | "geometry": { 213 | "type": "Polygon", 214 | "coordinates": [ 215 | [ 216 | [ 217 | -77.0526123046875, 218 | 38.98076276501632 219 | ], 220 | [ 221 | -77.0526123046875, 222 | 38.985032786959096 223 | ], 224 | [ 225 | -77.047119140625, 226 | 38.985032786959096 227 | ], 228 | [ 229 | -77.047119140625, 230 | 38.98076276501632 231 | ], 232 | [ 233 | -77.0526123046875, 234 | 38.98076276501632 235 | ] 236 | ] 237 | ] 238 | } 239 | }, 240 | { 241 | "type": "Feature", 242 | "properties": { 243 | "data": 1, 244 | "_quadKey": "0320100322132120" 245 | }, 246 | "geometry": { 247 | "type": "Polygon", 248 | "coordinates": [ 249 | [ 250 | [ 251 | -77.05810546875, 252 | 38.98076276501632 253 | ], 254 | [ 255 | -77.05810546875, 256 | 38.985032786959096 257 | ], 258 | [ 259 | -77.0526123046875, 260 | 38.985032786959096 261 | ], 262 | [ 263 | -77.0526123046875, 264 | 38.98076276501632 265 | ], 266 | [ 267 | -77.05810546875, 268 | 38.98076276501632 269 | ] 270 | ] 271 | ] 272 | } 273 | }, 274 | { 275 | "type": "Feature", 276 | "properties": { 277 | "data": 1, 278 | "_quadKey": "0320100322132132" 279 | }, 280 | "geometry": { 281 | "type": "Polygon", 282 | "coordinates": [ 283 | [ 284 | [ 285 | -77.047119140625, 286 | 38.97649248553941 287 | ], 288 | [ 289 | -77.047119140625, 290 | 38.98076276501632 291 | ], 292 | [ 293 | -77.0416259765625, 294 | 38.98076276501632 295 | ], 296 | [ 297 | -77.0416259765625, 298 | 38.97649248553941 299 | ], 300 | [ 301 | -77.047119140625, 302 | 38.97649248553941 303 | ] 304 | ] 305 | ] 306 | } 307 | }, 308 | { 309 | "type": "Feature", 310 | "properties": { 311 | "data": 1, 312 | "_quadKey": "0320100322132130" 313 | }, 314 | "geometry": { 315 | "type": "Polygon", 316 | "coordinates": [ 317 | [ 318 | [ 319 | -77.047119140625, 320 | 38.98076276501632 321 | ], 322 | [ 323 | -77.047119140625, 324 | 38.985032786959096 325 | ], 326 | [ 327 | -77.0416259765625, 328 | 38.985032786959096 329 | ], 330 | [ 331 | -77.0416259765625, 332 | 38.98076276501632 333 | ], 334 | [ 335 | -77.047119140625, 336 | 38.98076276501632 337 | ] 338 | ] 339 | ] 340 | } 341 | }, 342 | { 343 | "type": "Feature", 344 | "properties": { 345 | "data": 1, 346 | "_quadKey": "0320100322132112" 347 | }, 348 | "geometry": { 349 | "type": "Polygon", 350 | "coordinates": [ 351 | [ 352 | [ 353 | -77.047119140625, 354 | 38.985032786959096 355 | ], 356 | [ 357 | -77.047119140625, 358 | 38.989302551359515 359 | ], 360 | [ 361 | -77.0416259765625, 362 | 38.989302551359515 363 | ], 364 | [ 365 | -77.0416259765625, 366 | 38.985032786959096 367 | ], 368 | [ 369 | -77.047119140625, 370 | 38.985032786959096 371 | ] 372 | ] 373 | ] 374 | } 375 | }, 376 | { 377 | "type": "Feature", 378 | "properties": { 379 | "data": 1, 380 | "_quadKey": "0320100322132102" 381 | }, 382 | "geometry": { 383 | "type": "Polygon", 384 | "coordinates": [ 385 | [ 386 | [ 387 | -77.05810546875, 388 | 38.985032786959096 389 | ], 390 | [ 391 | -77.05810546875, 392 | 38.989302551359515 393 | ], 394 | [ 395 | -77.0526123046875, 396 | 38.989302551359515 397 | ], 398 | [ 399 | -77.0526123046875, 400 | 38.985032786959096 401 | ], 402 | [ 403 | -77.05810546875, 404 | 38.985032786959096 405 | ] 406 | ] 407 | ] 408 | } 409 | }, 410 | { 411 | "type": "Feature", 412 | "properties": { 413 | "data": 1, 414 | "_quadKey": "0320100322132103" 415 | }, 416 | "geometry": { 417 | "type": "Polygon", 418 | "coordinates": [ 419 | [ 420 | [ 421 | -77.0526123046875, 422 | 38.985032786959096 423 | ], 424 | [ 425 | -77.0526123046875, 426 | 38.989302551359515 427 | ], 428 | [ 429 | -77.047119140625, 430 | 38.989302551359515 431 | ], 432 | [ 433 | -77.047119140625, 434 | 38.985032786959096 435 | ], 436 | [ 437 | -77.0526123046875, 438 | 38.985032786959096 439 | ] 440 | ] 441 | ] 442 | } 443 | }, 444 | { 445 | "type": "Feature", 446 | "properties": { 447 | "data": 1, 448 | "_quadKey": "0320100322132033" 449 | }, 450 | "geometry": { 451 | "type": "Polygon", 452 | "coordinates": [ 453 | [ 454 | [ 455 | -77.0635986328125, 456 | 38.97649248553941 457 | ], 458 | [ 459 | -77.0635986328125, 460 | 38.98076276501632 461 | ], 462 | [ 463 | -77.05810546875, 464 | 38.98076276501632 465 | ], 466 | [ 467 | -77.05810546875, 468 | 38.97649248553941 469 | ], 470 | [ 471 | -77.0635986328125, 472 | 38.97649248553941 473 | ] 474 | ] 475 | ] 476 | } 477 | }, 478 | { 479 | "type": "Feature", 480 | "properties": { 481 | "data": 1, 482 | "_quadKey": "0320100322132031" 483 | }, 484 | "geometry": { 485 | "type": "Polygon", 486 | "coordinates": [ 487 | [ 488 | [ 489 | -77.0635986328125, 490 | 38.98076276501632 491 | ], 492 | [ 493 | -77.0635986328125, 494 | 38.985032786959096 495 | ], 496 | [ 497 | -77.05810546875, 498 | 38.985032786959096 499 | ], 500 | [ 501 | -77.05810546875, 502 | 38.98076276501632 503 | ], 504 | [ 505 | -77.0635986328125, 506 | 38.98076276501632 507 | ] 508 | ] 509 | ] 510 | } 511 | }, 512 | { 513 | "type": "Feature", 514 | "properties": { 515 | "data": 0, 516 | "_quadKey": "0320100322132013" 517 | }, 518 | "geometry": { 519 | "type": "Polygon", 520 | "coordinates": [ 521 | [ 522 | [ 523 | -77.0635986328125, 524 | 38.985032786959096 525 | ], 526 | [ 527 | -77.0635986328125, 528 | 38.989302551359515 529 | ], 530 | [ 531 | -77.05810546875, 532 | 38.989302551359515 533 | ], 534 | [ 535 | -77.05810546875, 536 | 38.985032786959096 537 | ], 538 | [ 539 | -77.0635986328125, 540 | 38.985032786959096 541 | ] 542 | ] 543 | ] 544 | } 545 | } 546 | ] 547 | -------------------------------------------------------------------------------- /test/fixture/degenerate-features.geojson: -------------------------------------------------------------------------------- 1 | [{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-76.805419921875,39.06184913429155],[-76.805419921875,39.06184913429155],[-76.805419921875,39.05758374935667],[-76.805419921875,39.05758374935667],[-76.805419921875,39.06184913429155]]]},"properties":{"data":0,"_quadKey":"0320100323102001"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-76.805419921875,39.05758374935667],[-76.805419921875,39.05758374935667],[-76.805419921875,39.0533181067413],[-76.805419921875,39.0533181067413],[-76.805419921875,39.05758374935667]]]},"properties":{"data":0,"_quadKey":"0320100323102003"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-76.805419921875,39.05758374935667],[-76.805419921875,39.06184913429155],[-76.7999267578125,39.06184913429155],[-76.7999267578125,39.05758374935667],[-76.805419921875,39.05758374935667]]]},"properties":{"data":0,"_quadKey":"0320100323102010"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-76.7999267578125,39.05758374935667],[-76.7999267578125,39.06184913429155],[-76.79443359375,39.06184913429155],[-76.79443359375,39.05758374935667],[-76.7999267578125,39.05758374935667]]]},"properties":{"data":0,"_quadKey":"0320100323102011"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-76.7999267578125,39.0533181067413],[-76.7999267578125,39.05758374935667],[-76.79443359375,39.05758374935667],[-76.79443359375,39.0533181067413],[-76.7999267578125,39.0533181067413]]]},"properties":{"data":0,"_quadKey":"0320100323102013"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-76.805419921875,39.0533181067413],[-76.805419921875,39.05758374935667],[-76.7999267578125,39.05758374935667],[-76.7999267578125,39.0533181067413],[-76.805419921875,39.0533181067413]]]},"properties":{"data":0,"_quadKey":"0320100323102012"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-76.805419921875,39.0533181067413],[-76.805419921875,39.0533181067413],[-76.7999267578125,39.0533181067413],[-76.7999267578125,39.0533181067413],[-76.805419921875,39.0533181067413]]]},"properties":{"data":0,"_quadKey":"0320100323102030"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-76.7999267578125,39.0533181067413],[-76.7999267578125,39.0533181067413],[-76.79443359375,39.0533181067413],[-76.79443359375,39.0533181067413],[-76.7999267578125,39.0533181067413]]]},"properties":{"data":0,"_quadKey":"0320100323102031"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-76.805419921875,39.0533181067413],[-76.805419921875,39.0533181067413],[-76.805419921875,39.0533181067413],[-76.805419921875,39.0533181067413],[-76.805419921875,39.0533181067413]]]},"properties":{"data":0,"_quadKey":"0320100323102021"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-76.79443359375,39.05758374935667],[-76.79443359375,39.06184913429155],[-76.79443359375,39.06184913429155],[-76.79443359375,39.05758374935667],[-76.79443359375,39.05758374935667]]]},"properties":{"data":0,"_quadKey":"0320100323102100"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-76.79443359375,39.0533181067413],[-76.79443359375,39.05758374935667],[-76.79443359375,39.05758374935667],[-76.79443359375,39.0533181067413],[-76.79443359375,39.0533181067413]]]},"properties":{"data":0,"_quadKey":"0320100323102102"}},{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[-76.79443359375,39.0533181067413],[-76.79443359375,39.0533181067413],[-76.79443359375,39.0533181067413],[-76.79443359375,39.0533181067413]]]},"properties":{"data":0,"_quadKey":"0320100323102120"}}] -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs-extra') 2 | var path = require('path') 3 | var test = require('tap').test 4 | var tmp = require('tmp') 5 | var vtgeojson = require('vt-geojson') 6 | var MBTiles = require('mbtiles') 7 | var vtgrid = require('../') 8 | var tilebelt = require('tilebelt') 9 | 10 | test('main module', function (t) { 11 | tmp.file({postfix: '.mbtiles'}, function (err, output) { 12 | t.error(err) 13 | vtgrid(path.resolve(output), path.resolve(__dirname, 'fixture', 'dc.mbtiles'), { 14 | minzoom: 12, 15 | gridsize: 64, 16 | jobs: 1, 17 | aggregations: { 18 | 'dc': { 19 | 'data': 'sum' 20 | } 21 | } 22 | }, function (err) { 23 | t.error(err) 24 | 25 | var expected = fs.readFileSync(path.join(__dirname, '/fixture/dc.z12-grid-quadkeys.txt'), 'utf-8') 26 | .split('\n') 27 | .filter(Boolean) 28 | 29 | var results = {} 30 | vtgeojson('mbtiles://' + output, { 31 | minzoom: 12, 32 | maxzoom: 12, 33 | bounds: JSON.parse(fs.readFileSync(path.join(__dirname, '/fixture/dc.geojson'))) 34 | }) 35 | .on('data', function (feature) { 36 | results[feature.properties._quadKey] = feature.properties.data 37 | }) 38 | .on('end', function () { 39 | expected.forEach(function (key) { 40 | var tile = tilebelt.quadkeyToTile(key) 41 | t.ok(results[key] > 0, results[key] + ' > 0 for tile ' + tile) 42 | delete results[key] 43 | }) 44 | 45 | for (var key in results) { 46 | t.ok(results[key] === 0 || !results[key], 'no value for tile ' + key) 47 | } 48 | 49 | getInfo(output, function (err, info) { 50 | t.error(err) 51 | t.ok(info.vector_layers && info.vector_layers[0], 'vector_layers') 52 | t.same(info.vector_layers[0].id, 'dc', 'layer id') 53 | t.ok(info.vector_layers[0].fields['data'], '"data" field') 54 | t.same(info.minzoom, 12) 55 | t.same(info.maxzoom, 13) 56 | t.end() 57 | }) 58 | }) 59 | }) 60 | }) 61 | }) 62 | 63 | function getInfo (file, callback) { 64 | var mbtiles = new MBTiles('mbtiles://' + file, function (err) { 65 | if (err) return callback(err) 66 | mbtiles.getInfo(callback) 67 | }) 68 | } 69 | -------------------------------------------------------------------------------- /test/lib/aggregate-cells.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var path = require('path') 3 | var test = require('tap').test 4 | var reducers = require('geojson-polygon-aggregate/reducers') 5 | var tilebelt = require('tilebelt') 6 | 7 | var aggregateCells = require('../../lib/aggregate-cells') 8 | 9 | test('aggregate cells: raw features', function (t) { 10 | var input = fs.readFileSync(path.join(__dirname, '../fixture/aggregate-cells.input.geojson')) 11 | var aggs = { 12 | 'densitypph': reducers.areaWeightedMean('densitypph'), 13 | 'tile': function (memo, feature, _, tile) { return memo || tile.join(',') } 14 | } 15 | var postAggs = {} 16 | var currentTile = [ 9631, 8139, 14 ] 17 | var gridZoom = 14 + 5 // 4 ^ 5 = 1024 18 | var result = aggregateCells(JSON.parse(input).features, currentTile, gridZoom, aggs, postAggs) 19 | t.equal(result.length, 1024) 20 | var valid = result.filter(function (feat) { 21 | return feat.properties.densitypph <= 2 && feat.properties.densitypph >= 0 22 | }) 23 | t.same(result, valid) 24 | t.same(result[0].properties.tile, '14,9631,8139', 'pass tile coordinates to reducer') 25 | t.end() 26 | }) 27 | 28 | test('aggregate cells: grid features', function (t) { 29 | var raw = fs.readFileSync(path.join(__dirname, '../fixture/aggregate-cells.input.geojson')) 30 | var aggs = { 'densitypph': reducers.areaWeightedMean('densitypph') } 31 | var postAggs = {} 32 | var currentTile = [ 9631, 8139, 14 ] 33 | var gridZoom = 14 + 5 // 4 ^ 5 = 1024 34 | // aggregate the raw features into a grid 35 | var grid = aggregateCells(JSON.parse(raw).features, currentTile, gridZoom, aggs, postAggs) 36 | // now do it again, with gridzoom being one less, so that we can treat the 37 | // grid features we just made the 'incoming' features to be aggregated on the 38 | // same tile 39 | aggs = { 40 | 'densitypph': reducers.sum('densitypph'), 41 | 'tile': function (memo, feature, _, tile) { return memo || tile.join(',') } 42 | } 43 | var result = aggregateCells(grid, [ 9631, 8139, 14 ], gridZoom - 1, aggs, postAggs) 44 | result.forEach(function (feat) { 45 | var parentkey = feat.properties._quadKey 46 | var gridsum = grid.filter(function (child) { 47 | var parent = tilebelt.getParent(tilebelt.quadkeyToTile(child.properties._quadKey)) 48 | return parentkey === tilebelt.tileToQuadkey(parent) 49 | }) 50 | .map(function (f) { return f.properties.densitypph || 0 }) 51 | .reduce(function (a, b) { return a + b }, 0) 52 | t.equal(round(feat.properties.densitypph), round(gridsum), parentkey) 53 | }) 54 | t.same(result[0].properties.tile, '14,9631,8139', 'pass tile coordinates to reducer') 55 | t.end() 56 | }) 57 | 58 | function round (x) { 59 | return Math.round(x * 1e6) / 1e6 60 | } 61 | 62 | -------------------------------------------------------------------------------- /test/lib/degenerate.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var path = require('path') 3 | var test = require('tap').test 4 | 5 | var degenerate = require('../../lib/degenerate') 6 | 7 | test('degenerate filter', function (t) { 8 | var pre = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../fixture/degenerate-features.geojson'))) 9 | var post = pre.filter(degenerate([ 15, 9393, 12516 ])) 10 | t.equal(post.length, 4) 11 | t.end() 12 | }) 13 | 14 | test('degenerate filter 2', function (t) { 15 | var pre = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../fixture/degenerate-features-2.geojson'))) 16 | var post = pre.filter(degenerate([ 15, 9370, 12525 ])) 17 | t.equal(post.length, 4) 18 | t.end() 19 | }) 20 | --------------------------------------------------------------------------------