├── .gitignore ├── LICENSE ├── README.md ├── demo ├── README.md ├── demo_config.sh ├── index.html ├── package.json ├── points.html └── pointserver.js ├── lib └── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # Compiled binary addons (http://nodejs.org/api/addons.html) 20 | build/Release 21 | 22 | # Dependency directory 23 | # Deployed apps should consider commenting this line out: 24 | # see https://npmjs.org/doc/faq.html#Should-I-check-my-node_modules-folder-into-git 25 | node_modules 26 | 27 | *.DS_Store 28 | *.zip 29 | *.vrt 30 | *.csv 31 | *.txt -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Faraday 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | TILESPLASH 2 | ========== 3 | 4 | A light and quick nodejs webserver for serving topojson and mapbox vector tiles from a [postgis](http://www.postgis.net/) backend. inspired by [Michal Migurski](http://mike.teczno.com/)'s [TileStache](http://tilestache.org/). Works great for powering [Mapbox-GL-based](https://www.mapbox.com/mapbox-gl-js/example/set-perspective/) apps like this: 5 | 6 | ![example](https://www.dropbox.com/s/viwv9layui7vw7x/Screenshot%202016-10-14%2011.20.03.png?dl=1) 7 | 8 | # Dependencies 9 | 10 | Tilesplash depends on `node` and `npm` 11 | 12 | # Installation 13 | 14 | ```bash 15 | npm install tilesplash 16 | ``` 17 | 18 | # Example 19 | 20 | Here's a simple tile server with one layer 21 | 22 | ```javascript 23 | var Tilesplash = require('tilesplash'); 24 | 25 | // invoke tilesplash with DB options 26 | var app = new Tilesplash({ 27 | user: myUser, 28 | password: myPassword, 29 | host: localhost, 30 | port: 5432, 31 | database: myDb 32 | }); 33 | 34 | // define a layer 35 | app.layer('test_layer', function(tile, render){ 36 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM layer WHERE ST_Intersects(the_geom, !bbox_4326!)'); 37 | }); 38 | 39 | // serve tiles at port 3000 40 | app.server.listen(3000); 41 | ``` 42 | 43 | - Topojson tiles will be available at `http://localhost:3000/test_layer/{z}/{x}/{y}.topojson` 44 | - Mapbox vector tiles will be available at `http://localhost:3000/test_layer/{z}/{x}/{y}.mvt` 45 | 46 | (See [client implementation examples](https://github.com/faradayio/tilesplash#client) below, and complete demo implementation in [`demo/`](demo/README.md)) 47 | 48 | # Usage 49 | 50 | ## `new Tilesplash(connection_details, [cacheType])` 51 | 52 | creates a new tilesplash server using the given postgres database 53 | 54 | ```javascript 55 | var dbConfig = { 56 | user: username, 57 | password: password, 58 | host: hostname, 59 | port: 5432, 60 | database: dbname 61 | } 62 | 63 | var app = new Tilesplash(dbConfig); 64 | ``` 65 | 66 | To cache using redis, pass `'redis'` as the second argument. Otherwise an in-process cache will be used. 67 | 68 | ### `Tilesplash.server` 69 | 70 | an [express](http://expressjs.com/) object, mostly used internally but you can use it to add middleware for authentication, browser caching, gzip, etc. 71 | 72 | ### `Tilesplash.layer(name, [middleware, ...], [mvtOptions], callback)` 73 | 74 | __name__: the name of your layer. Tiles will be served at /__name__/z/x/y.topojson 75 | 76 | __middleware__: a [middleware function](#middleware) 77 | 78 | __mvtOptions__: optional [mapnik parameters](http://mapnik.org/documentation/node-mapnik/3.5/#VectorTile.addGeoJSON), e.g. `{ strictly_simple: true }` 79 | 80 | __callback__: your tile building function with the following arguments. function([tile](#tile), [render](#render)) 81 | 82 | 83 | #### Simple layer 84 | 85 | This layer renders tiles containing geometry from the `the_geom` column in `test_table` 86 | 87 | ```javascript 88 | app.layer('simpleLayer', function(tile, render){ 89 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM test_table WHERE ST_Intersects(the_geom, !bbox_4326!)'); 90 | }); 91 | ``` 92 | 93 | #### Combined layers 94 | 95 | Tilesplash can render tiles from multiple queries at once 96 | 97 | ```javascript 98 | app.layer('multiLayer', function(tile, render){ 99 | render({ 100 | circles: 'SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM circles WHERE ST_Intersects(the_geom, !bbox_4326!)', 101 | squares: 'SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM squares WHERE ST_Intersects(the_geom, !bbox_4326!)' 102 | }); 103 | }); 104 | ``` 105 | 106 | In fact, `simpleLayer` is just a shorthand for `multiLayer` with the only query named `vectile`. 107 | The code below is equivalent to the _simple layer example_: 108 | 109 | ```javascript 110 | app.layer('multiLayer', function(tile, render){ 111 | render({ 112 | vectile: 'SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM test_table WHERE ST_Intersects(the_geom, !bbox_4326!)', 113 | }); 114 | }); 115 | ``` 116 | 117 | Knowing the name of the query (i.e. `vectile`, `circles`, `squares`, etc.) is important when rendering tiles on the client. 118 | The value corresponds to `source-layer` in [Mapbox GL JS](https://www.mapbox.com/mapbox-gl-js/api/) layer spec (see [example](https://www.mapbox.com/mapbox-gl-js/example/vector-source/)). 119 | 120 | 121 | #### Using mapnik geometry parameters 122 | 123 | This layer renders tiles containing geometry features simplified to a threshold of `4`. Full parameters are documented [here](http://mapnik.org/documentation/node-mapnik/3.5/#VectorTile.addGeoJSON). 124 | 125 | ```javascript 126 | app.layer('simpleLayer', { simplify_distance: 4 }, function(tile, render){ 127 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM test_table WHERE ST_Intersects(the_geom, !bbox_4326!)'); 128 | }); 129 | ``` 130 | 131 | #### Escaping variables 132 | 133 | Tilesplash has support for escaping variables in sql queries. You can do so by passing an array instead of a string wherever a sql string is accepted. 134 | 135 | ```javascript 136 | app.layer('escapedLayer', function(tile, render){ 137 | render(['SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM points WHERE ST_Intersects(the_geom, !bbox_4326!) AND state=$1', 'California']); 138 | }); 139 | 140 | app.layer('escapedMultiLayer', function(tile, render){ 141 | render({ 142 | hotels: ['SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM hotels WHERE ST_Intersects(the_geom, !bbox_4326!) AND state=$1', 'California'], 143 | restaurants: ['SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM restaurants WHERE ST_Intersects(the_geom, !bbox_4326!) AND state=$1', 'California'] 144 | }); 145 | }); 146 | ``` 147 | 148 | #### Restricting zoom level 149 | 150 | Sometimes you only want a layer to be visible on certain zoom levels. To do that, we simply render an empty tile when tile.z is too low or too high. 151 | 152 | ```javascript 153 | app.layer('zoomDependentLayer', function(tile, render){ 154 | if (tile.z < 8 || tile.z > 20) { 155 | render.empty(); //render an empty tile 156 | } else { 157 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM points WHERE ST_Intersects(the_geom, !bbox_4326!)'); 158 | } 159 | }); 160 | ``` 161 | 162 | You can also adapt your layer by zoom level to show different views in different situations. 163 | 164 | In this example we show data from the `heatmap` table when the zoom level is below 8, data from `points` up to zoom 20, and empty tiles when you zoom in further than that. 165 | 166 | ```javascript 167 | app.layer('fancyLayer', function(tile, render){ 168 | if (tile.z < 8) { 169 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM heatmap WHERE ST_Intersects(the_geom, !bbox_4326!)'); 170 | } else if (tile.z > 20) { 171 | render.empty(); 172 | } else { 173 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM points WHERE ST_Intersects(the_geom, !bbox_4326!)'); 174 | } 175 | }); 176 | ``` 177 | 178 | ## Middleware 179 | 180 | Middleware allows you to easily extend tilesplash to add additional functionality. Middleware is defined like this: 181 | 182 | ```javascript 183 | var userMiddleware = function(req, res, tile, next){ 184 | tile.logged_in = true; 185 | tile.user_id = req.query.user_id; 186 | next(); 187 | }; 188 | ``` 189 | 190 | You can layer include this in your layers 191 | 192 | ```javascript 193 | app.layer('thisOneHasMiddleware', userMiddleware, function(tile, render){ 194 | if (!tile.logged_in) { 195 | render.error(); 196 | } else { 197 | render(['SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM placesVisited WHERE ST_Intersects(the_geom, !bbox_4326!) AND visitor=$1', tile.user_id]); 198 | } 199 | }); 200 | ``` 201 | 202 | Middleware can be synchronous or asynchronous, just be sure to call `next()` when you're done! 203 | 204 | ## tile 205 | 206 | `tile` is a parameter passed to middleware and layer callbacks. It is an object containing information about the tile being requested. It will look something like this: 207 | 208 | ```javascript 209 | { 210 | x: 100, 211 | y: 100, 212 | z: 10, 213 | bounds: [w, s, e, n] //output from SphericalMercator.bounds(x,y,z) using https://github.com/mapbox/node-sphericalmercator 214 | bbox: 'BBOX SQL for webmercator', 215 | bbox_4326: 'BBOX SQL for 4326 projection' //you probably need this 216 | } 217 | ``` 218 | 219 | Anything in __tile__ can be substituted into your SQL query by wrapping it in exclamation marks like `!this!` 220 | 221 | You can add custom items into __tile__ like so: 222 | 223 | ```javascript 224 | tile.table = "states"; 225 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM !table! WHERE !bbox!') 226 | ``` 227 | 228 | Note that when you interpolate tile variables into your queries with the exclamation point syntax, that data will __not be escaped__. This allows you to insert custom SQL from tile variables, like with `!bbox!`, but it can be a security risk if you allow any user input to be interpolated that way. 229 | 230 | When you want to use user input in a query, see [Escaping variables](#escaping-variables) above. 231 | 232 | ## `render` 233 | 234 | `render` is the second argument passed to your layer callback function. You can use it to render different kinds of tiles. 235 | 236 | ### `render(sql)` 237 | 238 | Runs a SQL query and displays the result as a tile 239 | 240 | ### `render(object)` 241 | 242 | Runs multiple SQL queries and renders them in seperate topojson layers. See [Combined layers](#combined-layers) above. 243 | 244 | ### `render.query()` 245 | 246 | Alias of render() 247 | 248 | ### `render.queryFile(fileName)` 249 | 250 | Use this if your SQL is really long and/or you want to keep it seperate. 251 | 252 | ```javascript 253 | app.layer('complicatedLayer', function(tile, render){ 254 | render.queryFile('important_stuff/advanced_tile.sql'); 255 | }); 256 | ``` 257 | 258 | ### `render.empty()` 259 | 260 | Renders an empty tile 261 | 262 | ### `render.error()` 263 | 264 | Replies with a 500 error 265 | 266 | ### `render.raw(string or http code)` 267 | 268 | Sends a raw reply. I can't think of any reason you would want to do this, but feel free to experiment. 269 | 270 | ```javascript 271 | app.layer('smileyLayer', function(tile, render){ 272 | render.raw(':)'); 273 | }); 274 | ``` 275 | 276 | ```javascript 277 | app.layer('notThereLayer', function(tile, render){ 278 | render.raw(404); 279 | }); 280 | ``` 281 | 282 | ### `render.rawFile(fileName)` 283 | 284 | Replies with the specified file 285 | 286 | ```javascript 287 | app.layer('staticLayer', function(tile, render){ 288 | render.rawFile('thing.topojson'); 289 | }); 290 | ``` 291 | 292 | ## Caching 293 | 294 | Caching is very important. By default, Tilesplash uses an in-memory cache. You can use redis instead by passing `'redis'` as the second argument when initializing a Tilesplash server. 295 | 296 | There are two ways to implement caching. You can either do it globally or on a layer by layer basis. 297 | 298 | ### app.cache([keyGenerator], ttl) 299 | 300 | Use this to define caching across your entire application 301 | 302 | __`keyGenerator(tile)`__ 303 | 304 | keyGenerator is a function that takes a `tile` object as it's only parameter and returns a cache key (__string__) 305 | 306 | If you don't specify a key generator, `app.defaultCacheKeyGenerator` will be used, which returns a key derived from your database connection, tile layer, and tile x, y, and z. 307 | 308 | __`ttl`__ 309 | 310 | TTL stands for time-to-live. It's how long tiles will remain in your cache, and it's defined in milliseconds. For most applications, anywhere between one day (86400000) to one week (604800000) should be fine. 311 | 312 | __Example__ 313 | 314 | In this example, we have `tile.user_id` available to us and we don't want to show one user tiles belonging to another user. By starting with `app.defaultCacheKeyGenerator(tile)` we get a cache key based on things we already want to cache by (like `x`, `y`, and `z`) and we can then add `user_id` to prevent people from seeing cached tiles unless their `user_id` matches. 315 | 316 | ```javascript 317 | app.cache(function(tile){ 318 | return app.defaultCacheKeyGenerator(tile) + ':' + tile.user_id; //cache by tile.user_id as well 319 | }, 1000 * 60 * 60 * 24 * 30); //ttl 30 days 320 | ``` 321 | 322 | ### `this.cache([keyGenerator], ttl)` 323 | 324 | Layer-specific caching works identically to global caching as defined above, except that it only applies to one layer and you define it within that layer. 325 | 326 | In this example, slowLayer uses the same key generator as the rest of the app, but specifies a longer TTL. 327 | 328 | ```javascript 329 | app.cache(keyGenerator, 1000 * 60 * 60 * 24); //cache for one day 330 | 331 | app.layer('slowLayer', function(tile, render){ 332 | this.cache(1000 * 60 * 60 * 24 * 30); //cache for 30 days 333 | 334 | render.queryFile('slowQuery.sql'); 335 | }); 336 | ``` 337 | 338 | In this example, only slowLayer is cached. 339 | 340 | ```javascript 341 | app.layer('fastLayer', function(tile, render){ 342 | render.queryFile('fastQuery.sql'); 343 | }); 344 | 345 | var userMiddleware = function(req, res, tile, next){ 346 | tile.user_id = 1; 347 | next(); 348 | }; 349 | 350 | app.layer('slowLayer', userMiddleware, function(tile, render){ 351 | this.cache(function(tile){ 352 | return app.defaultCacheKeyGenerator(tile) + ':' + tile.user_id; 353 | }, 1000 * 60 * 60 * 24); //cache for one day 354 | 355 | render.queryFile('slowQuery.sql'); 356 | }); 357 | ``` 358 | 359 | ## Client 360 | Some in-browser examples of how to use the tiles generated by tilesplash: 361 | 362 | ### .mvt endpoint 363 | - [Mapbox GL third-party source example](https://www.mapbox.com/mapbox-gl-js/example/third-party/) 364 | - [Mapzen's Tangram](https://github.com/tangrams/tangram) 365 | - [Mapzen's d3 vector tiles implementation](http://mapzen.github.io/d3-vector-tiles) 366 | 367 | ### .topojson endpoint 368 | - [D3 + leaflet](http://bl.ocks.org/wboykinm/7393674) 369 | - [Mapzen Tangram](https://github.com/tangrams/tangram) 370 | -------------------------------------------------------------------------------- /demo/README.md: -------------------------------------------------------------------------------- 1 | # Tilesplash local demo 2 | 3 | This is a simple [Mapbox GL](https://www.mapbox.com/mapbox-gl-js/api/) example of tilesplash in use, serving points from the [openaddresses project](https://openaddresses.io/). 4 | 5 | ![denver](https://www.dropbox.com/s/oxrn5t7e48pvkum/denver.gif?dl=1) 6 | 7 | ## Dependencies 8 | 9 | - [PostgreSQL](https://www.postgresql.org/download/) 10 | - [PostGIS](http://postgis.net/install/) 11 | - [Node v6+](https://nodejs.org/en/download/) 12 | - [wget](https://www.gnu.org/software/wget/) 13 | 14 | ## Setup 15 | 16 | 1. Get a [Mapbox Token](http://mapbox.com/signup) and put it [in index.html](https://github.com/faradayio/tilesplash/blob/master/demo/index.html#L18) 17 | 18 | 2. Run this collection of should-really-be-in-a-docker-container commands to get sample data and configure the local DB: 19 | 20 | ``` 21 | bash demo_config.sh 22 | ``` 23 | 24 | ## Run the demo 25 | 26 | From this `demo/` directory: 27 | ``` 28 | npm install 29 | npm run points 30 | ``` 31 | 32 | Then visit [http://localhost:8000/points.html](http://localhost:8000/points.html) and watch as the glorious city of Denver unfolds in point form. -------------------------------------------------------------------------------- /demo/demo_config.sh: -------------------------------------------------------------------------------- 1 | # set up a testing environment for tilesplash using data from openaddresses 2 | 3 | # get the data 4 | wget -c https://s3.amazonaws.com/data.openaddresses.io/runs/227042/us/co/denver.zip 5 | # when feeling more ambitious, use all of the US West (~1GB): 6 | # wget -c https://s3.amazonaws.com/data.openaddresses.io/openaddr-collected-us_west.zip 7 | unzip denver.zip 8 | 9 | # create and configure the db 10 | dropdb tilesplash_demo --if-exists 11 | createdb tilesplash_demo 12 | psql tilesplash_demo -c "CREATE EXTENSION IF NOT EXISTS postgis" 13 | psql tilesplash_demo -c "CREATE TABLE oa ( 14 | lon float, 15 | lat float, 16 | number text, 17 | street text, 18 | unit text, 19 | city text, 20 | district text, 21 | region text, 22 | postcode text, 23 | id text, 24 | hash text 25 | )" 26 | psql tilesplash_demo -c "\COPY oa FROM 'us/co/denver.csv' CSV HEADER" 27 | psql tilesplash_demo -c "SELECT AddGeometryColumn ('public','oa','the_geom',4326,'GEOMETRY',2)" 28 | psql tilesplash_demo -c "UPDATE oa SET the_geom = ST_GeomFromText('POINT(' || lon || ' ' || lat || ')',4326)" 29 | 30 | echo "Ready to run the demo!" -------------------------------------------------------------------------------- /demo/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Tilesplash demo 6 | 7 | 8 | 9 | 13 | 14 | 15 | 16 |
17 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /demo/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tilesplash-demo", 3 | "version": "0.2.1", 4 | "description": "a demo server and client for tilesplash", 5 | "main": "test.js", 6 | "scripts": { 7 | "points": "node pointserver.js & static-server -p 8000" 8 | }, 9 | "author": "bill@faraday.io", 10 | "license": "MIT", 11 | "dependencies": { 12 | "static-server": "^2.0.5", 13 | "tilesplash": "^3.1.3" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /demo/points.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Tilesplash demo 6 | 7 | 8 | 9 | 13 | 14 | 15 | 16 |
17 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /demo/pointserver.js: -------------------------------------------------------------------------------- 1 | // USAGE: node tileserver.js 2 | "use strict"; 3 | let Tilesplash = require('tilesplash') 4 | 5 | // Set up DB connection params 6 | let config = { 7 | host: 'localhost', 8 | port: '5432', 9 | database: 'tilesplash_demo' 10 | } 11 | 12 | let app = new Tilesplash(config) 13 | 14 | // Add a cross-origin handler because EVERYTHING IS AWFUL 15 | function cors (req, res, next) { 16 | res.setHeader('Access-Control-Allow-Origin', '*') 17 | res.setHeader('Access-Control-Allow-Headers', 'X-Requested-With, authorization, content-type') 18 | next() 19 | } 20 | app.server.use(cors) 21 | 22 | // Build the tile layer 23 | app.layer("addresses", (tile, render) => { 24 | 25 | // Set up a zoom-based sampling scale to reduce the load on the client 26 | let sample = 0.1 27 | if (tile.z >= 12 && tile.z < 14) { 28 | sample = 0.25 29 | } else if (tile.z >= 14 && tile.z < 16) { 30 | sample = 0.5 31 | } else if (tile.z >= 16) { 32 | sample = 1 33 | } 34 | // Generate the PostGIS query 35 | let sql = ` 36 | SELECT 37 | ST_AsGeoJSON(the_geom) AS the_geom_geojson 38 | FROM oa 39 | WHERE ST_Intersects(the_geom, !bbox_4326!) 40 | AND random() < 41 | ` + sample 42 | ; 43 | render(sql); 44 | }); 45 | 46 | // Send it to port 47 | app.server.listen(3000) -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'); 2 | var express = require('express'); 3 | var async = require('async'); 4 | var topojson = require('topojson'); 5 | var pg = require('pg'); 6 | var SphericalMercator = require('@mapbox/sphericalmercator'); 7 | var Caching = require('caching'); 8 | var clone = require('clone'); 9 | var connectTimeout = require('connect-timeout'); 10 | var mapnik = require('mapnik'); 11 | var path = require('path'); 12 | 13 | mapnik.register_datasource(path.join(mapnik.settings.paths.input_plugins, 'geojson.input')); 14 | 15 | function setStatementTimeout(client) { 16 | return new Promise(function(resolve, reject){ 17 | var statementTimeout = parseInt(process.env.TILESPLASH_STATEMENT_TIMEOUT) || 0; 18 | if (statementTimeout && statementTimeout > 0) { 19 | client.query('SET statement_timeout='+statementTimeout, [], function(err){ 20 | if (err) { 21 | reject(err); 22 | } else { 23 | resolve(); 24 | } 25 | }); 26 | } else { 27 | resolve(); 28 | } 29 | }); 30 | } 31 | 32 | // allowing tho options object for addGeoJson() - simplification settings, etc. 33 | function stringifyProtobuf(layers, tile, mvtOptions) { 34 | var vtile = new mapnik.VectorTile(tile.z, tile.x, tile.y); 35 | for (var layerName in layers) { 36 | vtile.addGeoJSON(JSON.stringify(layers[layerName]), layerName, mvtOptions); 37 | } 38 | return vtile.getData(); 39 | } 40 | 41 | var pgMiddleware = function(_dbOptions){ 42 | var dbOptions; 43 | if (typeof _dbOptions === 'string') { 44 | dbOptions = _dbOptions; 45 | } else { 46 | dbOptions = {}; 47 | Object.keys(_dbOptions).forEach(function(key){ 48 | dbOptions[key] = _dbOptions[key]; 49 | }); 50 | dbOptions.poolSize = dbOptions.poolSize || 64; 51 | } 52 | 53 | return function(req, res, next){ 54 | req.db = {}; 55 | req.db.query = function(sql, bindvars, callback){ 56 | if (typeof bindvars === 'function') { 57 | callback = bindvars; 58 | bindvars = undefined; 59 | } 60 | 61 | var poolTimedOut = false; 62 | var poolTimeout = setTimeout(function(){ 63 | poolTimedOut = true; 64 | callback(new Error('failed to get db connection')); 65 | }, 5000); 66 | 67 | var pool = new pg.Pool(dbOptions); 68 | 69 | pool.connect(function(err, client, done){ 70 | if (poolTimedOut) { 71 | done(); 72 | return; 73 | } 74 | 75 | clearTimeout(poolTimeout); 76 | if (err) { 77 | callback(err); 78 | done(); 79 | return; 80 | } 81 | 82 | res.on('finish', done); 83 | res.on('error', done); 84 | req.on('timeout', done); 85 | 86 | setStatementTimeout(client).then(function(){ 87 | client.query(sql, bindvars, function(err, result){ 88 | callback(err, result); 89 | done(); 90 | }); 91 | }).catch(function(err){ 92 | callback(err); 93 | done(); 94 | }); 95 | }); 96 | }; 97 | 98 | next(); 99 | }; 100 | }; 101 | 102 | /* 103 | Tilesplash 104 | @constructor 105 | 106 | - dbOptions : options passed to the postgres database, required 107 | - cacheType : how to cache tiles, defaults to 'memory', optional 108 | - cacheOptions : options to pass to the caching library , optional 109 | - instrument : probes to instrument various parts of the query cycle, optional. Instrumentation points: 110 | - parseSql: time it takes to parse sql templates into sql strings 111 | - toGeoJson: the time it takes to run the sql and translate it to geojson 112 | - runQuery: the time it takes to run a query (I think) 113 | - gotTile: the round-trip time to get a tile (I think) 114 | */ 115 | var Tilesplash = function(dbOptions, cacheType, cacheOptions, instrument){ 116 | this.projection = new SphericalMercator({ 117 | size: 256 118 | }); 119 | 120 | this.dbOptions = dbOptions; 121 | this.dbCacheKey = JSON.stringify(dbOptions); 122 | 123 | this.server = express(); 124 | this.instrument = instrument || {}; 125 | var timeout = parseInt(process.env.TILESPLASH_REQUEST_TIMEOUT) || 0; 126 | 127 | if (timeout) { 128 | this.server.use(connectTimeout(timeout)); 129 | } 130 | 131 | var self = this; 132 | this.server.use(function(req, res, next){ 133 | req.on('timeout', function(){ 134 | self.log('Request timed out', 'error'); 135 | }); 136 | next(); 137 | }); 138 | 139 | this.server.use(pgMiddleware(dbOptions)); 140 | 141 | this.cacheOptions = cacheOptions || {}; 142 | this._cache = new Caching(cacheType || 'memory', cacheOptions); 143 | 144 | this.logLevel('info'); 145 | }; 146 | 147 | Tilesplash.prototype.logging = function(callback){ 148 | this.log = callback; 149 | }; 150 | 151 | Tilesplash.prototype.logLevel = function(logLevel){ 152 | var logLevels = ['debug', 'info', 'warn', 'error']; 153 | logLevel = logLevels.indexOf(logLevel); 154 | 155 | this.logging(function(message, level){ 156 | var messageLevel = logLevels.indexOf(level); 157 | if (logLevel <= messageLevel) { 158 | console.log('['+level+']', JSON.stringify(message, null, 2)); 159 | } 160 | }); 161 | }; 162 | 163 | Tilesplash.prototype.defaultCacheKeyGenerator = function(tile){ 164 | return 'tilesplash_tile:'+this.dbCacheKey+'/'+tile.layer+'/'+tile.z+'/'+tile.x+'/'+tile.y; 165 | }; 166 | 167 | Tilesplash.prototype.cache = function(cacheKeyGenerator, ttl){ 168 | var self = this; 169 | if (typeof cacheKeyGenerator === 'number') { 170 | ttl = cacheKeyGenerator; 171 | cacheKeyGenerator = undefined; 172 | } 173 | this.defaultCacher = cacheKeyGenerator || function(tile){ 174 | return self.defaultCacheKeyGenerator(tile); 175 | }; 176 | this.defaultTtl = ttl; 177 | }; 178 | 179 | function stringifyTopojson(layers){ 180 | layers = clone(layers); 181 | return topojson.topology(layers, { 182 | 'property-transform': function(properties, key, value){ 183 | properties[key] = value; 184 | return true; 185 | } 186 | }); 187 | } 188 | 189 | Tilesplash.prototype.layer = function(name){ 190 | var callbacks = Array.prototype.slice.call(arguments); 191 | callbacks.shift(); 192 | 193 | // pop off renderer for future use 194 | var tileRenderer = callbacks.pop(); 195 | 196 | var mvtOptions = {}; 197 | // if the last argument is an object, use it as mvtOptions 198 | if (typeof callbacks[callbacks.length - 1] === 'object') { 199 | mvtOptions = callbacks.pop() 200 | } 201 | 202 | var self = this; 203 | 204 | // actual route creation 205 | this.server.get('/'+name+'/:z/:x/:y.:ext', function(req, res, throwError){ 206 | var render = function(){ 207 | render.query.apply(render, arguments); 208 | }; 209 | render.query = function(data){ 210 | gotTile(data); 211 | }; 212 | render.queryFile = function(file, encoding){ 213 | fs.readFile(file, encoding || 'utf8', function(err, data){ 214 | if (err) { 215 | render.error(err); 216 | } else { 217 | gotTile(data); 218 | } 219 | }); 220 | }; 221 | render.raw = function(){ 222 | res.send.apply(res, arguments); 223 | }; 224 | render.rawFile = function(file, encoding){ 225 | fs.readFile(file, encoding || 'utf8', function(err, data){ 226 | if (err) { 227 | render.error(err); 228 | } else { 229 | render.raw(data); 230 | } 231 | }); 232 | }; 233 | 234 | render.error = function(msg){ 235 | render.raw(500); 236 | throwError( (msg instanceof Error) ? msg : new Error(msg) ); 237 | }; 238 | 239 | if (req.params.ext != 'topojson' && req.params.ext != 'mvt') { 240 | render.error('unsupported extension '+req.params.ext); 241 | return; 242 | } 243 | 244 | var stringify = (req.params.ext == 'topojson') ? stringifyTopojson : stringifyProtobuf; 245 | 246 | var tile = {}; 247 | tile.layer = name; 248 | tile.x = req.params.x*1; 249 | tile.y = req.params.y*1; 250 | tile.z = req.params.z*1; 251 | 252 | render.empty = function(){ 253 | render.raw(stringify([], tile, {})); 254 | }; 255 | 256 | tile.bounds = self.projection.bbox(req.params.x, req.params.y, req.params.z, false, '900913'); 257 | tile.bbox = [ 258 | 'ST_SetSRID(', 259 | 'ST_MakeBox2D(', 260 | 'ST_MakePoint(', tile.bounds[0], ', ', tile.bounds[1], '), ', 261 | 'ST_MakePoint(', tile.bounds[2], ', ', tile.bounds[3], ')', 262 | '), ', 263 | '3857', 264 | ')' 265 | ].join(''); 266 | tile.bbox_4326 = 'ST_Transform('+tile.bbox+', 4326)'; 267 | tile.geom_hash = 'Substr(MD5(ST_AsBinary(the_geom)), 1, 10)'; 268 | 269 | self.log('Rendering tile '+tile.layer+'/'+tile.z+'/'+tile.x+'/'+tile.y, 'debug'); 270 | 271 | function startTrace() { 272 | var start = process.hrtime(); 273 | return function endTrace() { 274 | var diff = process.hrtime(start); 275 | 276 | return (diff[0] * 1e9 + diff[1])/1e6; 277 | }; 278 | } 279 | 280 | 281 | var parseSql = function(sql, done){ 282 | var trace = startTrace(), 283 | instrument = self.instrument.parseSql || function() {}; 284 | 285 | if (typeof sql === 'number') { 286 | instrument(trace()); 287 | done(null, sql); 288 | return; 289 | } 290 | if (typeof sql === 'object') { 291 | if (Array.isArray(sql)) { 292 | async.map(sql, function(item, next){ 293 | parseSql(item, function(err, out){ 294 | next(err, out); 295 | }); 296 | }, function(err, results){ 297 | instrument(trace()); 298 | done(err, results); 299 | }); 300 | return; 301 | } else { 302 | var keys = Object.keys(sql); 303 | async.map(keys, function(item, next){ 304 | parseSql(sql[item], function(err, out){ 305 | next(err, out); 306 | }); 307 | }, function(err, results){ 308 | var output = {}; 309 | keys.forEach(function(d, i){ 310 | output[d] = results[i]; 311 | }); 312 | 313 | instrument(trace()); 314 | done(err, output); 315 | }); 316 | return; 317 | } 318 | } 319 | if (typeof sql !== 'string') { 320 | done(['Trying to parse non-string SQL', sql]); 321 | return; 322 | } 323 | var templatePattern = /!([0-9a-zA-Z_\-]+)!/g; 324 | 325 | sql = sql.replace(templatePattern, function(match){ 326 | match = match.substr(1, match.length-2); 327 | return tile[match]; 328 | }); 329 | done(null, sql); 330 | }; 331 | 332 | var sqlToGeojson = function(sql, done){ 333 | var instrument = self.instrument.toGeoJson || function() {}, 334 | trace = startTrace(); 335 | 336 | if (sql === false || sql === null) { 337 | instrument(trace()); 338 | done(null, {}); 339 | return; 340 | } 341 | parseSql(sql, function(parsingError, fullsql){ 342 | if (parsingError) { 343 | instrument(trace()); 344 | done(parsingError); 345 | return; 346 | } 347 | var args = []; 348 | if (typeof fullsql === 'object' && Array.isArray(fullsql)) { 349 | args = fullsql; 350 | fullsql = args.shift(); 351 | } 352 | self.log('Running Query', 'debug'); 353 | self.log(' SQL: '+fullsql, 'debug'); 354 | self.log(' Arguments: '+JSON.stringify(args), 'debug'); 355 | 356 | 357 | req.db.query(fullsql, args, function(sqlError, result){ 358 | if (sqlError) { 359 | instrument(trace()); 360 | done([sqlError, fullsql, args]); 361 | } else { 362 | var geojson = { 363 | "type": "FeatureCollection", 364 | "features": [] 365 | }; 366 | result.rows.forEach(function(row){ 367 | var properties = {}; 368 | for (var attribute in row) { 369 | if (attribute !== 'the_geom_geojson') { 370 | properties[attribute] = row[attribute]; 371 | } 372 | } 373 | geojson.features.push({ 374 | "type": "Feature", 375 | "geometry": JSON.parse(row.the_geom_geojson), 376 | "properties": properties 377 | }); 378 | }); 379 | 380 | instrument(trace()); 381 | done(null, geojson); 382 | } 383 | }); 384 | }); 385 | }; 386 | 387 | var tileContext = { 388 | cache: function(cacheKeyGenerator, ttl){ 389 | if (typeof cacheKeyGenerator === 'number') { 390 | ttl = cacheKeyGenerator; 391 | cacheKeyGenerator = undefined; 392 | } 393 | if (typeof cacheKeyGenerator === 'function') { 394 | this._cacher = cacheKeyGenerator; 395 | } 396 | this._ttl = ttl; 397 | } 398 | }; 399 | 400 | var runQuery = function(structuredSql, done){ 401 | var instrument = self.instrument.runQuery || function() {}, 402 | trace = startTrace(); 403 | 404 | if (structuredSql === false || structuredSql === null) { 405 | instrument(trace()); 406 | done(null, {}); 407 | return; 408 | } 409 | 410 | if (typeof structuredSql !== 'object' || Array.isArray(structuredSql)) { 411 | structuredSql = {vectile: structuredSql}; 412 | } 413 | 414 | var geojsonLayers = {}; 415 | async.forEach(Object.keys(structuredSql), function(layer, next){ 416 | sqlToGeojson(structuredSql[layer], function(err, geojson){ 417 | if (err) { 418 | next(err); 419 | } else { 420 | geojsonLayers[layer] = geojson; 421 | next(); 422 | } 423 | }); 424 | }, function(err){ 425 | instrument(trace()); 426 | if (err) { 427 | done(err); 428 | } else { 429 | done(null, geojsonLayers); 430 | } 431 | }); 432 | }; 433 | 434 | var gotTile = function(tileOutput){ 435 | var instrument = self.instrument.gotTile || function() {}, 436 | trace = startTrace(); 437 | 438 | parseSql(tileOutput, function(parsingError, structuredSql){ 439 | if (parsingError) { 440 | instrument(trace()); 441 | render.error(['SQL Parsing Error', parsingError]); 442 | return; 443 | } 444 | var cacher = tileContext._cacher || self.defaultCacher || false; 445 | 446 | if (cacher) { 447 | var cacheKey = cacher(tile); 448 | var ttl; 449 | if (typeof tileContext._ttl === 'number') { 450 | ttl = tileContext._ttl; 451 | } else if (typeof self.defaultTtl === 'number') { 452 | ttl = self.defaultTtl; 453 | } else { 454 | ttl = 0; 455 | } 456 | self._cache(cacheKey, ttl, function(done){ 457 | instrument(trace()); 458 | self.log('cache miss', 'debug'); 459 | runQuery(structuredSql, function(queryError, layers){ 460 | done(queryError, layers); 461 | }); 462 | }, function(layerError, layers){ 463 | instrument(trace()); 464 | if (layerError) { 465 | render.error(['layer error', layerError]); 466 | } else { 467 | res.send(stringify(layers, tile, mvtOptions)); 468 | } 469 | }); 470 | } else { 471 | runQuery(structuredSql, function(queryError, layers){ 472 | instrument(trace()); 473 | if (queryError) { 474 | render.error(['error running query', queryError]); 475 | } else { 476 | res.send(stringify(layers, tile, mvtOptions)); 477 | } 478 | }); 479 | } 480 | }); 481 | }; 482 | 483 | async.eachSeries(callbacks, function(middleware, next){ 484 | middleware(req, res, tile, function(err){ 485 | next(err); 486 | }); 487 | }, function(err){ 488 | if (err) { 489 | render.error(['Middleware error', err]); 490 | } else { 491 | tileRenderer.call(tileContext, tile, render); 492 | } 493 | }); 494 | }); 495 | }; 496 | 497 | module.exports = Tilesplash; 498 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tilesplash", 3 | "description": "A light and quick nodejs webserver for serving mapbox vector tiles (and topojson) from a postgis backend", 4 | "version": "3.1.3", 5 | "maintainers": [ 6 | { 7 | "name": "Tristan Davies", 8 | "email": "npm@tristan.io" 9 | }, 10 | { 11 | "name": "Nick Husher", 12 | "email": "nhusher@gmail.com" 13 | }, 14 | { 15 | "name": "Bill Morris", 16 | "email": "bill@faraday.io" 17 | } 18 | ], 19 | "license": "MIT", 20 | "repository": { 21 | "type": "git", 22 | "url": "https://github.com/faradayio/tilesplash.git" 23 | }, 24 | "main": "lib/index.js", 25 | "scripts": { 26 | "test": "mocha" 27 | }, 28 | "devDependencies": { 29 | "path": "^0.12.7", 30 | "request": "~2.36.0", 31 | "mocha": "~1.20.0" 32 | }, 33 | "dependencies": { 34 | "@mapbox/sphericalmercator": "^1.0.5", 35 | "async": "^2.5.0", 36 | "caching": "^0.1.4", 37 | "clone": "^2.1.1", 38 | "connect-timeout": "^1.9.0", 39 | "express": "^4.15.3", 40 | "mapnik": "~3.6.2", 41 | "pg": "^6.4.1", 42 | "topojson": "^3.0.0" 43 | } 44 | } --------------------------------------------------------------------------------