├── demo
├── package.json
├── demo_config.sh
├── README.md
├── pointserver.js
├── index.html
└── points.html
├── .gitignore
├── LICENSE
├── package.json
├── README.md
└── lib
└── index.js
/demo/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "tilesplash-demo",
3 | "version": "0.2.1",
4 | "description": "a demo server and client for tilesplash",
5 | "main": "test.js",
6 | "scripts": {
7 | "points": "node pointserver.js & static-server -p 8000"
8 | },
9 | "author": "bill@faraday.io",
10 | "license": "MIT",
11 | "dependencies": {
12 | "static-server": "^2.0.5",
13 | "tilesplash": "^3.1.3"
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 |
5 | # Runtime data
6 | pids
7 | *.pid
8 | *.seed
9 |
10 | # Directory for instrumented libs generated by jscoverage/JSCover
11 | lib-cov
12 |
13 | # Coverage directory used by tools like istanbul
14 | coverage
15 |
16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
17 | .grunt
18 |
19 | # Compiled binary addons (http://nodejs.org/api/addons.html)
20 | build/Release
21 |
22 | # Dependency directory
23 | # Deployed apps should consider commenting this line out:
24 | # see https://npmjs.org/doc/faq.html#Should-I-check-my-node_modules-folder-into-git
25 | node_modules
26 |
27 | *.DS_Store
28 | *.zip
29 | *.vrt
30 | *.csv
31 | *.txt
--------------------------------------------------------------------------------
/demo/demo_config.sh:
--------------------------------------------------------------------------------
1 | # set up a testing environment for tilesplash using data from openaddresses
2 |
3 | # get the data
4 | wget -c https://s3.amazonaws.com/data.openaddresses.io/runs/227042/us/co/denver.zip
5 | # when feeling more ambitious, use all of the US West (~1GB):
6 | # wget -c https://s3.amazonaws.com/data.openaddresses.io/openaddr-collected-us_west.zip
7 | unzip denver.zip
8 |
9 | # create and configure the db
10 | dropdb tilesplash_demo --if-exists
11 | createdb tilesplash_demo
12 | psql tilesplash_demo -c "CREATE EXTENSION IF NOT EXISTS postgis"
13 | psql tilesplash_demo -c "CREATE TABLE oa (
14 | lon float,
15 | lat float,
16 | number text,
17 | street text,
18 | unit text,
19 | city text,
20 | district text,
21 | region text,
22 | postcode text,
23 | id text,
24 | hash text
25 | )"
26 | psql tilesplash_demo -c "\COPY oa FROM 'us/co/denver.csv' CSV HEADER"
27 | psql tilesplash_demo -c "SELECT AddGeometryColumn ('public','oa','the_geom',4326,'GEOMETRY',2)"
28 | psql tilesplash_demo -c "UPDATE oa SET the_geom = ST_GeomFromText('POINT(' || lon || ' ' || lat || ')',4326)"
29 |
30 | echo "Ready to run the demo!"
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2014 Faraday
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
--------------------------------------------------------------------------------
/demo/README.md:
--------------------------------------------------------------------------------
1 | # Tilesplash local demo
2 |
3 | This is a simple [Mapbox GL](https://www.mapbox.com/mapbox-gl-js/api/) example of tilesplash in use, serving points from the [openaddresses project](https://openaddresses.io/).
4 |
5 | 
6 |
7 | ## Dependencies
8 |
9 | - [PostgreSQL](https://www.postgresql.org/download/)
10 | - [PostGIS](http://postgis.net/install/)
11 | - [Node v6+](https://nodejs.org/en/download/)
12 | - [wget](https://www.gnu.org/software/wget/)
13 |
14 | ## Setup
15 |
16 | 1. Get a [Mapbox Token](http://mapbox.com/signup) and put it [in index.html](https://github.com/faradayio/tilesplash/blob/master/demo/index.html#L18)
17 |
18 | 2. Run this collection of should-really-be-in-a-docker-container commands to get sample data and configure the local DB:
19 |
20 | ```
21 | bash demo_config.sh
22 | ```
23 |
24 | ## Run the demo
25 |
26 | From this `demo/` directory:
27 | ```
28 | npm install
29 | npm run points
30 | ```
31 |
32 | Then visit [http://localhost:8000/points.html](http://localhost:8000/points.html) and watch as the glorious city of Denver unfolds in point form.
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "tilesplash",
3 | "description": "A light and quick nodejs webserver for serving mapbox vector tiles (and topojson) from a postgis backend",
4 | "version": "3.1.3",
5 | "maintainers": [
6 | {
7 | "name": "Tristan Davies",
8 | "email": "npm@tristan.io"
9 | },
10 | {
11 | "name": "Nick Husher",
12 | "email": "nhusher@gmail.com"
13 | },
14 | {
15 | "name": "Bill Morris",
16 | "email": "bill@faraday.io"
17 | }
18 | ],
19 | "license": "MIT",
20 | "repository": {
21 | "type": "git",
22 | "url": "https://github.com/faradayio/tilesplash.git"
23 | },
24 | "main": "lib/index.js",
25 | "scripts": {
26 | "test": "mocha"
27 | },
28 | "devDependencies": {
29 | "path": "^0.12.7",
30 | "request": "~2.36.0",
31 | "mocha": "~1.20.0"
32 | },
33 | "dependencies": {
34 | "@mapbox/sphericalmercator": "^1.0.5",
35 | "async": "^2.5.0",
36 | "caching": "^0.1.4",
37 | "clone": "^2.1.1",
38 | "connect-timeout": "^1.9.0",
39 | "express": "^4.15.3",
40 | "mapnik": "~3.6.2",
41 | "pg": "^6.4.1",
42 | "topojson": "^3.0.0"
43 | }
44 | }
--------------------------------------------------------------------------------
/demo/pointserver.js:
--------------------------------------------------------------------------------
1 | // USAGE: node tileserver.js
2 | "use strict";
3 | let Tilesplash = require('tilesplash')
4 |
5 | // Set up DB connection params
6 | let config = {
7 | host: 'localhost',
8 | port: '5432',
9 | database: 'tilesplash_demo'
10 | }
11 |
12 | let app = new Tilesplash(config)
13 |
14 | // Add a cross-origin handler because EVERYTHING IS AWFUL
15 | function cors (req, res, next) {
16 | res.setHeader('Access-Control-Allow-Origin', '*')
17 | res.setHeader('Access-Control-Allow-Headers', 'X-Requested-With, authorization, content-type')
18 | next()
19 | }
20 | app.server.use(cors)
21 |
22 | // Build the tile layer
23 | app.layer("addresses", (tile, render) => {
24 |
25 | // Set up a zoom-based sampling scale to reduce the load on the client
26 | let sample = 0.1
27 | if (tile.z >= 12 && tile.z < 14) {
28 | sample = 0.25
29 | } else if (tile.z >= 14 && tile.z < 16) {
30 | sample = 0.5
31 | } else if (tile.z >= 16) {
32 | sample = 1
33 | }
34 | // Generate the PostGIS query
35 | let sql = `
36 | SELECT
37 | ST_AsGeoJSON(the_geom) AS the_geom_geojson
38 | FROM oa
39 | WHERE ST_Intersects(the_geom, !bbox_4326!)
40 | AND random() <
41 | ` + sample
42 | ;
43 | render(sql);
44 | });
45 |
46 | // Send it to port
47 | app.server.listen(3000)
--------------------------------------------------------------------------------
/demo/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Tilesplash demo
6 |
7 |
8 |
9 |
13 |
14 |
15 |
16 |
17 |
64 |
65 |
66 |
67 |
--------------------------------------------------------------------------------
/demo/points.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Tilesplash demo
6 |
7 |
8 |
9 |
13 |
14 |
15 |
16 |
17 |
70 |
71 |
72 |
73 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | TILESPLASH
2 | ==========
3 |
4 | A light and quick nodejs webserver for serving topojson and mapbox vector tiles from a [postgis](http://www.postgis.net/) backend. inspired by [Michal Migurski](http://mike.teczno.com/)'s [TileStache](http://tilestache.org/). Works great for powering [Mapbox-GL-based](https://www.mapbox.com/mapbox-gl-js/example/set-perspective/) apps like this:
5 |
6 | 
7 |
8 | # Dependencies
9 |
10 | Tilesplash depends on `node` and `npm`
11 |
12 | # Installation
13 |
14 | ```bash
15 | npm install tilesplash
16 | ```
17 |
18 | # Example
19 |
20 | Here's a simple tile server with one layer
21 |
22 | ```javascript
23 | var Tilesplash = require('tilesplash');
24 |
25 | // invoke tilesplash with DB options
26 | var app = new Tilesplash({
27 | user: myUser,
28 | password: myPassword,
29 | host: localhost,
30 | port: 5432,
31 | database: myDb
32 | });
33 |
34 | // define a layer
35 | app.layer('test_layer', function(tile, render){
36 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM layer WHERE ST_Intersects(the_geom, !bbox_4326!)');
37 | });
38 |
39 | // serve tiles at port 3000
40 | app.server.listen(3000);
41 | ```
42 |
43 | - Topojson tiles will be available at `http://localhost:3000/test_layer/{z}/{x}/{y}.topojson`
44 | - Mapbox vector tiles will be available at `http://localhost:3000/test_layer/{z}/{x}/{y}.mvt`
45 |
46 | (See [client implementation examples](https://github.com/faradayio/tilesplash#client) below, and complete demo implementation in [`demo/`](demo/README.md))
47 |
48 | # Usage
49 |
50 | ## `new Tilesplash(connection_details, [cacheType])`
51 |
52 | creates a new tilesplash server using the given postgres database
53 |
54 | ```javascript
55 | var dbConfig = {
56 | user: username,
57 | password: password,
58 | host: hostname,
59 | port: 5432,
60 | database: dbname
61 | }
62 |
63 | var app = new Tilesplash(dbConfig);
64 | ```
65 |
66 | To cache using redis, pass `'redis'` as the second argument. Otherwise an in-process cache will be used.
67 |
68 | ### `Tilesplash.server`
69 |
70 | an [express](http://expressjs.com/) object, mostly used internally but you can use it to add middleware for authentication, browser caching, gzip, etc.
71 |
72 | ### `Tilesplash.layer(name, [middleware, ...], [mvtOptions], callback)`
73 |
74 | __name__: the name of your layer. Tiles will be served at /__name__/z/x/y.topojson
75 |
76 | __middleware__: a [middleware function](#middleware)
77 |
78 | __mvtOptions__: optional [mapnik parameters](http://mapnik.org/documentation/node-mapnik/3.5/#VectorTile.addGeoJSON), e.g. `{ strictly_simple: true }`
79 |
80 | __callback__: your tile building function with the following arguments. function([tile](#tile), [render](#render))
81 |
82 |
83 | #### Simple layer
84 |
85 | This layer renders tiles containing geometry from the `the_geom` column in `test_table`
86 |
87 | ```javascript
88 | app.layer('simpleLayer', function(tile, render){
89 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM test_table WHERE ST_Intersects(the_geom, !bbox_4326!)');
90 | });
91 | ```
92 |
93 | #### Combined layers
94 |
95 | Tilesplash can render tiles from multiple queries at once
96 |
97 | ```javascript
98 | app.layer('multiLayer', function(tile, render){
99 | render({
100 | circles: 'SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM circles WHERE ST_Intersects(the_geom, !bbox_4326!)',
101 | squares: 'SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM squares WHERE ST_Intersects(the_geom, !bbox_4326!)'
102 | });
103 | });
104 | ```
105 |
106 | In fact, `simpleLayer` is just a shorthand for `multiLayer` with the only query named `vectile`.
107 | The code below is equivalent to the _simple layer example_:
108 |
109 | ```javascript
110 | app.layer('multiLayer', function(tile, render){
111 | render({
112 | vectile: 'SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM test_table WHERE ST_Intersects(the_geom, !bbox_4326!)',
113 | });
114 | });
115 | ```
116 |
117 | Knowing the name of the query (i.e. `vectile`, `circles`, `squares`, etc.) is important when rendering tiles on the client.
118 | The value corresponds to `source-layer` in [Mapbox GL JS](https://www.mapbox.com/mapbox-gl-js/api/) layer spec (see [example](https://www.mapbox.com/mapbox-gl-js/example/vector-source/)).
119 |
120 |
121 | #### Using mapnik geometry parameters
122 |
123 | This layer renders tiles containing geometry features simplified to a threshold of `4`. Full parameters are documented [here](http://mapnik.org/documentation/node-mapnik/3.5/#VectorTile.addGeoJSON).
124 |
125 | ```javascript
126 | app.layer('simpleLayer', { simplify_distance: 4 }, function(tile, render){
127 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM test_table WHERE ST_Intersects(the_geom, !bbox_4326!)');
128 | });
129 | ```
130 |
131 | #### Escaping variables
132 |
133 | Tilesplash has support for escaping variables in sql queries. You can do so by passing an array instead of a string wherever a sql string is accepted.
134 |
135 | ```javascript
136 | app.layer('escapedLayer', function(tile, render){
137 | render(['SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM points WHERE ST_Intersects(the_geom, !bbox_4326!) AND state=$1', 'California']);
138 | });
139 |
140 | app.layer('escapedMultiLayer', function(tile, render){
141 | render({
142 | hotels: ['SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM hotels WHERE ST_Intersects(the_geom, !bbox_4326!) AND state=$1', 'California'],
143 | restaurants: ['SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM restaurants WHERE ST_Intersects(the_geom, !bbox_4326!) AND state=$1', 'California']
144 | });
145 | });
146 | ```
147 |
148 | #### Restricting zoom level
149 |
150 | Sometimes you only want a layer to be visible on certain zoom levels. To do that, we simply render an empty tile when tile.z is too low or too high.
151 |
152 | ```javascript
153 | app.layer('zoomDependentLayer', function(tile, render){
154 | if (tile.z < 8 || tile.z > 20) {
155 | render.empty(); //render an empty tile
156 | } else {
157 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM points WHERE ST_Intersects(the_geom, !bbox_4326!)');
158 | }
159 | });
160 | ```
161 |
162 | You can also adapt your layer by zoom level to show different views in different situations.
163 |
164 | In this example we show data from the `heatmap` table when the zoom level is below 8, data from `points` up to zoom 20, and empty tiles when you zoom in further than that.
165 |
166 | ```javascript
167 | app.layer('fancyLayer', function(tile, render){
168 | if (tile.z < 8) {
169 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM heatmap WHERE ST_Intersects(the_geom, !bbox_4326!)');
170 | } else if (tile.z > 20) {
171 | render.empty();
172 | } else {
173 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM points WHERE ST_Intersects(the_geom, !bbox_4326!)');
174 | }
175 | });
176 | ```
177 |
178 | ## Middleware
179 |
180 | Middleware allows you to easily extend tilesplash to add additional functionality. Middleware is defined like this:
181 |
182 | ```javascript
183 | var userMiddleware = function(req, res, tile, next){
184 | tile.logged_in = true;
185 | tile.user_id = req.query.user_id;
186 | next();
187 | };
188 | ```
189 |
190 | You can layer include this in your layers
191 |
192 | ```javascript
193 | app.layer('thisOneHasMiddleware', userMiddleware, function(tile, render){
194 | if (!tile.logged_in) {
195 | render.error();
196 | } else {
197 | render(['SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM placesVisited WHERE ST_Intersects(the_geom, !bbox_4326!) AND visitor=$1', tile.user_id]);
198 | }
199 | });
200 | ```
201 |
202 | Middleware can be synchronous or asynchronous, just be sure to call `next()` when you're done!
203 |
204 | ## tile
205 |
206 | `tile` is a parameter passed to middleware and layer callbacks. It is an object containing information about the tile being requested. It will look something like this:
207 |
208 | ```javascript
209 | {
210 | x: 100,
211 | y: 100,
212 | z: 10,
213 | bounds: [w, s, e, n] //output from SphericalMercator.bounds(x,y,z) using https://github.com/mapbox/node-sphericalmercator
214 | bbox: 'BBOX SQL for webmercator',
215 | bbox_4326: 'BBOX SQL for 4326 projection' //you probably need this
216 | }
217 | ```
218 |
219 | Anything in __tile__ can be substituted into your SQL query by wrapping it in exclamation marks like `!this!`
220 |
221 | You can add custom items into __tile__ like so:
222 |
223 | ```javascript
224 | tile.table = "states";
225 | render('SELECT ST_AsGeoJSON(the_geom) as the_geom_geojson FROM !table! WHERE !bbox!')
226 | ```
227 |
228 | Note that when you interpolate tile variables into your queries with the exclamation point syntax, that data will __not be escaped__. This allows you to insert custom SQL from tile variables, like with `!bbox!`, but it can be a security risk if you allow any user input to be interpolated that way.
229 |
230 | When you want to use user input in a query, see [Escaping variables](#escaping-variables) above.
231 |
232 | ## `render`
233 |
234 | `render` is the second argument passed to your layer callback function. You can use it to render different kinds of tiles.
235 |
236 | ### `render(sql)`
237 |
238 | Runs a SQL query and displays the result as a tile
239 |
240 | ### `render(object)`
241 |
242 | Runs multiple SQL queries and renders them in seperate topojson layers. See [Combined layers](#combined-layers) above.
243 |
244 | ### `render.query()`
245 |
246 | Alias of render()
247 |
248 | ### `render.queryFile(fileName)`
249 |
250 | Use this if your SQL is really long and/or you want to keep it seperate.
251 |
252 | ```javascript
253 | app.layer('complicatedLayer', function(tile, render){
254 | render.queryFile('important_stuff/advanced_tile.sql');
255 | });
256 | ```
257 |
258 | ### `render.empty()`
259 |
260 | Renders an empty tile
261 |
262 | ### `render.error()`
263 |
264 | Replies with a 500 error
265 |
266 | ### `render.raw(string or http code)`
267 |
268 | Sends a raw reply. I can't think of any reason you would want to do this, but feel free to experiment.
269 |
270 | ```javascript
271 | app.layer('smileyLayer', function(tile, render){
272 | render.raw(':)');
273 | });
274 | ```
275 |
276 | ```javascript
277 | app.layer('notThereLayer', function(tile, render){
278 | render.raw(404);
279 | });
280 | ```
281 |
282 | ### `render.rawFile(fileName)`
283 |
284 | Replies with the specified file
285 |
286 | ```javascript
287 | app.layer('staticLayer', function(tile, render){
288 | render.rawFile('thing.topojson');
289 | });
290 | ```
291 |
292 | ## Caching
293 |
294 | Caching is very important. By default, Tilesplash uses an in-memory cache. You can use redis instead by passing `'redis'` as the second argument when initializing a Tilesplash server.
295 |
296 | There are two ways to implement caching. You can either do it globally or on a layer by layer basis.
297 |
298 | ### app.cache([keyGenerator], ttl)
299 |
300 | Use this to define caching across your entire application
301 |
302 | __`keyGenerator(tile)`__
303 |
304 | keyGenerator is a function that takes a `tile` object as it's only parameter and returns a cache key (__string__)
305 |
306 | If you don't specify a key generator, `app.defaultCacheKeyGenerator` will be used, which returns a key derived from your database connection, tile layer, and tile x, y, and z.
307 |
308 | __`ttl`__
309 |
310 | TTL stands for time-to-live. It's how long tiles will remain in your cache, and it's defined in milliseconds. For most applications, anywhere between one day (86400000) to one week (604800000) should be fine.
311 |
312 | __Example__
313 |
314 | In this example, we have `tile.user_id` available to us and we don't want to show one user tiles belonging to another user. By starting with `app.defaultCacheKeyGenerator(tile)` we get a cache key based on things we already want to cache by (like `x`, `y`, and `z`) and we can then add `user_id` to prevent people from seeing cached tiles unless their `user_id` matches.
315 |
316 | ```javascript
317 | app.cache(function(tile){
318 | return app.defaultCacheKeyGenerator(tile) + ':' + tile.user_id; //cache by tile.user_id as well
319 | }, 1000 * 60 * 60 * 24 * 30); //ttl 30 days
320 | ```
321 |
322 | ### `this.cache([keyGenerator], ttl)`
323 |
324 | Layer-specific caching works identically to global caching as defined above, except that it only applies to one layer and you define it within that layer.
325 |
326 | In this example, slowLayer uses the same key generator as the rest of the app, but specifies a longer TTL.
327 |
328 | ```javascript
329 | app.cache(keyGenerator, 1000 * 60 * 60 * 24); //cache for one day
330 |
331 | app.layer('slowLayer', function(tile, render){
332 | this.cache(1000 * 60 * 60 * 24 * 30); //cache for 30 days
333 |
334 | render.queryFile('slowQuery.sql');
335 | });
336 | ```
337 |
338 | In this example, only slowLayer is cached.
339 |
340 | ```javascript
341 | app.layer('fastLayer', function(tile, render){
342 | render.queryFile('fastQuery.sql');
343 | });
344 |
345 | var userMiddleware = function(req, res, tile, next){
346 | tile.user_id = 1;
347 | next();
348 | };
349 |
350 | app.layer('slowLayer', userMiddleware, function(tile, render){
351 | this.cache(function(tile){
352 | return app.defaultCacheKeyGenerator(tile) + ':' + tile.user_id;
353 | }, 1000 * 60 * 60 * 24); //cache for one day
354 |
355 | render.queryFile('slowQuery.sql');
356 | });
357 | ```
358 |
359 | ## Client
360 | Some in-browser examples of how to use the tiles generated by tilesplash:
361 |
362 | ### .mvt endpoint
363 | - [Mapbox GL third-party source example](https://www.mapbox.com/mapbox-gl-js/example/third-party/)
364 | - [Mapzen's Tangram](https://github.com/tangrams/tangram)
365 | - [Mapzen's d3 vector tiles implementation](http://mapzen.github.io/d3-vector-tiles)
366 |
367 | ### .topojson endpoint
368 | - [D3 + leaflet](http://bl.ocks.org/wboykinm/7393674)
369 | - [Mapzen Tangram](https://github.com/tangrams/tangram)
370 |
--------------------------------------------------------------------------------
/lib/index.js:
--------------------------------------------------------------------------------
1 | var fs = require('fs');
2 | var express = require('express');
3 | var async = require('async');
4 | var topojson = require('topojson');
5 | var pg = require('pg');
6 | var SphericalMercator = require('@mapbox/sphericalmercator');
7 | var Caching = require('caching');
8 | var clone = require('clone');
9 | var connectTimeout = require('connect-timeout');
10 | var mapnik = require('mapnik');
11 | var path = require('path');
12 |
13 | mapnik.register_datasource(path.join(mapnik.settings.paths.input_plugins, 'geojson.input'));
14 |
15 | function setStatementTimeout(client) {
16 | return new Promise(function(resolve, reject){
17 | var statementTimeout = parseInt(process.env.TILESPLASH_STATEMENT_TIMEOUT) || 0;
18 | if (statementTimeout && statementTimeout > 0) {
19 | client.query('SET statement_timeout='+statementTimeout, [], function(err){
20 | if (err) {
21 | reject(err);
22 | } else {
23 | resolve();
24 | }
25 | });
26 | } else {
27 | resolve();
28 | }
29 | });
30 | }
31 |
32 | // allowing tho options object for addGeoJson() - simplification settings, etc.
33 | function stringifyProtobuf(layers, tile, mvtOptions) {
34 | var vtile = new mapnik.VectorTile(tile.z, tile.x, tile.y);
35 | for (var layerName in layers) {
36 | vtile.addGeoJSON(JSON.stringify(layers[layerName]), layerName, mvtOptions);
37 | }
38 | return vtile.getData();
39 | }
40 |
41 | var pgMiddleware = function(_dbOptions){
42 | var dbOptions;
43 | if (typeof _dbOptions === 'string') {
44 | dbOptions = _dbOptions;
45 | } else {
46 | dbOptions = {};
47 | Object.keys(_dbOptions).forEach(function(key){
48 | dbOptions[key] = _dbOptions[key];
49 | });
50 | dbOptions.poolSize = dbOptions.poolSize || 64;
51 | }
52 |
53 | return function(req, res, next){
54 | req.db = {};
55 | req.db.query = function(sql, bindvars, callback){
56 | if (typeof bindvars === 'function') {
57 | callback = bindvars;
58 | bindvars = undefined;
59 | }
60 |
61 | var poolTimedOut = false;
62 | var poolTimeout = setTimeout(function(){
63 | poolTimedOut = true;
64 | callback(new Error('failed to get db connection'));
65 | }, 5000);
66 |
67 | var pool = new pg.Pool(dbOptions);
68 |
69 | pool.connect(function(err, client, done){
70 | if (poolTimedOut) {
71 | done();
72 | return;
73 | }
74 |
75 | clearTimeout(poolTimeout);
76 | if (err) {
77 | callback(err);
78 | done();
79 | return;
80 | }
81 |
82 | res.on('finish', done);
83 | res.on('error', done);
84 | req.on('timeout', done);
85 |
86 | setStatementTimeout(client).then(function(){
87 | client.query(sql, bindvars, function(err, result){
88 | callback(err, result);
89 | done();
90 | });
91 | }).catch(function(err){
92 | callback(err);
93 | done();
94 | });
95 | });
96 | };
97 |
98 | next();
99 | };
100 | };
101 |
102 | /*
103 | Tilesplash
104 | @constructor
105 |
106 | - dbOptions : options passed to the postgres database, required
107 | - cacheType : how to cache tiles, defaults to 'memory', optional
108 | - cacheOptions : options to pass to the caching library , optional
109 | - instrument : probes to instrument various parts of the query cycle, optional. Instrumentation points:
110 | - parseSql: time it takes to parse sql templates into sql strings
111 | - toGeoJson: the time it takes to run the sql and translate it to geojson
112 | - runQuery: the time it takes to run a query (I think)
113 | - gotTile: the round-trip time to get a tile (I think)
114 | */
115 | var Tilesplash = function(dbOptions, cacheType, cacheOptions, instrument){
116 | this.projection = new SphericalMercator({
117 | size: 256
118 | });
119 |
120 | this.dbOptions = dbOptions;
121 | this.dbCacheKey = JSON.stringify(dbOptions);
122 |
123 | this.server = express();
124 | this.instrument = instrument || {};
125 | var timeout = parseInt(process.env.TILESPLASH_REQUEST_TIMEOUT) || 0;
126 |
127 | if (timeout) {
128 | this.server.use(connectTimeout(timeout));
129 | }
130 |
131 | var self = this;
132 | this.server.use(function(req, res, next){
133 | req.on('timeout', function(){
134 | self.log('Request timed out', 'error');
135 | });
136 | next();
137 | });
138 |
139 | this.server.use(pgMiddleware(dbOptions));
140 |
141 | this.cacheOptions = cacheOptions || {};
142 | this._cache = new Caching(cacheType || 'memory', cacheOptions);
143 |
144 | this.logLevel('info');
145 | };
146 |
147 | Tilesplash.prototype.logging = function(callback){
148 | this.log = callback;
149 | };
150 |
151 | Tilesplash.prototype.logLevel = function(logLevel){
152 | var logLevels = ['debug', 'info', 'warn', 'error'];
153 | logLevel = logLevels.indexOf(logLevel);
154 |
155 | this.logging(function(message, level){
156 | var messageLevel = logLevels.indexOf(level);
157 | if (logLevel <= messageLevel) {
158 | console.log('['+level+']', JSON.stringify(message, null, 2));
159 | }
160 | });
161 | };
162 |
163 | Tilesplash.prototype.defaultCacheKeyGenerator = function(tile){
164 | return 'tilesplash_tile:'+this.dbCacheKey+'/'+tile.layer+'/'+tile.z+'/'+tile.x+'/'+tile.y;
165 | };
166 |
167 | Tilesplash.prototype.cache = function(cacheKeyGenerator, ttl){
168 | var self = this;
169 | if (typeof cacheKeyGenerator === 'number') {
170 | ttl = cacheKeyGenerator;
171 | cacheKeyGenerator = undefined;
172 | }
173 | this.defaultCacher = cacheKeyGenerator || function(tile){
174 | return self.defaultCacheKeyGenerator(tile);
175 | };
176 | this.defaultTtl = ttl;
177 | };
178 |
179 | function stringifyTopojson(layers){
180 | layers = clone(layers);
181 | return topojson.topology(layers, {
182 | 'property-transform': function(properties, key, value){
183 | properties[key] = value;
184 | return true;
185 | }
186 | });
187 | }
188 |
189 | Tilesplash.prototype.layer = function(name){
190 | var callbacks = Array.prototype.slice.call(arguments);
191 | callbacks.shift();
192 |
193 | // pop off renderer for future use
194 | var tileRenderer = callbacks.pop();
195 |
196 | var mvtOptions = {};
197 | // if the last argument is an object, use it as mvtOptions
198 | if (typeof callbacks[callbacks.length - 1] === 'object') {
199 | mvtOptions = callbacks.pop()
200 | }
201 |
202 | var self = this;
203 |
204 | // actual route creation
205 | this.server.get('/'+name+'/:z/:x/:y.:ext', function(req, res, throwError){
206 | var render = function(){
207 | render.query.apply(render, arguments);
208 | };
209 | render.query = function(data){
210 | gotTile(data);
211 | };
212 | render.queryFile = function(file, encoding){
213 | fs.readFile(file, encoding || 'utf8', function(err, data){
214 | if (err) {
215 | render.error(err);
216 | } else {
217 | gotTile(data);
218 | }
219 | });
220 | };
221 | render.raw = function(){
222 | res.send.apply(res, arguments);
223 | };
224 | render.rawFile = function(file, encoding){
225 | fs.readFile(file, encoding || 'utf8', function(err, data){
226 | if (err) {
227 | render.error(err);
228 | } else {
229 | render.raw(data);
230 | }
231 | });
232 | };
233 |
234 | render.error = function(msg){
235 | render.raw(500);
236 | throwError( (msg instanceof Error) ? msg : new Error(msg) );
237 | };
238 |
239 | if (req.params.ext != 'topojson' && req.params.ext != 'mvt') {
240 | render.error('unsupported extension '+req.params.ext);
241 | return;
242 | }
243 |
244 | var stringify = (req.params.ext == 'topojson') ? stringifyTopojson : stringifyProtobuf;
245 |
246 | var tile = {};
247 | tile.layer = name;
248 | tile.x = req.params.x*1;
249 | tile.y = req.params.y*1;
250 | tile.z = req.params.z*1;
251 |
252 | render.empty = function(){
253 | render.raw(stringify([], tile, {}));
254 | };
255 |
256 | tile.bounds = self.projection.bbox(req.params.x, req.params.y, req.params.z, false, '900913');
257 | tile.bbox = [
258 | 'ST_SetSRID(',
259 | 'ST_MakeBox2D(',
260 | 'ST_MakePoint(', tile.bounds[0], ', ', tile.bounds[1], '), ',
261 | 'ST_MakePoint(', tile.bounds[2], ', ', tile.bounds[3], ')',
262 | '), ',
263 | '3857',
264 | ')'
265 | ].join('');
266 | tile.bbox_4326 = 'ST_Transform('+tile.bbox+', 4326)';
267 | tile.geom_hash = 'Substr(MD5(ST_AsBinary(the_geom)), 1, 10)';
268 |
269 | self.log('Rendering tile '+tile.layer+'/'+tile.z+'/'+tile.x+'/'+tile.y, 'debug');
270 |
271 | function startTrace() {
272 | var start = process.hrtime();
273 | return function endTrace() {
274 | var diff = process.hrtime(start);
275 |
276 | return (diff[0] * 1e9 + diff[1])/1e6;
277 | };
278 | }
279 |
280 |
281 | var parseSql = function(sql, done){
282 | var trace = startTrace(),
283 | instrument = self.instrument.parseSql || function() {};
284 |
285 | if (typeof sql === 'number') {
286 | instrument(trace());
287 | done(null, sql);
288 | return;
289 | }
290 | if (typeof sql === 'object') {
291 | if (Array.isArray(sql)) {
292 | async.map(sql, function(item, next){
293 | parseSql(item, function(err, out){
294 | next(err, out);
295 | });
296 | }, function(err, results){
297 | instrument(trace());
298 | done(err, results);
299 | });
300 | return;
301 | } else {
302 | var keys = Object.keys(sql);
303 | async.map(keys, function(item, next){
304 | parseSql(sql[item], function(err, out){
305 | next(err, out);
306 | });
307 | }, function(err, results){
308 | var output = {};
309 | keys.forEach(function(d, i){
310 | output[d] = results[i];
311 | });
312 |
313 | instrument(trace());
314 | done(err, output);
315 | });
316 | return;
317 | }
318 | }
319 | if (typeof sql !== 'string') {
320 | done(['Trying to parse non-string SQL', sql]);
321 | return;
322 | }
323 | var templatePattern = /!([0-9a-zA-Z_\-]+)!/g;
324 |
325 | sql = sql.replace(templatePattern, function(match){
326 | match = match.substr(1, match.length-2);
327 | return tile[match];
328 | });
329 | done(null, sql);
330 | };
331 |
332 | var sqlToGeojson = function(sql, done){
333 | var instrument = self.instrument.toGeoJson || function() {},
334 | trace = startTrace();
335 |
336 | if (sql === false || sql === null) {
337 | instrument(trace());
338 | done(null, {});
339 | return;
340 | }
341 | parseSql(sql, function(parsingError, fullsql){
342 | if (parsingError) {
343 | instrument(trace());
344 | done(parsingError);
345 | return;
346 | }
347 | var args = [];
348 | if (typeof fullsql === 'object' && Array.isArray(fullsql)) {
349 | args = fullsql;
350 | fullsql = args.shift();
351 | }
352 | self.log('Running Query', 'debug');
353 | self.log(' SQL: '+fullsql, 'debug');
354 | self.log(' Arguments: '+JSON.stringify(args), 'debug');
355 |
356 |
357 | req.db.query(fullsql, args, function(sqlError, result){
358 | if (sqlError) {
359 | instrument(trace());
360 | done([sqlError, fullsql, args]);
361 | } else {
362 | var geojson = {
363 | "type": "FeatureCollection",
364 | "features": []
365 | };
366 | result.rows.forEach(function(row){
367 | var properties = {};
368 | for (var attribute in row) {
369 | if (attribute !== 'the_geom_geojson') {
370 | properties[attribute] = row[attribute];
371 | }
372 | }
373 | geojson.features.push({
374 | "type": "Feature",
375 | "geometry": JSON.parse(row.the_geom_geojson),
376 | "properties": properties
377 | });
378 | });
379 |
380 | instrument(trace());
381 | done(null, geojson);
382 | }
383 | });
384 | });
385 | };
386 |
387 | var tileContext = {
388 | cache: function(cacheKeyGenerator, ttl){
389 | if (typeof cacheKeyGenerator === 'number') {
390 | ttl = cacheKeyGenerator;
391 | cacheKeyGenerator = undefined;
392 | }
393 | if (typeof cacheKeyGenerator === 'function') {
394 | this._cacher = cacheKeyGenerator;
395 | }
396 | this._ttl = ttl;
397 | }
398 | };
399 |
400 | var runQuery = function(structuredSql, done){
401 | var instrument = self.instrument.runQuery || function() {},
402 | trace = startTrace();
403 |
404 | if (structuredSql === false || structuredSql === null) {
405 | instrument(trace());
406 | done(null, {});
407 | return;
408 | }
409 |
410 | if (typeof structuredSql !== 'object' || Array.isArray(structuredSql)) {
411 | structuredSql = {vectile: structuredSql};
412 | }
413 |
414 | var geojsonLayers = {};
415 | async.forEach(Object.keys(structuredSql), function(layer, next){
416 | sqlToGeojson(structuredSql[layer], function(err, geojson){
417 | if (err) {
418 | next(err);
419 | } else {
420 | geojsonLayers[layer] = geojson;
421 | next();
422 | }
423 | });
424 | }, function(err){
425 | instrument(trace());
426 | if (err) {
427 | done(err);
428 | } else {
429 | done(null, geojsonLayers);
430 | }
431 | });
432 | };
433 |
434 | var gotTile = function(tileOutput){
435 | var instrument = self.instrument.gotTile || function() {},
436 | trace = startTrace();
437 |
438 | parseSql(tileOutput, function(parsingError, structuredSql){
439 | if (parsingError) {
440 | instrument(trace());
441 | render.error(['SQL Parsing Error', parsingError]);
442 | return;
443 | }
444 | var cacher = tileContext._cacher || self.defaultCacher || false;
445 |
446 | if (cacher) {
447 | var cacheKey = cacher(tile);
448 | var ttl;
449 | if (typeof tileContext._ttl === 'number') {
450 | ttl = tileContext._ttl;
451 | } else if (typeof self.defaultTtl === 'number') {
452 | ttl = self.defaultTtl;
453 | } else {
454 | ttl = 0;
455 | }
456 | self._cache(cacheKey, ttl, function(done){
457 | instrument(trace());
458 | self.log('cache miss', 'debug');
459 | runQuery(structuredSql, function(queryError, layers){
460 | done(queryError, layers);
461 | });
462 | }, function(layerError, layers){
463 | instrument(trace());
464 | if (layerError) {
465 | render.error(['layer error', layerError]);
466 | } else {
467 | res.send(stringify(layers, tile, mvtOptions));
468 | }
469 | });
470 | } else {
471 | runQuery(structuredSql, function(queryError, layers){
472 | instrument(trace());
473 | if (queryError) {
474 | render.error(['error running query', queryError]);
475 | } else {
476 | res.send(stringify(layers, tile, mvtOptions));
477 | }
478 | });
479 | }
480 | });
481 | };
482 |
483 | async.eachSeries(callbacks, function(middleware, next){
484 | middleware(req, res, tile, function(err){
485 | next(err);
486 | });
487 | }, function(err){
488 | if (err) {
489 | render.error(['Middleware error', err]);
490 | } else {
491 | tileRenderer.call(tileContext, tile, render);
492 | }
493 | });
494 | });
495 | };
496 |
497 | module.exports = Tilesplash;
498 |
--------------------------------------------------------------------------------