├── .eslintignore
├── .eslintrc
├── .github
└── workflows
│ └── CI.yml
├── .gitignore
├── .jshintrc
├── .nvmrc
├── README.md
├── config_example.js
├── examples
└── server.js
├── index.js
├── lib
├── assets
│ └── blank.png
├── elastic_mapper.js
├── elastic_request.js
├── map_generator.js
└── styles.js
├── package-lock.json
├── package.json
└── test
├── .eslintrc
├── elastic_mapper.js
├── elastic_request.js
├── lib
└── helpers.js
├── map_generator.js
└── styles.js
/.eslintignore:
--------------------------------------------------------------------------------
1 | coverage/**
2 | node_modules/**
3 |
--------------------------------------------------------------------------------
/.eslintrc:
--------------------------------------------------------------------------------
1 | {
2 | "env": {
3 | "node": true,
4 | "mocha": true,
5 | "es6": true
6 | },
7 | "parser": "@babel/eslint-parser",
8 | "parserOptions": {
9 | "requireConfigFile": false,
10 | },
11 | "extends": "airbnb",
12 | "rules": {
13 | "arrow-parens": [2, "as-needed"],
14 | "comma-dangle": [2, "never"],
15 | "consistent-return": [2, { "treatUndefinedAsUnspecified": true }],
16 | "func-names": 0,
17 | "no-underscore-dangle": 0,
18 | "no-void": 0,
19 | "prefer-destructuring": [2, { "object": true, "array": false }],
20 | "quotes": [2, "double"],
21 | "space-in-parens": [2, "always"],
22 | "no-param-reassign": 0,
23 | "prefer-const": [2, { "destructuring": "all" }],
24 | "function-paren-newline": [2, "consistent"]
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/.github/workflows/CI.yml:
--------------------------------------------------------------------------------
1 | name: elasticmaps CI
2 |
3 | on: [push, pull_request]
4 |
5 | jobs:
6 | pre_build:
7 | runs-on: ubuntu-20.04
8 | steps:
9 | - uses: fkirc/skip-duplicate-actions@master
10 | with:
11 | github_token: ${{ github.token }}
12 |
13 | build:
14 | needs: pre_build
15 | runs-on: ubuntu-20.04
16 | steps:
17 |
18 | - uses: actions/checkout@v3
19 |
20 | - name: Use Node.js
21 | uses: actions/setup-node@v3
22 | with:
23 | node-version: 16.x
24 |
25 | - name: Configure sysctl limits
26 | run: |
27 | sudo swapoff -a
28 | sudo sysctl -w vm.swappiness=1
29 | sudo sysctl -w fs.file-max=262144
30 | sudo sysctl -w vm.max_map_count=262144
31 |
32 | - name: Runs Elasticsearch
33 | uses: miyataka/elastic-github-actions/elasticsearch@feature/plugin_support
34 | with:
35 | stack-version: 7.17.1
36 | plugins: analysis-kuromoji
37 |
38 | - name: Elasticsearch is reachable
39 | run: |
40 | curl --verbose --show-error http://localhost:9200
41 |
42 | - run: npm install
43 |
44 | - run: npm run coverage
45 |
46 | notify:
47 | name: Notify Slack
48 | needs: build
49 | if: ${{ success() || failure() }}
50 | runs-on: ubuntu-20.04
51 | steps:
52 | - uses: iRoachie/slack-github-actions@v2.3.2
53 | if: env.SLACK_WEBHOOK_URL != null
54 | env:
55 | SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }}
56 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
57 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | **/.DS_Store
2 | .coveralls.yml
3 | .nyc_output
4 | coverage
5 | node_modules
6 |
7 |
--------------------------------------------------------------------------------
/.jshintrc:
--------------------------------------------------------------------------------
1 | {
2 | "multistr" : true // Tolerate multi-line strings
3 | }
4 |
--------------------------------------------------------------------------------
/.nvmrc:
--------------------------------------------------------------------------------
1 | 16.13.2
2 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Elasticmaps
2 |
3 | [](https://github.com/inaturalist/elasticmaps/actions)
4 | [](https://coveralls.io/r/inaturalist/elasticmaps?branch=main)
5 |
6 | A Node.js map tile server based on node-mapnik and elasticsearch
7 |
8 | Installation
9 | -------
10 | ```
11 | npm install elasticmaps --save
12 | ```
13 |
14 | Usage
15 | -----
16 | ```js
17 | // This is the most basic example. It assumes elasticsearch
18 | // is running on http://localhost:9200, and that there is an index
19 | // named elasticmaps_development which has documents with minimally
20 | // an integer `id` and geo_point `location` field
21 |
22 | const Elasticmaps = require( "elasticmaps" );
23 | const port = Number( process.env.PORT || 4000 );
24 |
25 | const app = Elasticmaps.server( );
26 | // create the tile route
27 | app.get( "/:style/:zoom/:x/:y.:format([a-z\.]+)", Elasticmaps.route );
28 |
29 | app.listen( port, ( ) => {
30 | console.log( "Listening on " + port );
31 | } );
32 | ```
33 |
34 | ----
35 |
36 | ```js
37 | // In this example a custom config object is supplied
38 | // when creating the server. Functions can be provided
39 | // to create custom queries and styles based on the request
40 |
41 | const Elasticmaps = require( "elasticmaps" );
42 | const port = Number( process.env.PORT || 4000 );
43 |
44 | const config = {
45 | environment: "production",
46 | debug: true,
47 | tileSize: 256,
48 | elasticsearch: {
49 | host: "http://localhost:9200",
50 | searchIndex: "points_index",
51 | geoPointField: "location"
52 | },
53 | prepareQuery: req => {
54 | req.elastic_query = ...;
55 | },
56 | prepareStyle: req => {
57 | req.style = ...;
58 | }
59 | };
60 |
61 | const server = Elasticmaps.server( config );
62 | server.listen( port, ( ) => {
63 | console.log( "Listening on " + port );
64 | } );
65 | ```
66 |
--------------------------------------------------------------------------------
/config_example.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | elasticsearch: {
3 | host: "http://localhost:9200",
4 | searchIndex: "points_index",
5 | geoPointField: "location"
6 | },
7 | tileSize: 256,
8 | debug: true
9 | };
10 |
--------------------------------------------------------------------------------
/examples/server.js:
--------------------------------------------------------------------------------
1 | const server = require( "../lib/elastic_mapper" );
2 |
3 | server( { environment: "test", debug: true } );
4 | const port = Number( process.env.PORT || 4000 );
5 |
6 | server.listen( port, ( ) => {
7 | console.log( `Listening on ${port}` ); // eslint-disable-line no-console
8 | } );
9 |
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | const _ = require( "lodash" );
2 | const ElasticMapper = require( "./lib/elastic_mapper" );
3 | const ElasticRequest = require( "./lib/elastic_request" );
4 |
5 | module.exports = _.assignIn( ElasticMapper, {
6 | geohashPrecision: ElasticRequest.geohashPrecision,
7 | geohashAggregation: ElasticRequest.geohashAggregation,
8 | torqueAggregation: ElasticRequest.torqueAggregation
9 | } );
10 |
--------------------------------------------------------------------------------
/lib/assets/blank.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/inaturalist/elasticmaps/16ee71c4a7a97c23c2f5725cc22324536b9f756d/lib/assets/blank.png
--------------------------------------------------------------------------------
/lib/elastic_mapper.js:
--------------------------------------------------------------------------------
1 | const express = require( "express" );
2 | const querystring = require( "querystring" );
3 | const _ = require( "lodash" );
4 | const Geohash = require( "latlon-geohash" );
5 | const MapGenerator = require( "./map_generator" );
6 | const ElasticRequest = require( "./elastic_request" );
7 |
8 | const ElasticMapper = { };
9 | let value = null;
10 |
11 | ElasticMapper.renderMessage = ( res, message, status ) => {
12 | res.set( "Content-Type", "text/html" );
13 | res.status( status ).send( message ).end( );
14 | };
15 |
16 | ElasticMapper.renderError = ( res, error ) => {
17 | ElasticMapper.debug( error );
18 | if ( error.message && error.status ) {
19 | ElasticMapper.renderMessage( res, error.message, error.status );
20 | } else {
21 | ElasticMapper.renderMessage( res, "Error", 500 );
22 | }
23 | };
24 |
25 | ElasticMapper.renderResult = ( req, res, data ) => {
26 | if ( req.params.format === "grid.json" ) {
27 | res.jsonp( data );
28 | } else {
29 | res.writeHead( 200, { "Content-Type": "image/png" } );
30 | res.end( data );
31 | }
32 | };
33 |
34 | ElasticMapper.prepareQuery = async req => {
35 | req.elastic_query = { };
36 | req.elastic_query.sort = { id: "desc" };
37 | req.query.source = { includes: ElasticRequest.defaultMapFields( ) };
38 | req.elastic_query.query = { bool: { } };
39 | switch ( req.params.style ) {
40 | case "points":
41 | req.elastic_query.size = 10000;
42 | break;
43 | case "geotile":
44 | req.geotilegrid = true;
45 | req.includeTotalHits = true;
46 | req.elastic_query.size = 0;
47 | req.elastic_query.aggregations = ElasticRequest.geohashAggregation( req );
48 | break;
49 | case "geohash":
50 | req.elastic_query.size = 0;
51 | if ( req.params.format === "torque.json" ) {
52 | req.elastic_query.aggregations = ElasticRequest.torqueAggregation( req );
53 | } else {
54 | req.elastic_query.aggregations = ElasticRequest.geohashAggregation( req );
55 | }
56 | break;
57 | default:
58 | // eslint-disable-next-line no-case-declarations
59 | const e = new Error( );
60 | e.status = 404;
61 | e.message = `unknown style: ${req.params.style}`;
62 | throw e;
63 | }
64 | };
65 |
66 | ElasticMapper.geotileGridGeojson = hit => {
67 | const parts = hit.key.split( "/" );
68 | MapGenerator.createMercator( );
69 | const bbox = MapGenerator.merc.bbox( parts[1], parts[2], parts[0] );
70 | return {
71 | type: "Feature",
72 | geometry: {
73 | type: "Polygon",
74 | coordinates: [[
75 | [bbox[0], bbox[1]],
76 | [bbox[2], bbox[1]],
77 | [bbox[2], bbox[3]],
78 | [bbox[0], bbox[3]],
79 | [bbox[0], bbox[1]]
80 | ]]
81 | },
82 | properties: {
83 | cellCount: hit.doc_count
84 | }
85 | };
86 | };
87 |
88 | ElasticMapper.geohashGridGeojson = hit => {
89 | const bbox = Geohash.bounds( hit.key );
90 | return {
91 | type: "Feature",
92 | geometry: {
93 | type: "Polygon",
94 | coordinates: [[
95 | [bbox.sw.lon, bbox.sw.lat],
96 | [bbox.sw.lon, bbox.ne.lat],
97 | [bbox.ne.lon, bbox.ne.lat],
98 | [bbox.ne.lon, bbox.sw.lat],
99 | [bbox.sw.lon, bbox.sw.lat]
100 | ]]
101 | },
102 | properties: {
103 | cellCount: hit.doc_count
104 | }
105 | };
106 | };
107 |
108 | ElasticMapper.csvFromResult = ( req, result ) => {
109 | if ( req.params.dataType === "geojson" ) {
110 | return ElasticMapper.polygonCSVFromResult( req, result );
111 | }
112 | let target;
113 | if ( result.aggregations && result.aggregations.zoom1 ) {
114 | target = _.sortBy( result.aggregations.zoom1.buckets, hit => (
115 | hit.geohash ? hit.geohash.hits.hits[0].sort[0] : null
116 | ) );
117 | } else if ( result.hits ) {
118 | target = result.hits.hits;
119 | } else { return []; }
120 | const { geoPointField } = global.config.elasticsearch;
121 | const fieldsToMap = ( req.query.source && req.query.source.includes )
122 | ? _.without( req.query.source.includes, geoPointField )
123 | : [];
124 | fieldsToMap.push( "cellCount" );
125 | const csvData = _.map( target, hit => {
126 | // grids get rendered as polygons
127 | if ( req.geotilegrid && req.params.format !== "grid.json" ) {
128 | return ElasticMapper.geotileGridGeojson( hit );
129 | }
130 | if ( req.geogrid ) {
131 | return ElasticMapper.geohashGridGeojson( hit );
132 | }
133 | const fieldData = hit._source || hit.geohash.hits.hits[0]._source;
134 | fieldData.private_location = !_.isEmpty( fieldData.private_location );
135 | if ( !hit._source && hit.geohash ) {
136 | fieldData.cellCount = hit.geohash.hits.total.value;
137 | }
138 | const properties = { };
139 | _.each( fieldsToMap, f => {
140 | if ( f.match( /\./ ) ) {
141 | const parts = f.split( "." );
142 | if ( fieldData[parts[0]] && fieldData[parts[0]][parts[1]] ) {
143 | value = fieldData[parts[0]][parts[1]];
144 | } else {
145 | value = null;
146 | }
147 | } else {
148 | value = fieldData[f] ? fieldData[f] : null;
149 | }
150 | if ( value === "F" ) { value = false; }
151 | if ( value === "T" ) { value = true; }
152 | properties[f] = value;
153 | } );
154 | let latitude;
155 | let longitude;
156 | if ( req.geotilegrid ) {
157 | const parts = hit.key.split( "/" );
158 | MapGenerator.createMercator( );
159 | const bbox = MapGenerator.merc.bbox( parts[1], parts[2], parts[0] );
160 | latitude = _.mean( [bbox[1], bbox[3]] );
161 | longitude = _.mean( [bbox[0], bbox[2]] );
162 | } else if ( _.isObject( fieldData[geoPointField] ) ) {
163 | latitude = fieldData[geoPointField].lat;
164 | longitude = fieldData[geoPointField].lon;
165 | } else {
166 | const coords = fieldData[geoPointField].split( "," );
167 | latitude = Number( coords[0] );
168 | longitude = Number( coords[1] );
169 | }
170 | if ( req.params.format === "grid.json" ) {
171 | properties.latitude = latitude;
172 | properties.longitude = longitude;
173 | }
174 | return {
175 | type: "Feature",
176 | geometry: {
177 | type: "Point",
178 | coordinates: [longitude, latitude]
179 | },
180 | properties
181 | };
182 | } );
183 | return csvData;
184 | };
185 |
186 | ElasticMapper.polygonCSVFromResult = ( req, result ) => (
187 | _.map( result.hits.hits, hit => (
188 | { id: hit._source.id, geojson: hit._source.geometry_geojson }
189 | ) )
190 | );
191 |
192 | ElasticMapper.route = async ( req, res ) => {
193 | req.startTime = Date.now( );
194 | req.params.zoom = parseInt( req.params.zoom, 10 );
195 | req.params.x = parseInt( req.params.x, 10 );
196 | req.params.y = parseInt( req.params.y, 10 );
197 | if ( req.params.zoom < 0 || req.params.zoom > 21 ) {
198 | ElasticMapper.renderMessage( res, "Invalid zoom", 404 );
199 | return;
200 | }
201 | const zoomDimension = 2 ** req.params.zoom;
202 | if ( req.params.x < 0 || req.params.x >= zoomDimension ) {
203 | ElasticMapper.renderMessage( res, "Invalid x value", 404 );
204 | return;
205 | }
206 | if ( req.params.y < 0 || req.params.y >= zoomDimension ) {
207 | ElasticMapper.renderMessage( res, "Invalid y value", 404 );
208 | return;
209 | }
210 | if ( !_.includes( ["png", "grid.json", "torque.json"], req.params.format ) ) {
211 | ElasticMapper.renderMessage( res, "Invalid format", 404 );
212 | return;
213 | }
214 |
215 | try {
216 | const prepareQuery = global.config.prepareQuery || ElasticMapper.prepareQuery;
217 | await prepareQuery( req );
218 | if ( req.includeTotalHits && req.params.dataType !== "postgis" ) {
219 | const cachedCount = await ElasticRequest.count( req, { }, this );
220 | req.totalHits = cachedCount || 0;
221 | }
222 | if ( req.params.dataType !== "postgis" ) {
223 | ElasticRequest.applyBoundingBoxFilter( req );
224 | const result = await ElasticRequest.search( req, {
225 | track_total_hits: false
226 | } );
227 | if ( req.params.format === "torque.json" ) {
228 | MapGenerator.torqueJson( result, req, res );
229 | return;
230 | }
231 | ElasticMapper.debug( `${result.took}ms :: [${req.bbox}]` );
232 | req.csvData = ElasticMapper.csvFromResult( req, result );
233 | }
234 | if ( global.config.prepareStyle && req.params.format !== "grid.json" ) {
235 | await global.config.prepareStyle( req );
236 | }
237 | const { map, layer } = await MapGenerator.createMapTemplate( req );
238 | await MapGenerator.finishMap( req, res, map, layer, req.csvData );
239 | if ( global.config.beforeSendResult ) {
240 | await global.config.beforeSendResult( req, res );
241 | }
242 | req.endTime = new Date( );
243 | ElasticMapper.renderResult( req, res, req.tileData );
244 | if ( global.config.debug ) {
245 | ElasticMapper.printRequestLog( req );
246 | }
247 | } catch ( e ) {
248 | ElasticMapper.renderError( res, e );
249 | }
250 | };
251 |
252 | ElasticMapper.printRequestLog = req => {
253 | let logText = `[ ${new Date( ).toString( )}] GET /${req.params.style}`
254 | + `/${req.params.zoom}/${req.params.x}`
255 | + `/${req.params.y}.${req.params.format}`;
256 | if ( !_.isEmpty( req.query ) ) {
257 | logText += `?${querystring.stringify( req.query )}`;
258 | }
259 | logText += ` ${req.endTime - req.startTime}ms`;
260 | ElasticMapper.debug( logText );
261 | };
262 |
263 | ElasticMapper.debug = text => {
264 | if ( global.config.debug ) {
265 | console.log( text ); // eslint-disable-line no-console
266 | }
267 | };
268 |
269 | ElasticMapper.server = ( config = { } ) => {
270 | global.config = _.defaults( config, {
271 | environment: config.NODE_ENV || process.env.NODE_ENV || "development",
272 | tileSize: 256,
273 | debug: !( config.debug === false )
274 | } );
275 | global.config.log = global.config.log
276 | || `elasticmaps.${global.config.environment}.log`;
277 | global.config.elasticsearch = _.defaults(
278 | global.config.elasticsearch || { },
279 | {
280 | host: "http://localhost:9200",
281 | searchIndex: `elasticmaps_${global.config.environment}`,
282 | geoPointField: "location"
283 | }
284 | );
285 | // create the server and the map route
286 | const server = express( );
287 | if ( global.config.prepareApp && _.isFunction( global.config.prepareApp ) ) {
288 | global.config.prepareApp( server, config );
289 | }
290 | return server;
291 | };
292 |
293 | module.exports = ElasticMapper;
294 |
--------------------------------------------------------------------------------
/lib/elastic_request.js:
--------------------------------------------------------------------------------
1 | /* eslint no-underscore-dangle: 0 */
2 | /* eslint prefer-destructuring: 0 */
3 |
4 | const _ = require( "lodash" );
5 | const { Client } = require( "@elastic/elasticsearch" );
6 | const Cacheman = require( "recacheman" );
7 | const crypto = require( "crypto" );
8 | const MapGenerator = require( "./map_generator" );
9 |
10 | const ElasticRequest = { esClient: null };
11 | const cache = new Cacheman( );
12 |
13 | const RESERVED_COUNT_TIMEOUT = 10000; // 10s
14 | const RESERVED_COUNT_CHECK_DELAY = 100; // 100ms
15 | const COUNT_CACHE_SECONDS = 60 * 60 * 24; // 1 day
16 |
17 | ElasticRequest.createClient = ( ) => {
18 | if ( ElasticRequest.esClient === null ) {
19 | const clientConfig = { };
20 | if ( global.config.elasticsearch.hosts ) {
21 | clientConfig.node = _.isArray( global.config.elasticsearch.hosts )
22 | ? global.config.elasticsearch.hosts
23 | : global.config.elasticsearch.hosts.split( " " );
24 | } else {
25 | clientConfig.node = global.config.elasticsearch.host;
26 | }
27 | clientConfig.nodeSelector = "random";
28 | clientConfig.requestTimeout = 60000;
29 | clientConfig.maxRetries = 1;
30 | ElasticRequest.esClient = new Client( clientConfig );
31 | }
32 | return ElasticRequest.esClient;
33 | };
34 |
35 | ElasticRequest.count = async function ( req, options = { } ) {
36 | const query = { ...req.elastic_query };
37 | delete query.aggregations;
38 | query.size = 0;
39 | req.elastic_client ||= ElasticRequest.createClient( );
40 | const queryHash = crypto
41 | .createHash( "md5" )
42 | .update( JSON.stringify( query ), "utf8" )
43 | .digest( "hex" );
44 | const cacheKey = `elasticmaps-${queryHash}`;
45 |
46 | const countMethod = async ( ) => {
47 | const response = await req.elastic_client.search( {
48 | preference: global.config.elasticsearch.preference,
49 | index: req.elastic_index || global.config.elasticsearch.searchIndex,
50 | body: query,
51 | track_total_hits: true,
52 | ...options
53 | } );
54 | return response.hits.total.value;
55 | };
56 |
57 | if ( req.redisCacheClient ) {
58 | return ElasticRequest.reservedCount( req, cacheKey, countMethod );
59 | }
60 |
61 | const cachedCount = await cache.get( cacheKey );
62 | if ( cachedCount ) {
63 | return cachedCount;
64 | }
65 | const response = await countMethod( );
66 | await cache.set( cacheKey, response, COUNT_CACHE_SECONDS );
67 | return response;
68 | };
69 |
70 | ElasticRequest.reservedCount = async function (
71 | req, cacheKey, countMethod, checkReservations = true
72 | ) {
73 | const reservationCacheKey = `${cacheKey}:reservation`;
74 | const cacheCountMethod = async ( ) => {
75 | // mark the count as reserved
76 | await req.redisCacheClient.set( reservationCacheKey, "reserved", { EX: RESERVED_COUNT_TIMEOUT / 1000 } );
77 | // run the count query
78 | const countResult = await countMethod( );
79 | // cache the count
80 | await req.redisCacheClient.set( cacheKey, countResult, { EX: COUNT_CACHE_SECONDS } );
81 | // delete the count reservation
82 | await req.redisCacheClient.del( reservationCacheKey );
83 | return countResult;
84 | };
85 |
86 | if ( checkReservations ) {
87 | const retryCountMethod = async ( ) => (
88 | ElasticRequest.reservedCount( req, cacheKey, countMethod, false )
89 | );
90 | // check to see if the count query is reserved (being run) by another process
91 | return ElasticRequest.waitForReservedCount(
92 | req, reservationCacheKey, cacheCountMethod, retryCountMethod
93 | );
94 | }
95 |
96 | // check if the count is cached
97 | const cachedCount = await req.redisCacheClient.get( cacheKey );
98 | if ( cachedCount ) {
99 | return Number( cachedCount );
100 | }
101 | // otherwise perform the count process
102 | return cacheCountMethod( );
103 | };
104 |
105 | ElasticRequest.waitForReservedCount = async function (
106 | req, reservationCacheKey, countMethod, retryCountMethod, startTime = Date.now( )
107 | ) {
108 | if ( Date.now( ) - startTime >= RESERVED_COUNT_TIMEOUT ) {
109 | // reservation is too old, so run the count
110 | return retryCountMethod( );
111 | }
112 | const reservationExists = await req.redisCacheClient.get( reservationCacheKey );
113 | if ( reservationExists ) {
114 | // there is a reservation for the count, so wait a bit and check again
115 | return ElasticRequest.waitForReservedCountDelay(
116 | req, reservationCacheKey, countMethod, retryCountMethod, startTime
117 | );
118 | }
119 | // there is no reservation, so run the count
120 | return retryCountMethod( );
121 | };
122 |
123 | ElasticRequest.waitForReservedCountDelay = function (
124 | req, reservationCacheKey, countMethod, retryCountMethod, startTime
125 | ) {
126 | return new Promise( ( resolve, reject ) => {
127 | // wait RESERVED_COUNT_CHECK_DELAY ms to check again for a reservation
128 | setTimeout( ( ) => {
129 | ElasticRequest.waitForReservedCount(
130 | req, reservationCacheKey, countMethod, retryCountMethod, startTime
131 | ).then( resolve ).catch( reject );
132 | }, RESERVED_COUNT_CHECK_DELAY );
133 | } );
134 | };
135 |
136 | ElasticRequest.search = async ( req, options = { } ) => {
137 | const query = { ...req.elastic_query };
138 | req.elastic_client ||= ElasticRequest.createClient( );
139 | return req.elastic_client.search( {
140 | preference: global.config.elasticsearch.preference,
141 | index: req.elastic_index || global.config.elasticsearch.searchIndex,
142 | body: query,
143 | ...options
144 | } );
145 | };
146 |
147 | ElasticRequest.expandBoxForSmoothEdges = ( req, bbox ) => {
148 | const qbbox = bbox;
149 | const height = Math.abs( qbbox[2] - qbbox[0] );
150 | const width = Math.abs( qbbox[3] - qbbox[1] );
151 | const factor = ( req && req.params && req.params.style === "grid" ) ? 0.02 : 0.07;
152 | qbbox[0] -= ( height * factor );
153 | qbbox[2] += ( height * factor );
154 | qbbox[1] -= ( width * factor );
155 | qbbox[3] += ( width * factor );
156 | if ( qbbox[0] < -180 ) { qbbox[0] = -180; }
157 | if ( qbbox[1] < -90 ) { qbbox[1] = -90; }
158 | if ( qbbox[2] > 180 ) { qbbox[2] = 180; }
159 | if ( qbbox[3] > 90 ) { qbbox[3] = 90; }
160 | return qbbox;
161 | };
162 |
163 | ElasticRequest.boundingBoxFilter = ( req, bbox, smoothing ) => {
164 | let qbbox = bbox;
165 | if ( smoothing !== false ) {
166 | qbbox = ElasticRequest.expandBoxForSmoothEdges( req, qbbox );
167 | }
168 | if ( qbbox[2] < qbbox[0] ) {
169 | // the envelope crosses the dateline. Unfortunately, elasticsearch
170 | // doesn't handle this well and we need to split the envelope at
171 | // the dateline and do an OR query
172 | const left = _.clone( qbbox );
173 | const right = _.clone( qbbox );
174 | left[2] = 180;
175 | right[0] = -180;
176 | return {
177 | bool: {
178 | should: [
179 | ElasticRequest.boundingBoxFilter( req, left, false ),
180 | ElasticRequest.boundingBoxFilter( req, right, false )
181 | ]
182 | }
183 | };
184 | }
185 |
186 | const field = global.config.elasticsearch.geoPointField;
187 | const boundingBox = { };
188 | boundingBox[field] = {
189 | bottom_left: [qbbox[0], qbbox[1]],
190 | top_right: [qbbox[2], qbbox[3]]
191 | };
192 | boundingBox.type = "indexed";
193 | return { geo_bounding_box: boundingBox };
194 | };
195 |
196 | ElasticRequest.geohashPrecision = ( zoom, offset ) => {
197 | let precision = 3;
198 | if ( zoom >= 3 ) { precision = 4; }
199 | if ( zoom >= 6 ) { precision = 5; }
200 | if ( zoom >= 8 ) { precision = 6; }
201 | if ( zoom >= 11 ) { precision = 7; }
202 | if ( zoom >= 12 ) { precision = 8; }
203 | if ( zoom >= 13 ) { precision = 9; }
204 | if ( zoom >= 15 ) { precision = 10; }
205 | if ( zoom >= 16 ) { precision = 12; }
206 | if ( offset && offset > 0 && ( precision - offset ) > 0 ) {
207 | precision -= offset;
208 | }
209 | return precision;
210 | };
211 |
212 | ElasticRequest.geotileGridPrecision = ( zoom, offset ) => {
213 | let precision = zoom + 5;
214 | if ( offset && ( precision - offset ) > 0 ) {
215 | precision -= offset;
216 | }
217 | return precision;
218 | };
219 |
220 | ElasticRequest.defaultMapFields = ( ) => (
221 | ["id", global.config.elasticsearch.geoPointField]
222 | );
223 |
224 | ElasticRequest.defaultMapQuery = ( ) => (
225 | { match_all: { } }
226 | );
227 |
228 | ElasticRequest.applyBoundingBoxFilter = req => {
229 | if ( req.params.dataType === "postgis" ) {
230 | return;
231 | }
232 | MapGenerator.createMercator( );
233 | req.bbox = MapGenerator.merc.convert( MapGenerator.bboxFromParams( req ) );
234 | const smoothing = req.params.format !== "torque.json";
235 | const bboxFilter = ElasticRequest.boundingBoxFilter( req, req.bbox, smoothing );
236 | if ( !req.elastic_query ) { req.elastic_query = { }; }
237 | if ( !req.elastic_query.query ) { req.elastic_query.query = { }; }
238 | if ( !req.elastic_query.query.bool ) { req.elastic_query.query.bool = { }; }
239 | if ( !req.elastic_query.query.bool.filter ) { req.elastic_query.query.bool.filter = []; }
240 | req.elastic_query.query.bool.filter.push( bboxFilter );
241 | };
242 |
243 | ElasticRequest.geohashAggregation = req => {
244 | let agg;
245 | if ( req.geotile || req.geotilegrid ) {
246 | // use the geotile aggregation added in Elasticsearch 7
247 | agg = {
248 | zoom1: {
249 | geotile_grid: {
250 | field: global.config.elasticsearch.geoPointField,
251 | size: 10000,
252 | precision: ElasticRequest.geotileGridPrecision(
253 | req.params.zoom, req.query ? Number( req.query.precision_offset ) : null
254 | )
255 | }
256 | }
257 | };
258 | } else {
259 | agg = {
260 | zoom1: {
261 | geohash_grid: {
262 | field: global.config.elasticsearch.geoPointField,
263 | size: 30000,
264 | precision: ElasticRequest.geohashPrecision(
265 | req.params.zoom, req.query ? Number( req.query.precision_offset ) : null
266 | )
267 | }
268 | }
269 | };
270 | }
271 | // UTFGrids always need topHits to get metadata for each cell.
272 | // PNG versions of grid-based styles do not need topHits
273 | if ( req.params.format === "grid.json" || ( !req.skipTopHits && !req.geogrid && !req.geotilegrid ) ) {
274 | agg.zoom1.aggs = {
275 | geohash: {
276 | top_hits: {
277 | sort: { id: { order: "desc" } },
278 | _source: ( ( req.query && req.query.source )
279 | ? req.query.source : false ),
280 | size: 1
281 | }
282 | }
283 | };
284 | }
285 | return agg;
286 | };
287 |
288 | ElasticRequest.torqueAggregation = req => {
289 | const interval = req.query.interval === "weekly" ? "week" : "month";
290 | return {
291 | zoom1: {
292 | geohash_grid: {
293 | field: global.config.elasticsearch.geoPointField,
294 | size: 30000,
295 | precision: ElasticRequest.geohashPrecision(
296 | req.params.zoom, req.query ? Number( req.query.precision_offset ) : null
297 | )
298 | },
299 | aggs: {
300 | histogram: {
301 | terms: {
302 | field: `observed_on_details.${interval}`,
303 | size: 100
304 | },
305 | aggs: {
306 | geohash: {
307 | top_hits: {
308 | sort: { id: { order: "desc" } },
309 | _source: ( ( req.query && req.query.source )
310 | ? req.query.source : false ),
311 | size: 1
312 | }
313 | }
314 | }
315 | }
316 | }
317 | }
318 | };
319 | };
320 |
321 | module.exports = ElasticRequest;
322 |
--------------------------------------------------------------------------------
/lib/map_generator.js:
--------------------------------------------------------------------------------
1 | /* eslint no-underscore-dangle: 0 */
2 |
3 | const mapnik = require( "mapnik" );
4 | const _ = require( "lodash" );
5 | const fs = require( "fs" );
6 | const path = require( "path" );
7 | const flatten = require( "flat" );
8 | const { promisify } = require( "util" );
9 | const SphericalMercator = require( "@mapbox/sphericalmercator" );
10 | const Styles = require( "./styles" );
11 |
12 | // register shapefile plugin
13 | if ( mapnik.register_default_input_plugins ) {
14 | mapnik.register_default_input_plugins( );
15 | }
16 |
17 | const proj4 = "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 "
18 | + "+y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over";
19 |
20 | const MapGenerator = { merc: null };
21 |
22 | MapGenerator.blankImage = fs.readFileSync( path.join( __dirname, "assets/blank.png" ) );
23 |
24 | MapGenerator.createMercator = ( ) => {
25 | if ( MapGenerator.merc === null ) {
26 | MapGenerator.merc = new SphericalMercator( { size: global.config.tileSize } );
27 | }
28 | };
29 |
30 | MapGenerator.bboxFromParams = req => {
31 | MapGenerator.createMercator( );
32 | let zoom = parseInt( req.params.zoom, 10 );
33 | if ( req.largeTiles ) { zoom -= 1; }
34 | return MapGenerator.merc.bbox(
35 | parseInt( req.params.x, 10 ),
36 | parseInt( req.params.y, 10 ),
37 | zoom,
38 | false,
39 | "900913"
40 | );
41 | };
42 |
43 | MapGenerator.createLayer = ( ) => {
44 | const layer = new mapnik.Layer( "tile", "+init=epsg:4326" );
45 | layer.styles = ["style"];
46 | return layer;
47 | };
48 |
49 | MapGenerator.postgisDatasource = req => {
50 | let zoom = parseInt( req.params.zoom, 10 );
51 | if ( req.largeTiles ) { zoom -= 1; }
52 | const datasourceConfig = {
53 | ...global.config.database,
54 | type: "postgis",
55 | table: req.postgis.query,
56 | simplify_geometries: true,
57 | extent: MapGenerator.merc.bbox(
58 | parseInt( req.params.x, 10 ),
59 | parseInt( req.params.y, 10 ),
60 | zoom
61 | )
62 | };
63 | return new mapnik.Datasource( datasourceConfig );
64 | };
65 |
66 | MapGenerator.geojsonDatasource = features => {
67 | if ( _.isEmpty( features ) ) { return null; }
68 | const datasourceConfig = {
69 | type: "geojson",
70 | inline: JSON.stringify( {
71 | type: "FeatureCollection",
72 | features
73 | } )
74 | };
75 | return new mapnik.Datasource( datasourceConfig );
76 | };
77 |
78 | /* eslint-disable no-param-reassign */
79 | MapGenerator.finishMap = async ( req, res, map, layer, features ) => {
80 | if ( features && features.length === 0 && req.params.format !== "grid.json" ) {
81 | req.tileData = MapGenerator.blankImage;
82 | return;
83 | }
84 | let fields;
85 | if ( req.params.dataType === "postgis" ) {
86 | layer.datasource = MapGenerator.postgisDatasource( req );
87 | } else {
88 | const memDS = MapGenerator.geojsonDatasource( features );
89 | if ( memDS ) { layer.datasource = memDS; }
90 | }
91 | map.add_layer( layer );
92 | let { tileSize } = global.config;
93 | if ( req.largeTiles ) { tileSize *= 2; }
94 | const mapRender = promisify( map.render.bind( map ) );
95 | if ( req.params.format === "grid.json" ) {
96 | fields = _.without( req.query.source.includes,
97 | global.config.elasticsearch.geoPointField );
98 | // geohash aggregations will have cellCount
99 | if ( req.elastic_query.aggregations && req.elastic_query.aggregations.zoom1 ) {
100 | fields.push( "cellCount" );
101 | }
102 | fields = fields.concat( ["latitude", "longitude"] );
103 |
104 | const options = { };
105 | options.layer = "tile";
106 | options.fields = fields;
107 | options.headers = { "Content-Type": "application/json" };
108 | const im = new mapnik.Grid( tileSize / 2, tileSize / 2, { key: "id" } );
109 | const img = await mapRender( im, options );
110 | req.tileData = img.encodeSync( );
111 | return;
112 | }
113 | const im = new mapnik.Image( tileSize, tileSize );
114 | const img = await mapRender( im, { scale: 2 } );
115 | req.tileData = img.encodeSync( );
116 | };
117 | /* eslint-enable no-param-reassign */
118 |
119 | MapGenerator.basicEscape = text => {
120 | // remove ', \n, \r, \t
121 | // turn " into '
122 | const replaced = text.replace( /'/g, "" )
123 | .replace( /"/g, "'" )
124 | .replace( /(\\n|\\r|\\t)/g, " " )
125 | .replace( /(\\)/g, "/" );
126 | return `"${replaced}"`;
127 | };
128 |
129 | MapGenerator.mapXML = specificStyle => (
130 | `
131 | `
134 | );
135 |
136 | MapGenerator.createMapTemplate = async req => {
137 | req.style = req.style || Styles.points( );
138 | let { tileSize } = global.config;
139 | if ( req.largeTiles ) { tileSize *= 2; }
140 | if ( req.params.format === "grid.json" ) {
141 | tileSize /= 2;
142 | }
143 | const map = new mapnik.Map( tileSize, tileSize, proj4 );
144 | const bbox = MapGenerator.bboxFromParams( req );
145 | const layer = MapGenerator.createLayer( );
146 | const xml = MapGenerator.mapXML( req.style );
147 | map.extent = bbox;
148 | const fromString = promisify( map.fromString.bind( map ) );
149 | const mapFromString = await fromString( xml, { strict: true, base: "./" } );
150 | MapGenerator.createMercator( );
151 | return {
152 | map: mapFromString,
153 | layer
154 | };
155 | };
156 |
157 | MapGenerator.torqueJson = ( queryResult, req, res ) => {
158 | const [minlon, minlat, maxlon, maxlat] = req.bbox;
159 | const mercMin = MapGenerator.merc.px( [minlon, minlat], req.params.zoom );
160 | const mercMax = MapGenerator.merc.px( [maxlon, maxlat], req.params.zoom );
161 | const latdiff = Math.abs( mercMax[1] - mercMin[1] );
162 | const londiff = Math.abs( mercMax[0] - mercMin[0] );
163 | const returnArray = [];
164 | const filteredBuckets = _.filter( queryResult.aggregations.zoom1.buckets,
165 | b => !_.isEmpty( b.histogram.buckets ) );
166 | _.each( filteredBuckets, b => {
167 | const vals = [];
168 | const dates = [];
169 | let hashInfo;
170 | _.each( b.histogram.buckets, hb => {
171 | if ( !hashInfo ) {
172 | hashInfo = hb.geohash.hits.hits[0]._source.location.split( "," );
173 | }
174 | const doc = hb.geohash.hits.hits[0]._source;
175 | vals.push( { value: hb.doc_count, ...flatten( doc ) } );
176 | dates.push( hb.key - 1 );
177 | } );
178 | const mercCoords = MapGenerator.merc.px( [hashInfo[1], hashInfo[0]], req.params.zoom );
179 | const torqueX = Math.floor( ( Math.abs( mercCoords[0] - mercMin[0] ) / latdiff ) * 255 );
180 | const torqueY = Math.floor( ( Math.abs( mercCoords[1] - mercMin[1] ) / londiff ) * 255 );
181 | returnArray.push( {
182 | x__uint8: torqueX,
183 | y__uint8: torqueY,
184 | vals__uint8: vals,
185 | dates__uint16: dates
186 | } );
187 | } );
188 | res.set( "Content-Type", "text/html" ).status( 200 ).json( returnArray ).end( );
189 | };
190 |
191 | module.exports = MapGenerator;
192 |
--------------------------------------------------------------------------------
/lib/styles.js:
--------------------------------------------------------------------------------
1 | const Styles = { };
2 |
3 | Styles.points = ( ) => (
4 | `
5 | `
13 | );
14 |
15 | module.exports = Styles;
16 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "elasticmaps",
3 | "version": "5.3.0",
4 | "description": "Map tileserver based on node-mapnik and elasticsearch",
5 | "main": "index.js",
6 | "scripts": {
7 | "test": "mocha --recursive",
8 | "coverage": "nyc --reporter=lcov --reporter=text-summary npm run test"
9 | },
10 | "repository": {
11 | "type": "git",
12 | "url": "https://github.com/inaturalist/elasticmaps.git"
13 | },
14 | "keywords": [
15 | "elasticmaps",
16 | "elasticsearch",
17 | "tileserver",
18 | "maps"
19 | ],
20 | "author": "Patrick Leary ",
21 | "license": "MIT",
22 | "dependencies": {
23 | "@mapbox/sphericalmercator": "^1.2.0",
24 | "recacheman": "^2.2.4",
25 | "del": "^6.0.0",
26 | "@elastic/elasticsearch": "^8.7.0",
27 | "es6-promise-pool": "^2.5.0",
28 | "express": "^4.17.2",
29 | "flat": "^5.0.2",
30 | "handlebars": "^4.7.7",
31 | "latlon-geohash": "^1.1.0",
32 | "lodash": "^4.17.21",
33 | "mapnik": "^4.5.9",
34 | "step": "^1.0.0"
35 | },
36 | "devDependencies": {
37 | "@babel/core": "^7.16.7",
38 | "@babel/eslint-parser": "^7.16.5",
39 | "chai": "^4.3.4",
40 | "chai-as-promised": "^7.1.1",
41 | "eslint": "^8.6.0",
42 | "eslint-config-airbnb": "^19.0.4",
43 | "eslint-plugin-import": "^2.25.4",
44 | "eslint-plugin-jsx-a11y": "^6.5.1",
45 | "eslint-plugin-react": "^7.28.0",
46 | "mocha": "^10.0.0",
47 | "nyc": "*",
48 | "supertest": "^6.2.0"
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/test/.eslintrc:
--------------------------------------------------------------------------------
1 | {
2 | "rules": {
3 | "no-unused-expressions": 0
4 | }
5 | }
6 |
--------------------------------------------------------------------------------
/test/elastic_mapper.js:
--------------------------------------------------------------------------------
1 | const { expect } = require( "chai" );
2 | const request = require( "supertest" );
3 | const _ = require( "lodash" );
4 | const Mapper = require( "../lib/elastic_mapper" );
5 | const helpers = require( "./lib/helpers" );
6 |
7 | let app;
8 |
9 | describe( "ElasticMapper", ( ) => {
10 | before( async function ( ) {
11 | this.timeout( 10000 );
12 | app = Mapper.server( helpers.testConfig( ) );
13 | app.get( "/:style/:zoom/:x/:y.:format([a-z.]+)", Mapper.route );
14 | await helpers.rebuildTestIndex( );
15 | } );
16 |
17 | after( async ( ) => {
18 | await helpers.deleteTestIndex( );
19 | } );
20 |
21 | describe( "routes", ( ) => {
22 | it( "only knows one route", done => {
23 | request( app ).get( "/" )
24 | .expect( res => {
25 | expect( res.text ).to.include( "Cannot GET /" );
26 | } ).expect( 404, done );
27 | } );
28 |
29 | it( "allows new routes to be created", done => {
30 | app.get( "/", ( req, res ) => {
31 | res.send( "success" ).status( 200 ).end( );
32 | } );
33 | request( app ).get( "/" ).expect( 200 )
34 | .expect( "success", done );
35 | } );
36 |
37 | it( "accepts parameters", done => {
38 | request( app ).get( "/points/1/0/0.png?param=test" ).expect( 200 )
39 | .expect( "content-type", "image/png", done );
40 | } );
41 | } );
42 |
43 | describe( "validation", ( ) => {
44 | it( "accepts the .png format", done => {
45 | request( app ).get( "/points/1/0/0.png" ).expect( 200 )
46 | .expect( "content-type", "image/png", done );
47 | } );
48 |
49 | it( "accepts the .grid.json format", done => {
50 | request( app ).get( "/points/1/1/1.grid.json" ).expect( 200 )
51 | .expect( "content-type", "application/json; charset=utf-8", done );
52 | } );
53 |
54 | it( "errors on all other formats format", done => {
55 | request( app ).get( "/points/1/0/0.html" ).expect( 404 )
56 | .expect( "Invalid format", done );
57 | } );
58 |
59 | it( "returns an error for an unknown style", done => {
60 | request( app ).get( "/nonsense/1/0/0.png" ).expect( 404 )
61 | .expect( "unknown style: nonsense", done );
62 | } );
63 |
64 | it( "zoom must be 0 or above", done => {
65 | request( app ).get( "/points/-1/0/0.png" ).expect( 404 )
66 | .expect( "Invalid zoom", done );
67 | } );
68 |
69 | it( "zoom must be 21 or below", done => {
70 | request( app ).get( "/points/22/0/0.png" ).expect( 404 )
71 | .expect( "Invalid zoom", done );
72 | } );
73 |
74 | it( "x must be 0 or above", done => {
75 | request( app ).get( "/points/5/-1/0.png" ).expect( 404 )
76 | .expect( "Invalid x value", done );
77 | } );
78 |
79 | it( "x must be within range", done => {
80 | request( app ).get( "/points/5/32/0.png" ).expect( 404 )
81 | .expect( "Invalid x value", done );
82 | } );
83 |
84 | it( "y must be 0 or above", done => {
85 | request( app ).get( "/points/5/0/-1.png" ).expect( 404 )
86 | .expect( "Invalid y value", done );
87 | } );
88 |
89 | it( "y must be within range", done => {
90 | request( app ).get( "/points/5/0/32.png" ).expect( 404 )
91 | .expect( "Invalid y value", done );
92 | } );
93 |
94 | it( "y must be within range", done => {
95 | request( app ).get( "/points/5/0/32.png" ).expect( 404 )
96 | .expect( "Invalid y value", done );
97 | } );
98 | } );
99 |
100 | describe( "points", ( ) => {
101 | it( "renders .png", done => {
102 | request( app ).get( "/points/1/1/1.png" ).expect( 200 )
103 | .expect( "content-type", "image/png", done );
104 | } );
105 |
106 | it( "renders .grid.json", done => {
107 | request( app ).get( "/points/1/0/0.grid.json" ).expect( 200 )
108 | .expect( "content-type", "application/json; charset=utf-8", done );
109 | } );
110 |
111 | it( "errors on all other formats format", done => {
112 | request( app ).get( "/points/1/0/0.html" ).expect( 404 )
113 | .expect( "Invalid format", done );
114 | } );
115 | } );
116 |
117 | describe( "geohash", ( ) => {
118 | it( "renders .png", done => {
119 | request( app ).get( "/geohash/1/0/0.png" ).expect( 200 )
120 | .expect( "content-type", "image/png", done );
121 | } );
122 |
123 | it( "renders .grid.json", done => {
124 | request( app ).get( "/geohash/1/1/0.grid.json" ).expect( 200 )
125 | .expect( "content-type", "application/json; charset=utf-8", done );
126 | } );
127 |
128 | it( "renders .torque.json", done => {
129 | request( app ).get( "/geohash/1/0/0.torque.json" )
130 | .expect( res => {
131 | expect( res.text ).to.include( "x__uint8" );
132 | expect( res.text ).to.include( "y__uint8" );
133 | expect( res.text ).to.include( "vals__uint8" );
134 | } )
135 | .expect( 200 )
136 | .expect( "content-type", "text/html; charset=utf-8", done );
137 | } );
138 |
139 | it( "errors on all other formats format", done => {
140 | request( app ).get( "/geohash/1/0/0.html" ).expect( 404 )
141 | .expect( "Invalid format", done );
142 | } );
143 | } );
144 |
145 | describe( "geotile", ( ) => {
146 | it( "renders .png", done => {
147 | request( app ).get( "/geotile/1/0/0.png" ).expect( 200 )
148 | .expect( "content-type", "image/png", done );
149 | } );
150 |
151 | it( "renders .grid.json", done => {
152 | request( app ).get( "/geotile/1/1/0.grid.json" ).expect( 200 )
153 | .expect( "content-type", "application/json; charset=utf-8", done );
154 | } );
155 |
156 | it( "errors on all other formats format", done => {
157 | request( app ).get( "/geotile/1/0/0.html" ).expect( 404 )
158 | .expect( "Invalid format", done );
159 | } );
160 | } );
161 |
162 | describe( "prepareStyle", ( ) => {
163 | it( "renders errors", done => {
164 | app = Mapper.server( _.assignIn( helpers.testConfig( ), {
165 | prepareStyle: ( ) => {
166 | const e = new Error( );
167 | e.status = 501;
168 | e.message = "fail";
169 | throw e;
170 | }
171 | } ) );
172 | app.get( "/:style/:zoom/:x/:y.:format([a-z.]+)", Mapper.route );
173 | request( app ).get( "/points/1/0/0.png" ).expect( 501 )
174 | .expect( "fail", done );
175 | } );
176 |
177 | it( "errors on bad styles", done => {
178 | app = Mapper.server( _.assignIn( helpers.testConfig( ), {
179 | prepareStyle: req => {
180 | req.style = "nonsense";
181 | }
182 | } ) );
183 | app.get( "/:style/:zoom/:x/:y.:format([a-z.]+)", Mapper.route );
184 | request( app ).get( "/points/1/0/0.png" ).expect( 500 )
185 | .expect( "Error", done );
186 | } );
187 | } );
188 |
189 | describe( "renderError", ( ) => {
190 | it( "defaults to 500 Error", done => {
191 | app = Mapper.server( _.assignIn( helpers.testConfig( ), {
192 | prepareQuery: ( ) => {
193 | throw new Error( );
194 | }
195 | } ) );
196 | app.get( "/:style/:zoom/:x/:y.:format([a-z.]+)", Mapper.route );
197 | request( app ).get( "/points/1/0/0.png" ).expect( 500 )
198 | .expect( "Error", done );
199 | } );
200 | } );
201 |
202 | describe( "defaults", ( ) => {
203 | it( "creates a default config", ( ) => {
204 | app = Mapper.server( );
205 | expect( global.config.environment ).to.eql( "development" );
206 | expect( global.config.tileSize ).to.eql( 256 );
207 | expect( global.config.debug ).to.eql( true );
208 | } );
209 | } );
210 | } );
211 |
--------------------------------------------------------------------------------
/test/elastic_request.js:
--------------------------------------------------------------------------------
1 | /* eslint no-underscore-dangle: 0 */
2 |
3 | const chai = require( "chai" );
4 | const chaiAsPromised = require( "chai-as-promised" );
5 | const _ = require( "lodash" );
6 | const ElasticRequest = require( "../lib/elastic_request" );
7 |
8 | const { expect } = chai;
9 | chai.use( chaiAsPromised );
10 |
11 | describe( "ElasticRequest", ( ) => {
12 | describe( "search", ( ) => {
13 | it( "returns an error with a malformed query", async ( ) => {
14 | await expect( ElasticRequest.search( { made: "up" } ) ).to.be
15 | .rejectedWith( Error, "index_not_found_exception" );
16 | } );
17 | } );
18 |
19 | describe( "boundingBoxFilter", ( ) => {
20 | it( "enlarges the boundary for better tile edges", ( ) => {
21 | expect( ElasticRequest.boundingBoxFilter( { }, [0, 0, 1, 1], true ) ).to.eql( {
22 | geo_bounding_box: {
23 | location: {
24 | bottom_left: [-0.07, -0.07],
25 | top_right: [1.07, 1.07]
26 | },
27 | type: "indexed"
28 | }
29 | } );
30 | } );
31 |
32 | it( "can skip smoothing", ( ) => {
33 | expect( ElasticRequest.boundingBoxFilter( { }, [0, 0, 1, 1], false ) ).to.eql( {
34 | geo_bounding_box: {
35 | location: {
36 | bottom_left: [0, 0],
37 | top_right: [1, 1]
38 | },
39 | type: "indexed"
40 | }
41 | } );
42 | } );
43 |
44 | it( "creates a conditional query for dateline wrapping bboxes", ( ) => {
45 | expect( ElasticRequest.boundingBoxFilter( { }, [179, 1, -179, 2], false ) ).to.eql( {
46 | bool: {
47 | should: [
48 | {
49 | geo_bounding_box: {
50 | location: {
51 | bottom_left: [179, 1],
52 | top_right: [180, 2]
53 | },
54 | type: "indexed"
55 | }
56 | },
57 | {
58 | geo_bounding_box: {
59 | location: {
60 | bottom_left: [-180, 1],
61 | top_right: [-179, 2]
62 | },
63 | type: "indexed"
64 | }
65 | }
66 | ]
67 | }
68 | } );
69 | } );
70 | } );
71 |
72 | describe( "applyBoundingBoxFilter", ( ) => {
73 | it( "can add to a prefiltered query", ( ) => {
74 | const req = {
75 | params: {
76 | x: 1,
77 | y: 1,
78 | zoom: 1
79 | },
80 | elastic_query: {
81 | query: {
82 | bool: {
83 | filter: []
84 | }
85 | }
86 | }
87 | };
88 | expect( req.elastic_query.query.bool.filter.length ).to.eql( 0 );
89 | ElasticRequest.applyBoundingBoxFilter( req );
90 | expect( req.elastic_query.query.bool.filter.length ).to.eql( 1 );
91 | } );
92 |
93 | it( "can add to a prefiltered bool", ( ) => {
94 | const req = {
95 | params: {
96 | x: 1,
97 | y: 1,
98 | zoom: 1
99 | },
100 | elastic_query: {
101 | query: {
102 | bool: {
103 | filter: [
104 | { something: "different" }
105 | ]
106 | }
107 | }
108 | }
109 | };
110 | expect( _.size( req.elastic_query.query.bool.filter ) ).to.eql( 1 );
111 | ElasticRequest.applyBoundingBoxFilter( req );
112 | expect( _.size( req.elastic_query.query.bool.filter ) ).to.eql( 2 );
113 | } );
114 | } );
115 |
116 | describe( "geohashPrecision", ( ) => {
117 | it( "returns the proper percision for a zoom", ( ) => {
118 | expect( ElasticRequest.geohashPrecision( 1 ) ).to.eql( 3 );
119 | expect( ElasticRequest.geohashPrecision( 2 ) ).to.eql( 3 );
120 | expect( ElasticRequest.geohashPrecision( 3 ) ).to.eql( 4 );
121 | expect( ElasticRequest.geohashPrecision( 4 ) ).to.eql( 4 );
122 | expect( ElasticRequest.geohashPrecision( 5 ) ).to.eql( 4 );
123 | expect( ElasticRequest.geohashPrecision( 6 ) ).to.eql( 5 );
124 | expect( ElasticRequest.geohashPrecision( 7 ) ).to.eql( 5 );
125 | expect( ElasticRequest.geohashPrecision( 8 ) ).to.eql( 6 );
126 | expect( ElasticRequest.geohashPrecision( 9 ) ).to.eql( 6 );
127 | expect( ElasticRequest.geohashPrecision( 10 ) ).to.eql( 6 );
128 | expect( ElasticRequest.geohashPrecision( 11 ) ).to.eql( 7 );
129 | expect( ElasticRequest.geohashPrecision( 12 ) ).to.eql( 8 );
130 | expect( ElasticRequest.geohashPrecision( 13 ) ).to.eql( 9 );
131 | expect( ElasticRequest.geohashPrecision( 14 ) ).to.eql( 9 );
132 | expect( ElasticRequest.geohashPrecision( 15 ) ).to.eql( 10 );
133 | expect( ElasticRequest.geohashPrecision( 16 ) ).to.eql( 12 );
134 | } );
135 | } );
136 |
137 | describe( "geohashAggregation", ( ) => {
138 | it( "returns the proper aggregation hash based on zoom", ( ) => {
139 | expect( ElasticRequest.geohashAggregation( {
140 | params: { zoom: 15 },
141 | elastic_query: { fields: ElasticRequest.defaultMapFields( ) }
142 | } ) ).to.eql( {
143 | zoom1: {
144 | geohash_grid: {
145 | field: "location",
146 | size: 30000,
147 | precision: 10
148 | },
149 | aggs: {
150 | geohash: {
151 | top_hits: {
152 | sort: {
153 | id: {
154 | order: "desc"
155 | }
156 | },
157 | _source: false,
158 | size: 1
159 | }
160 | }
161 | }
162 | }
163 | } );
164 | } );
165 |
166 | it( "returns the source if requested", ( ) => {
167 | const agg = ElasticRequest.geohashAggregation( {
168 | query: { source: true },
169 | params: { zoom: 15 },
170 | elastic_query: { fields: ElasticRequest.defaultMapFields( ) }
171 | } );
172 | expect( agg.zoom1.aggs.geohash.top_hits._source ).to.be.true;
173 | } );
174 | } );
175 | } );
176 |
--------------------------------------------------------------------------------
/test/lib/helpers.js:
--------------------------------------------------------------------------------
1 | const ElasticRequest = require( "../../lib/elastic_request" );
2 |
3 | const helpers = { };
4 |
5 | helpers.testConfig = ( ) => (
6 | { environment: "test", debug: false }
7 | );
8 |
9 | helpers.rebuildTestIndex = async ( ) => {
10 | ElasticRequest.createClient( );
11 | const indexOptions = { index: global.config.elasticsearch.searchIndex };
12 | if ( await ElasticRequest.esClient.indices.exists( indexOptions ) ) {
13 | await ElasticRequest.esClient.indices.delete( indexOptions );
14 | await helpers.createTestIndex( );
15 | } else {
16 | await helpers.createTestIndex( );
17 | }
18 | };
19 |
20 | helpers.createTestIndex = async ( ) => {
21 | const body = {
22 | properties: {
23 | id: { type: "integer" },
24 | user: {
25 | properties: {
26 | name: { type: "text" }
27 | }
28 | },
29 | location: { type: "geo_point" },
30 | geojson: { type: "geo_shape" },
31 | observed_on_details: {
32 | properties: {
33 | month: { type: "byte" }
34 | }
35 | }
36 | }
37 | };
38 | const indexOptions = { index: global.config.elasticsearch.searchIndex };
39 | await ElasticRequest.esClient.indices.create( indexOptions );
40 | await ElasticRequest.esClient.indices.putMapping( {
41 | ...indexOptions,
42 | body
43 | } );
44 | await ElasticRequest.esClient.create( {
45 | ...indexOptions,
46 | refresh: true,
47 | type: "_doc",
48 | id: "1",
49 | body: {
50 | id: 1,
51 | location: "51.18,-1.83",
52 | geojson: { type: "Point", coordinates: [-1.83, 51.18] },
53 | observed_on_details: {
54 | month: 1
55 | }
56 | }
57 | } );
58 | };
59 |
60 | helpers.deleteTestIndex = async ( ) => {
61 | ElasticRequest.createClient( );
62 | const indexOptions = { index: global.config.elasticsearch.searchIndex };
63 | if ( await ElasticRequest.esClient.indices.exists( indexOptions ) ) {
64 | await ElasticRequest.esClient.indices.delete( indexOptions );
65 | }
66 | };
67 |
68 | module.exports = helpers;
69 |
--------------------------------------------------------------------------------
/test/map_generator.js:
--------------------------------------------------------------------------------
1 | const { expect } = require( "chai" );
2 | const MapGenerator = require( "../lib/map_generator" );
3 |
4 | describe( "MapGenerator", ( ) => {
5 | describe( "createMapTemplate", ( ) => {
6 | it( "fails on invalid styles", async ( ) => {
7 | await expect( MapGenerator.createMapTemplate(
8 | { params: { x: 0, y: 0, zoom: 1 }, style: "nonsense" }
9 | ) ).to.be.rejectedWith( "Unable to process some data while parsing" );
10 | } );
11 | } );
12 |
13 | describe( "basicEscape", ( ) => {
14 | it( "removes single quotes", ( ) => {
15 | expect( MapGenerator.basicEscape( "How's things" ) ).to.eq( "\"Hows things\"" );
16 | } );
17 |
18 | it( "turn double quotes to single", ( ) => {
19 | expect( MapGenerator.basicEscape( "a \"quoted\" value" ) ).to.eq( "\"a 'quoted' value\"" );
20 | } );
21 | } );
22 |
23 | describe( "createMapTemplate", ( ) => {
24 | it( "returns errors", ( ) => {
25 | MapGenerator.createMapTemplate( { style: "nothing" }, err => {
26 | expect( err.message ).to.eq(
27 | "Cannot read properties of undefined (reading 'format')"
28 | );
29 | } );
30 | } );
31 | } );
32 |
33 | describe( "geojsonDatasource", ( ) => {
34 | it( "prepares data for GeoJSON responses", ( ) => {
35 | const features = [
36 | {
37 | type: "Feature",
38 | geometry: {
39 | type: "Point",
40 | coordinates: [12, 11]
41 | },
42 | properties: {
43 | name: "One"
44 | }
45 | },
46 | {
47 | type: "Feature",
48 | geometry: {
49 | type: "Point",
50 | coordinates: [22, 21]
51 | },
52 | properties: {
53 | name: "Two"
54 | }
55 | }
56 | ];
57 | const d = MapGenerator.geojsonDatasource( features );
58 | expect( d.extent( ) ).to.deep.eq( [12, 11, 22, 21] );
59 | } );
60 | } );
61 | } );
62 |
--------------------------------------------------------------------------------
/test/styles.js:
--------------------------------------------------------------------------------
1 | const { expect } = require( "chai" );
2 | const Styles = require( "../lib/styles" );
3 |
4 | describe( "Styles", ( ) => {
5 | describe( "points", ( ) => {
6 | it( "returns a style with the right name", ( ) => {
7 | expect( Styles.points( ) ).to.include( "Style name='style'" );
8 | } );
9 | } );
10 | } );
11 |
--------------------------------------------------------------------------------