├── .eslintignore ├── .eslintrc ├── .github └── workflows │ └── CI.yml ├── .gitignore ├── .jshintrc ├── .nvmrc ├── README.md ├── config_example.js ├── examples └── server.js ├── index.js ├── lib ├── assets │ └── blank.png ├── elastic_mapper.js ├── elastic_request.js ├── map_generator.js └── styles.js ├── package-lock.json ├── package.json └── test ├── .eslintrc ├── elastic_mapper.js ├── elastic_request.js ├── lib └── helpers.js ├── map_generator.js └── styles.js /.eslintignore: -------------------------------------------------------------------------------- 1 | coverage/** 2 | node_modules/** 3 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "node": true, 4 | "mocha": true, 5 | "es6": true 6 | }, 7 | "parser": "@babel/eslint-parser", 8 | "parserOptions": { 9 | "requireConfigFile": false, 10 | }, 11 | "extends": "airbnb", 12 | "rules": { 13 | "arrow-parens": [2, "as-needed"], 14 | "comma-dangle": [2, "never"], 15 | "consistent-return": [2, { "treatUndefinedAsUnspecified": true }], 16 | "func-names": 0, 17 | "no-underscore-dangle": 0, 18 | "no-void": 0, 19 | "prefer-destructuring": [2, { "object": true, "array": false }], 20 | "quotes": [2, "double"], 21 | "space-in-parens": [2, "always"], 22 | "no-param-reassign": 0, 23 | "prefer-const": [2, { "destructuring": "all" }], 24 | "function-paren-newline": [2, "consistent"] 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.github/workflows/CI.yml: -------------------------------------------------------------------------------- 1 | name: elasticmaps CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | pre_build: 7 | runs-on: ubuntu-20.04 8 | steps: 9 | - uses: fkirc/skip-duplicate-actions@master 10 | with: 11 | github_token: ${{ github.token }} 12 | 13 | build: 14 | needs: pre_build 15 | runs-on: ubuntu-20.04 16 | steps: 17 | 18 | - uses: actions/checkout@v3 19 | 20 | - name: Use Node.js 21 | uses: actions/setup-node@v3 22 | with: 23 | node-version: 16.x 24 | 25 | - name: Configure sysctl limits 26 | run: | 27 | sudo swapoff -a 28 | sudo sysctl -w vm.swappiness=1 29 | sudo sysctl -w fs.file-max=262144 30 | sudo sysctl -w vm.max_map_count=262144 31 | 32 | - name: Runs Elasticsearch 33 | uses: miyataka/elastic-github-actions/elasticsearch@feature/plugin_support 34 | with: 35 | stack-version: 7.17.1 36 | plugins: analysis-kuromoji 37 | 38 | - name: Elasticsearch is reachable 39 | run: | 40 | curl --verbose --show-error http://localhost:9200 41 | 42 | - run: npm install 43 | 44 | - run: npm run coverage 45 | 46 | notify: 47 | name: Notify Slack 48 | needs: build 49 | if: ${{ success() || failure() }} 50 | runs-on: ubuntu-20.04 51 | steps: 52 | - uses: iRoachie/slack-github-actions@v2.3.2 53 | if: env.SLACK_WEBHOOK_URL != null 54 | env: 55 | SLACK_WEBHOOK_URL: ${{ secrets.SLACK_BUILDS_WEBHOOK_URL }} 56 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/.DS_Store 2 | .coveralls.yml 3 | .nyc_output 4 | coverage 5 | node_modules 6 | 7 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "multistr" : true // Tolerate multi-line strings 3 | } 4 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 16.13.2 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Elasticmaps 2 | 3 | [![Build Status](https://github.com/inaturalist/elasticmaps/workflows/elasticmaps%20CI/badge.svg)](https://github.com/inaturalist/elasticmaps/actions) 4 | [![Coverage Status](https://coveralls.io/repos/inaturalist/elasticmaps/badge.svg?branch=main)](https://coveralls.io/r/inaturalist/elasticmaps?branch=main) 5 | 6 | A Node.js map tile server based on node-mapnik and elasticsearch 7 | 8 | Installation 9 | ------- 10 | ``` 11 | npm install elasticmaps --save 12 | ``` 13 | 14 | Usage 15 | ----- 16 | ```js 17 | // This is the most basic example. It assumes elasticsearch 18 | // is running on http://localhost:9200, and that there is an index 19 | // named elasticmaps_development which has documents with minimally 20 | // an integer `id` and geo_point `location` field 21 | 22 | const Elasticmaps = require( "elasticmaps" ); 23 | const port = Number( process.env.PORT || 4000 ); 24 | 25 | const app = Elasticmaps.server( ); 26 | // create the tile route 27 | app.get( "/:style/:zoom/:x/:y.:format([a-z\.]+)", Elasticmaps.route ); 28 | 29 | app.listen( port, ( ) => { 30 | console.log( "Listening on " + port ); 31 | } ); 32 | ``` 33 | 34 | ---- 35 | 36 | ```js 37 | // In this example a custom config object is supplied 38 | // when creating the server. Functions can be provided 39 | // to create custom queries and styles based on the request 40 | 41 | const Elasticmaps = require( "elasticmaps" ); 42 | const port = Number( process.env.PORT || 4000 ); 43 | 44 | const config = { 45 | environment: "production", 46 | debug: true, 47 | tileSize: 256, 48 | elasticsearch: { 49 | host: "http://localhost:9200", 50 | searchIndex: "points_index", 51 | geoPointField: "location" 52 | }, 53 | prepareQuery: req => { 54 | req.elastic_query = ...; 55 | }, 56 | prepareStyle: req => { 57 | req.style = ...; 58 | } 59 | }; 60 | 61 | const server = Elasticmaps.server( config ); 62 | server.listen( port, ( ) => { 63 | console.log( "Listening on " + port ); 64 | } ); 65 | ``` 66 | -------------------------------------------------------------------------------- /config_example.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | elasticsearch: { 3 | host: "http://localhost:9200", 4 | searchIndex: "points_index", 5 | geoPointField: "location" 6 | }, 7 | tileSize: 256, 8 | debug: true 9 | }; 10 | -------------------------------------------------------------------------------- /examples/server.js: -------------------------------------------------------------------------------- 1 | const server = require( "../lib/elastic_mapper" ); 2 | 3 | server( { environment: "test", debug: true } ); 4 | const port = Number( process.env.PORT || 4000 ); 5 | 6 | server.listen( port, ( ) => { 7 | console.log( `Listening on ${port}` ); // eslint-disable-line no-console 8 | } ); 9 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const _ = require( "lodash" ); 2 | const ElasticMapper = require( "./lib/elastic_mapper" ); 3 | const ElasticRequest = require( "./lib/elastic_request" ); 4 | 5 | module.exports = _.assignIn( ElasticMapper, { 6 | geohashPrecision: ElasticRequest.geohashPrecision, 7 | geohashAggregation: ElasticRequest.geohashAggregation, 8 | torqueAggregation: ElasticRequest.torqueAggregation 9 | } ); 10 | -------------------------------------------------------------------------------- /lib/assets/blank.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/inaturalist/elasticmaps/16ee71c4a7a97c23c2f5725cc22324536b9f756d/lib/assets/blank.png -------------------------------------------------------------------------------- /lib/elastic_mapper.js: -------------------------------------------------------------------------------- 1 | const express = require( "express" ); 2 | const querystring = require( "querystring" ); 3 | const _ = require( "lodash" ); 4 | const Geohash = require( "latlon-geohash" ); 5 | const MapGenerator = require( "./map_generator" ); 6 | const ElasticRequest = require( "./elastic_request" ); 7 | 8 | const ElasticMapper = { }; 9 | let value = null; 10 | 11 | ElasticMapper.renderMessage = ( res, message, status ) => { 12 | res.set( "Content-Type", "text/html" ); 13 | res.status( status ).send( message ).end( ); 14 | }; 15 | 16 | ElasticMapper.renderError = ( res, error ) => { 17 | ElasticMapper.debug( error ); 18 | if ( error.message && error.status ) { 19 | ElasticMapper.renderMessage( res, error.message, error.status ); 20 | } else { 21 | ElasticMapper.renderMessage( res, "Error", 500 ); 22 | } 23 | }; 24 | 25 | ElasticMapper.renderResult = ( req, res, data ) => { 26 | if ( req.params.format === "grid.json" ) { 27 | res.jsonp( data ); 28 | } else { 29 | res.writeHead( 200, { "Content-Type": "image/png" } ); 30 | res.end( data ); 31 | } 32 | }; 33 | 34 | ElasticMapper.prepareQuery = async req => { 35 | req.elastic_query = { }; 36 | req.elastic_query.sort = { id: "desc" }; 37 | req.query.source = { includes: ElasticRequest.defaultMapFields( ) }; 38 | req.elastic_query.query = { bool: { } }; 39 | switch ( req.params.style ) { 40 | case "points": 41 | req.elastic_query.size = 10000; 42 | break; 43 | case "geotile": 44 | req.geotilegrid = true; 45 | req.includeTotalHits = true; 46 | req.elastic_query.size = 0; 47 | req.elastic_query.aggregations = ElasticRequest.geohashAggregation( req ); 48 | break; 49 | case "geohash": 50 | req.elastic_query.size = 0; 51 | if ( req.params.format === "torque.json" ) { 52 | req.elastic_query.aggregations = ElasticRequest.torqueAggregation( req ); 53 | } else { 54 | req.elastic_query.aggregations = ElasticRequest.geohashAggregation( req ); 55 | } 56 | break; 57 | default: 58 | // eslint-disable-next-line no-case-declarations 59 | const e = new Error( ); 60 | e.status = 404; 61 | e.message = `unknown style: ${req.params.style}`; 62 | throw e; 63 | } 64 | }; 65 | 66 | ElasticMapper.geotileGridGeojson = hit => { 67 | const parts = hit.key.split( "/" ); 68 | MapGenerator.createMercator( ); 69 | const bbox = MapGenerator.merc.bbox( parts[1], parts[2], parts[0] ); 70 | return { 71 | type: "Feature", 72 | geometry: { 73 | type: "Polygon", 74 | coordinates: [[ 75 | [bbox[0], bbox[1]], 76 | [bbox[2], bbox[1]], 77 | [bbox[2], bbox[3]], 78 | [bbox[0], bbox[3]], 79 | [bbox[0], bbox[1]] 80 | ]] 81 | }, 82 | properties: { 83 | cellCount: hit.doc_count 84 | } 85 | }; 86 | }; 87 | 88 | ElasticMapper.geohashGridGeojson = hit => { 89 | const bbox = Geohash.bounds( hit.key ); 90 | return { 91 | type: "Feature", 92 | geometry: { 93 | type: "Polygon", 94 | coordinates: [[ 95 | [bbox.sw.lon, bbox.sw.lat], 96 | [bbox.sw.lon, bbox.ne.lat], 97 | [bbox.ne.lon, bbox.ne.lat], 98 | [bbox.ne.lon, bbox.sw.lat], 99 | [bbox.sw.lon, bbox.sw.lat] 100 | ]] 101 | }, 102 | properties: { 103 | cellCount: hit.doc_count 104 | } 105 | }; 106 | }; 107 | 108 | ElasticMapper.csvFromResult = ( req, result ) => { 109 | if ( req.params.dataType === "geojson" ) { 110 | return ElasticMapper.polygonCSVFromResult( req, result ); 111 | } 112 | let target; 113 | if ( result.aggregations && result.aggregations.zoom1 ) { 114 | target = _.sortBy( result.aggregations.zoom1.buckets, hit => ( 115 | hit.geohash ? hit.geohash.hits.hits[0].sort[0] : null 116 | ) ); 117 | } else if ( result.hits ) { 118 | target = result.hits.hits; 119 | } else { return []; } 120 | const { geoPointField } = global.config.elasticsearch; 121 | const fieldsToMap = ( req.query.source && req.query.source.includes ) 122 | ? _.without( req.query.source.includes, geoPointField ) 123 | : []; 124 | fieldsToMap.push( "cellCount" ); 125 | const csvData = _.map( target, hit => { 126 | // grids get rendered as polygons 127 | if ( req.geotilegrid && req.params.format !== "grid.json" ) { 128 | return ElasticMapper.geotileGridGeojson( hit ); 129 | } 130 | if ( req.geogrid ) { 131 | return ElasticMapper.geohashGridGeojson( hit ); 132 | } 133 | const fieldData = hit._source || hit.geohash.hits.hits[0]._source; 134 | fieldData.private_location = !_.isEmpty( fieldData.private_location ); 135 | if ( !hit._source && hit.geohash ) { 136 | fieldData.cellCount = hit.geohash.hits.total.value; 137 | } 138 | const properties = { }; 139 | _.each( fieldsToMap, f => { 140 | if ( f.match( /\./ ) ) { 141 | const parts = f.split( "." ); 142 | if ( fieldData[parts[0]] && fieldData[parts[0]][parts[1]] ) { 143 | value = fieldData[parts[0]][parts[1]]; 144 | } else { 145 | value = null; 146 | } 147 | } else { 148 | value = fieldData[f] ? fieldData[f] : null; 149 | } 150 | if ( value === "F" ) { value = false; } 151 | if ( value === "T" ) { value = true; } 152 | properties[f] = value; 153 | } ); 154 | let latitude; 155 | let longitude; 156 | if ( req.geotilegrid ) { 157 | const parts = hit.key.split( "/" ); 158 | MapGenerator.createMercator( ); 159 | const bbox = MapGenerator.merc.bbox( parts[1], parts[2], parts[0] ); 160 | latitude = _.mean( [bbox[1], bbox[3]] ); 161 | longitude = _.mean( [bbox[0], bbox[2]] ); 162 | } else if ( _.isObject( fieldData[geoPointField] ) ) { 163 | latitude = fieldData[geoPointField].lat; 164 | longitude = fieldData[geoPointField].lon; 165 | } else { 166 | const coords = fieldData[geoPointField].split( "," ); 167 | latitude = Number( coords[0] ); 168 | longitude = Number( coords[1] ); 169 | } 170 | if ( req.params.format === "grid.json" ) { 171 | properties.latitude = latitude; 172 | properties.longitude = longitude; 173 | } 174 | return { 175 | type: "Feature", 176 | geometry: { 177 | type: "Point", 178 | coordinates: [longitude, latitude] 179 | }, 180 | properties 181 | }; 182 | } ); 183 | return csvData; 184 | }; 185 | 186 | ElasticMapper.polygonCSVFromResult = ( req, result ) => ( 187 | _.map( result.hits.hits, hit => ( 188 | { id: hit._source.id, geojson: hit._source.geometry_geojson } 189 | ) ) 190 | ); 191 | 192 | ElasticMapper.route = async ( req, res ) => { 193 | req.startTime = Date.now( ); 194 | req.params.zoom = parseInt( req.params.zoom, 10 ); 195 | req.params.x = parseInt( req.params.x, 10 ); 196 | req.params.y = parseInt( req.params.y, 10 ); 197 | if ( req.params.zoom < 0 || req.params.zoom > 21 ) { 198 | ElasticMapper.renderMessage( res, "Invalid zoom", 404 ); 199 | return; 200 | } 201 | const zoomDimension = 2 ** req.params.zoom; 202 | if ( req.params.x < 0 || req.params.x >= zoomDimension ) { 203 | ElasticMapper.renderMessage( res, "Invalid x value", 404 ); 204 | return; 205 | } 206 | if ( req.params.y < 0 || req.params.y >= zoomDimension ) { 207 | ElasticMapper.renderMessage( res, "Invalid y value", 404 ); 208 | return; 209 | } 210 | if ( !_.includes( ["png", "grid.json", "torque.json"], req.params.format ) ) { 211 | ElasticMapper.renderMessage( res, "Invalid format", 404 ); 212 | return; 213 | } 214 | 215 | try { 216 | const prepareQuery = global.config.prepareQuery || ElasticMapper.prepareQuery; 217 | await prepareQuery( req ); 218 | if ( req.includeTotalHits && req.params.dataType !== "postgis" ) { 219 | const cachedCount = await ElasticRequest.count( req, { }, this ); 220 | req.totalHits = cachedCount || 0; 221 | } 222 | if ( req.params.dataType !== "postgis" ) { 223 | ElasticRequest.applyBoundingBoxFilter( req ); 224 | const result = await ElasticRequest.search( req, { 225 | track_total_hits: false 226 | } ); 227 | if ( req.params.format === "torque.json" ) { 228 | MapGenerator.torqueJson( result, req, res ); 229 | return; 230 | } 231 | ElasticMapper.debug( `${result.took}ms :: [${req.bbox}]` ); 232 | req.csvData = ElasticMapper.csvFromResult( req, result ); 233 | } 234 | if ( global.config.prepareStyle && req.params.format !== "grid.json" ) { 235 | await global.config.prepareStyle( req ); 236 | } 237 | const { map, layer } = await MapGenerator.createMapTemplate( req ); 238 | await MapGenerator.finishMap( req, res, map, layer, req.csvData ); 239 | if ( global.config.beforeSendResult ) { 240 | await global.config.beforeSendResult( req, res ); 241 | } 242 | req.endTime = new Date( ); 243 | ElasticMapper.renderResult( req, res, req.tileData ); 244 | if ( global.config.debug ) { 245 | ElasticMapper.printRequestLog( req ); 246 | } 247 | } catch ( e ) { 248 | ElasticMapper.renderError( res, e ); 249 | } 250 | }; 251 | 252 | ElasticMapper.printRequestLog = req => { 253 | let logText = `[ ${new Date( ).toString( )}] GET /${req.params.style}` 254 | + `/${req.params.zoom}/${req.params.x}` 255 | + `/${req.params.y}.${req.params.format}`; 256 | if ( !_.isEmpty( req.query ) ) { 257 | logText += `?${querystring.stringify( req.query )}`; 258 | } 259 | logText += ` ${req.endTime - req.startTime}ms`; 260 | ElasticMapper.debug( logText ); 261 | }; 262 | 263 | ElasticMapper.debug = text => { 264 | if ( global.config.debug ) { 265 | console.log( text ); // eslint-disable-line no-console 266 | } 267 | }; 268 | 269 | ElasticMapper.server = ( config = { } ) => { 270 | global.config = _.defaults( config, { 271 | environment: config.NODE_ENV || process.env.NODE_ENV || "development", 272 | tileSize: 256, 273 | debug: !( config.debug === false ) 274 | } ); 275 | global.config.log = global.config.log 276 | || `elasticmaps.${global.config.environment}.log`; 277 | global.config.elasticsearch = _.defaults( 278 | global.config.elasticsearch || { }, 279 | { 280 | host: "http://localhost:9200", 281 | searchIndex: `elasticmaps_${global.config.environment}`, 282 | geoPointField: "location" 283 | } 284 | ); 285 | // create the server and the map route 286 | const server = express( ); 287 | if ( global.config.prepareApp && _.isFunction( global.config.prepareApp ) ) { 288 | global.config.prepareApp( server, config ); 289 | } 290 | return server; 291 | }; 292 | 293 | module.exports = ElasticMapper; 294 | -------------------------------------------------------------------------------- /lib/elastic_request.js: -------------------------------------------------------------------------------- 1 | /* eslint no-underscore-dangle: 0 */ 2 | /* eslint prefer-destructuring: 0 */ 3 | 4 | const _ = require( "lodash" ); 5 | const { Client } = require( "@elastic/elasticsearch" ); 6 | const Cacheman = require( "recacheman" ); 7 | const crypto = require( "crypto" ); 8 | const MapGenerator = require( "./map_generator" ); 9 | 10 | const ElasticRequest = { esClient: null }; 11 | const cache = new Cacheman( ); 12 | 13 | const RESERVED_COUNT_TIMEOUT = 10000; // 10s 14 | const RESERVED_COUNT_CHECK_DELAY = 100; // 100ms 15 | const COUNT_CACHE_SECONDS = 60 * 60 * 24; // 1 day 16 | 17 | ElasticRequest.createClient = ( ) => { 18 | if ( ElasticRequest.esClient === null ) { 19 | const clientConfig = { }; 20 | if ( global.config.elasticsearch.hosts ) { 21 | clientConfig.node = _.isArray( global.config.elasticsearch.hosts ) 22 | ? global.config.elasticsearch.hosts 23 | : global.config.elasticsearch.hosts.split( " " ); 24 | } else { 25 | clientConfig.node = global.config.elasticsearch.host; 26 | } 27 | clientConfig.nodeSelector = "random"; 28 | clientConfig.requestTimeout = 60000; 29 | clientConfig.maxRetries = 1; 30 | ElasticRequest.esClient = new Client( clientConfig ); 31 | } 32 | return ElasticRequest.esClient; 33 | }; 34 | 35 | ElasticRequest.count = async function ( req, options = { } ) { 36 | const query = { ...req.elastic_query }; 37 | delete query.aggregations; 38 | query.size = 0; 39 | req.elastic_client ||= ElasticRequest.createClient( ); 40 | const queryHash = crypto 41 | .createHash( "md5" ) 42 | .update( JSON.stringify( query ), "utf8" ) 43 | .digest( "hex" ); 44 | const cacheKey = `elasticmaps-${queryHash}`; 45 | 46 | const countMethod = async ( ) => { 47 | const response = await req.elastic_client.search( { 48 | preference: global.config.elasticsearch.preference, 49 | index: req.elastic_index || global.config.elasticsearch.searchIndex, 50 | body: query, 51 | track_total_hits: true, 52 | ...options 53 | } ); 54 | return response.hits.total.value; 55 | }; 56 | 57 | if ( req.redisCacheClient ) { 58 | return ElasticRequest.reservedCount( req, cacheKey, countMethod ); 59 | } 60 | 61 | const cachedCount = await cache.get( cacheKey ); 62 | if ( cachedCount ) { 63 | return cachedCount; 64 | } 65 | const response = await countMethod( ); 66 | await cache.set( cacheKey, response, COUNT_CACHE_SECONDS ); 67 | return response; 68 | }; 69 | 70 | ElasticRequest.reservedCount = async function ( 71 | req, cacheKey, countMethod, checkReservations = true 72 | ) { 73 | const reservationCacheKey = `${cacheKey}:reservation`; 74 | const cacheCountMethod = async ( ) => { 75 | // mark the count as reserved 76 | await req.redisCacheClient.set( reservationCacheKey, "reserved", { EX: RESERVED_COUNT_TIMEOUT / 1000 } ); 77 | // run the count query 78 | const countResult = await countMethod( ); 79 | // cache the count 80 | await req.redisCacheClient.set( cacheKey, countResult, { EX: COUNT_CACHE_SECONDS } ); 81 | // delete the count reservation 82 | await req.redisCacheClient.del( reservationCacheKey ); 83 | return countResult; 84 | }; 85 | 86 | if ( checkReservations ) { 87 | const retryCountMethod = async ( ) => ( 88 | ElasticRequest.reservedCount( req, cacheKey, countMethod, false ) 89 | ); 90 | // check to see if the count query is reserved (being run) by another process 91 | return ElasticRequest.waitForReservedCount( 92 | req, reservationCacheKey, cacheCountMethod, retryCountMethod 93 | ); 94 | } 95 | 96 | // check if the count is cached 97 | const cachedCount = await req.redisCacheClient.get( cacheKey ); 98 | if ( cachedCount ) { 99 | return Number( cachedCount ); 100 | } 101 | // otherwise perform the count process 102 | return cacheCountMethod( ); 103 | }; 104 | 105 | ElasticRequest.waitForReservedCount = async function ( 106 | req, reservationCacheKey, countMethod, retryCountMethod, startTime = Date.now( ) 107 | ) { 108 | if ( Date.now( ) - startTime >= RESERVED_COUNT_TIMEOUT ) { 109 | // reservation is too old, so run the count 110 | return retryCountMethod( ); 111 | } 112 | const reservationExists = await req.redisCacheClient.get( reservationCacheKey ); 113 | if ( reservationExists ) { 114 | // there is a reservation for the count, so wait a bit and check again 115 | return ElasticRequest.waitForReservedCountDelay( 116 | req, reservationCacheKey, countMethod, retryCountMethod, startTime 117 | ); 118 | } 119 | // there is no reservation, so run the count 120 | return retryCountMethod( ); 121 | }; 122 | 123 | ElasticRequest.waitForReservedCountDelay = function ( 124 | req, reservationCacheKey, countMethod, retryCountMethod, startTime 125 | ) { 126 | return new Promise( ( resolve, reject ) => { 127 | // wait RESERVED_COUNT_CHECK_DELAY ms to check again for a reservation 128 | setTimeout( ( ) => { 129 | ElasticRequest.waitForReservedCount( 130 | req, reservationCacheKey, countMethod, retryCountMethod, startTime 131 | ).then( resolve ).catch( reject ); 132 | }, RESERVED_COUNT_CHECK_DELAY ); 133 | } ); 134 | }; 135 | 136 | ElasticRequest.search = async ( req, options = { } ) => { 137 | const query = { ...req.elastic_query }; 138 | req.elastic_client ||= ElasticRequest.createClient( ); 139 | return req.elastic_client.search( { 140 | preference: global.config.elasticsearch.preference, 141 | index: req.elastic_index || global.config.elasticsearch.searchIndex, 142 | body: query, 143 | ...options 144 | } ); 145 | }; 146 | 147 | ElasticRequest.expandBoxForSmoothEdges = ( req, bbox ) => { 148 | const qbbox = bbox; 149 | const height = Math.abs( qbbox[2] - qbbox[0] ); 150 | const width = Math.abs( qbbox[3] - qbbox[1] ); 151 | const factor = ( req && req.params && req.params.style === "grid" ) ? 0.02 : 0.07; 152 | qbbox[0] -= ( height * factor ); 153 | qbbox[2] += ( height * factor ); 154 | qbbox[1] -= ( width * factor ); 155 | qbbox[3] += ( width * factor ); 156 | if ( qbbox[0] < -180 ) { qbbox[0] = -180; } 157 | if ( qbbox[1] < -90 ) { qbbox[1] = -90; } 158 | if ( qbbox[2] > 180 ) { qbbox[2] = 180; } 159 | if ( qbbox[3] > 90 ) { qbbox[3] = 90; } 160 | return qbbox; 161 | }; 162 | 163 | ElasticRequest.boundingBoxFilter = ( req, bbox, smoothing ) => { 164 | let qbbox = bbox; 165 | if ( smoothing !== false ) { 166 | qbbox = ElasticRequest.expandBoxForSmoothEdges( req, qbbox ); 167 | } 168 | if ( qbbox[2] < qbbox[0] ) { 169 | // the envelope crosses the dateline. Unfortunately, elasticsearch 170 | // doesn't handle this well and we need to split the envelope at 171 | // the dateline and do an OR query 172 | const left = _.clone( qbbox ); 173 | const right = _.clone( qbbox ); 174 | left[2] = 180; 175 | right[0] = -180; 176 | return { 177 | bool: { 178 | should: [ 179 | ElasticRequest.boundingBoxFilter( req, left, false ), 180 | ElasticRequest.boundingBoxFilter( req, right, false ) 181 | ] 182 | } 183 | }; 184 | } 185 | 186 | const field = global.config.elasticsearch.geoPointField; 187 | const boundingBox = { }; 188 | boundingBox[field] = { 189 | bottom_left: [qbbox[0], qbbox[1]], 190 | top_right: [qbbox[2], qbbox[3]] 191 | }; 192 | boundingBox.type = "indexed"; 193 | return { geo_bounding_box: boundingBox }; 194 | }; 195 | 196 | ElasticRequest.geohashPrecision = ( zoom, offset ) => { 197 | let precision = 3; 198 | if ( zoom >= 3 ) { precision = 4; } 199 | if ( zoom >= 6 ) { precision = 5; } 200 | if ( zoom >= 8 ) { precision = 6; } 201 | if ( zoom >= 11 ) { precision = 7; } 202 | if ( zoom >= 12 ) { precision = 8; } 203 | if ( zoom >= 13 ) { precision = 9; } 204 | if ( zoom >= 15 ) { precision = 10; } 205 | if ( zoom >= 16 ) { precision = 12; } 206 | if ( offset && offset > 0 && ( precision - offset ) > 0 ) { 207 | precision -= offset; 208 | } 209 | return precision; 210 | }; 211 | 212 | ElasticRequest.geotileGridPrecision = ( zoom, offset ) => { 213 | let precision = zoom + 5; 214 | if ( offset && ( precision - offset ) > 0 ) { 215 | precision -= offset; 216 | } 217 | return precision; 218 | }; 219 | 220 | ElasticRequest.defaultMapFields = ( ) => ( 221 | ["id", global.config.elasticsearch.geoPointField] 222 | ); 223 | 224 | ElasticRequest.defaultMapQuery = ( ) => ( 225 | { match_all: { } } 226 | ); 227 | 228 | ElasticRequest.applyBoundingBoxFilter = req => { 229 | if ( req.params.dataType === "postgis" ) { 230 | return; 231 | } 232 | MapGenerator.createMercator( ); 233 | req.bbox = MapGenerator.merc.convert( MapGenerator.bboxFromParams( req ) ); 234 | const smoothing = req.params.format !== "torque.json"; 235 | const bboxFilter = ElasticRequest.boundingBoxFilter( req, req.bbox, smoothing ); 236 | if ( !req.elastic_query ) { req.elastic_query = { }; } 237 | if ( !req.elastic_query.query ) { req.elastic_query.query = { }; } 238 | if ( !req.elastic_query.query.bool ) { req.elastic_query.query.bool = { }; } 239 | if ( !req.elastic_query.query.bool.filter ) { req.elastic_query.query.bool.filter = []; } 240 | req.elastic_query.query.bool.filter.push( bboxFilter ); 241 | }; 242 | 243 | ElasticRequest.geohashAggregation = req => { 244 | let agg; 245 | if ( req.geotile || req.geotilegrid ) { 246 | // use the geotile aggregation added in Elasticsearch 7 247 | agg = { 248 | zoom1: { 249 | geotile_grid: { 250 | field: global.config.elasticsearch.geoPointField, 251 | size: 10000, 252 | precision: ElasticRequest.geotileGridPrecision( 253 | req.params.zoom, req.query ? Number( req.query.precision_offset ) : null 254 | ) 255 | } 256 | } 257 | }; 258 | } else { 259 | agg = { 260 | zoom1: { 261 | geohash_grid: { 262 | field: global.config.elasticsearch.geoPointField, 263 | size: 30000, 264 | precision: ElasticRequest.geohashPrecision( 265 | req.params.zoom, req.query ? Number( req.query.precision_offset ) : null 266 | ) 267 | } 268 | } 269 | }; 270 | } 271 | // UTFGrids always need topHits to get metadata for each cell. 272 | // PNG versions of grid-based styles do not need topHits 273 | if ( req.params.format === "grid.json" || ( !req.skipTopHits && !req.geogrid && !req.geotilegrid ) ) { 274 | agg.zoom1.aggs = { 275 | geohash: { 276 | top_hits: { 277 | sort: { id: { order: "desc" } }, 278 | _source: ( ( req.query && req.query.source ) 279 | ? req.query.source : false ), 280 | size: 1 281 | } 282 | } 283 | }; 284 | } 285 | return agg; 286 | }; 287 | 288 | ElasticRequest.torqueAggregation = req => { 289 | const interval = req.query.interval === "weekly" ? "week" : "month"; 290 | return { 291 | zoom1: { 292 | geohash_grid: { 293 | field: global.config.elasticsearch.geoPointField, 294 | size: 30000, 295 | precision: ElasticRequest.geohashPrecision( 296 | req.params.zoom, req.query ? Number( req.query.precision_offset ) : null 297 | ) 298 | }, 299 | aggs: { 300 | histogram: { 301 | terms: { 302 | field: `observed_on_details.${interval}`, 303 | size: 100 304 | }, 305 | aggs: { 306 | geohash: { 307 | top_hits: { 308 | sort: { id: { order: "desc" } }, 309 | _source: ( ( req.query && req.query.source ) 310 | ? req.query.source : false ), 311 | size: 1 312 | } 313 | } 314 | } 315 | } 316 | } 317 | } 318 | }; 319 | }; 320 | 321 | module.exports = ElasticRequest; 322 | -------------------------------------------------------------------------------- /lib/map_generator.js: -------------------------------------------------------------------------------- 1 | /* eslint no-underscore-dangle: 0 */ 2 | 3 | const mapnik = require( "mapnik" ); 4 | const _ = require( "lodash" ); 5 | const fs = require( "fs" ); 6 | const path = require( "path" ); 7 | const flatten = require( "flat" ); 8 | const { promisify } = require( "util" ); 9 | const SphericalMercator = require( "@mapbox/sphericalmercator" ); 10 | const Styles = require( "./styles" ); 11 | 12 | // register shapefile plugin 13 | if ( mapnik.register_default_input_plugins ) { 14 | mapnik.register_default_input_plugins( ); 15 | } 16 | 17 | const proj4 = "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 " 18 | + "+y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over"; 19 | 20 | const MapGenerator = { merc: null }; 21 | 22 | MapGenerator.blankImage = fs.readFileSync( path.join( __dirname, "assets/blank.png" ) ); 23 | 24 | MapGenerator.createMercator = ( ) => { 25 | if ( MapGenerator.merc === null ) { 26 | MapGenerator.merc = new SphericalMercator( { size: global.config.tileSize } ); 27 | } 28 | }; 29 | 30 | MapGenerator.bboxFromParams = req => { 31 | MapGenerator.createMercator( ); 32 | let zoom = parseInt( req.params.zoom, 10 ); 33 | if ( req.largeTiles ) { zoom -= 1; } 34 | return MapGenerator.merc.bbox( 35 | parseInt( req.params.x, 10 ), 36 | parseInt( req.params.y, 10 ), 37 | zoom, 38 | false, 39 | "900913" 40 | ); 41 | }; 42 | 43 | MapGenerator.createLayer = ( ) => { 44 | const layer = new mapnik.Layer( "tile", "+init=epsg:4326" ); 45 | layer.styles = ["style"]; 46 | return layer; 47 | }; 48 | 49 | MapGenerator.postgisDatasource = req => { 50 | let zoom = parseInt( req.params.zoom, 10 ); 51 | if ( req.largeTiles ) { zoom -= 1; } 52 | const datasourceConfig = { 53 | ...global.config.database, 54 | type: "postgis", 55 | table: req.postgis.query, 56 | simplify_geometries: true, 57 | extent: MapGenerator.merc.bbox( 58 | parseInt( req.params.x, 10 ), 59 | parseInt( req.params.y, 10 ), 60 | zoom 61 | ) 62 | }; 63 | return new mapnik.Datasource( datasourceConfig ); 64 | }; 65 | 66 | MapGenerator.geojsonDatasource = features => { 67 | if ( _.isEmpty( features ) ) { return null; } 68 | const datasourceConfig = { 69 | type: "geojson", 70 | inline: JSON.stringify( { 71 | type: "FeatureCollection", 72 | features 73 | } ) 74 | }; 75 | return new mapnik.Datasource( datasourceConfig ); 76 | }; 77 | 78 | /* eslint-disable no-param-reassign */ 79 | MapGenerator.finishMap = async ( req, res, map, layer, features ) => { 80 | if ( features && features.length === 0 && req.params.format !== "grid.json" ) { 81 | req.tileData = MapGenerator.blankImage; 82 | return; 83 | } 84 | let fields; 85 | if ( req.params.dataType === "postgis" ) { 86 | layer.datasource = MapGenerator.postgisDatasource( req ); 87 | } else { 88 | const memDS = MapGenerator.geojsonDatasource( features ); 89 | if ( memDS ) { layer.datasource = memDS; } 90 | } 91 | map.add_layer( layer ); 92 | let { tileSize } = global.config; 93 | if ( req.largeTiles ) { tileSize *= 2; } 94 | const mapRender = promisify( map.render.bind( map ) ); 95 | if ( req.params.format === "grid.json" ) { 96 | fields = _.without( req.query.source.includes, 97 | global.config.elasticsearch.geoPointField ); 98 | // geohash aggregations will have cellCount 99 | if ( req.elastic_query.aggregations && req.elastic_query.aggregations.zoom1 ) { 100 | fields.push( "cellCount" ); 101 | } 102 | fields = fields.concat( ["latitude", "longitude"] ); 103 | 104 | const options = { }; 105 | options.layer = "tile"; 106 | options.fields = fields; 107 | options.headers = { "Content-Type": "application/json" }; 108 | const im = new mapnik.Grid( tileSize / 2, tileSize / 2, { key: "id" } ); 109 | const img = await mapRender( im, options ); 110 | req.tileData = img.encodeSync( ); 111 | return; 112 | } 113 | const im = new mapnik.Image( tileSize, tileSize ); 114 | const img = await mapRender( im, { scale: 2 } ); 115 | req.tileData = img.encodeSync( ); 116 | }; 117 | /* eslint-enable no-param-reassign */ 118 | 119 | MapGenerator.basicEscape = text => { 120 | // remove ', \n, \r, \t 121 | // turn " into ' 122 | const replaced = text.replace( /'/g, "" ) 123 | .replace( /"/g, "'" ) 124 | .replace( /(\\n|\\r|\\t)/g, " " ) 125 | .replace( /(\\)/g, "/" ); 126 | return `"${replaced}"`; 127 | }; 128 | 129 | MapGenerator.mapXML = specificStyle => ( 130 | ` 131 | 132 | ${specificStyle} 133 | ` 134 | ); 135 | 136 | MapGenerator.createMapTemplate = async req => { 137 | req.style = req.style || Styles.points( ); 138 | let { tileSize } = global.config; 139 | if ( req.largeTiles ) { tileSize *= 2; } 140 | if ( req.params.format === "grid.json" ) { 141 | tileSize /= 2; 142 | } 143 | const map = new mapnik.Map( tileSize, tileSize, proj4 ); 144 | const bbox = MapGenerator.bboxFromParams( req ); 145 | const layer = MapGenerator.createLayer( ); 146 | const xml = MapGenerator.mapXML( req.style ); 147 | map.extent = bbox; 148 | const fromString = promisify( map.fromString.bind( map ) ); 149 | const mapFromString = await fromString( xml, { strict: true, base: "./" } ); 150 | MapGenerator.createMercator( ); 151 | return { 152 | map: mapFromString, 153 | layer 154 | }; 155 | }; 156 | 157 | MapGenerator.torqueJson = ( queryResult, req, res ) => { 158 | const [minlon, minlat, maxlon, maxlat] = req.bbox; 159 | const mercMin = MapGenerator.merc.px( [minlon, minlat], req.params.zoom ); 160 | const mercMax = MapGenerator.merc.px( [maxlon, maxlat], req.params.zoom ); 161 | const latdiff = Math.abs( mercMax[1] - mercMin[1] ); 162 | const londiff = Math.abs( mercMax[0] - mercMin[0] ); 163 | const returnArray = []; 164 | const filteredBuckets = _.filter( queryResult.aggregations.zoom1.buckets, 165 | b => !_.isEmpty( b.histogram.buckets ) ); 166 | _.each( filteredBuckets, b => { 167 | const vals = []; 168 | const dates = []; 169 | let hashInfo; 170 | _.each( b.histogram.buckets, hb => { 171 | if ( !hashInfo ) { 172 | hashInfo = hb.geohash.hits.hits[0]._source.location.split( "," ); 173 | } 174 | const doc = hb.geohash.hits.hits[0]._source; 175 | vals.push( { value: hb.doc_count, ...flatten( doc ) } ); 176 | dates.push( hb.key - 1 ); 177 | } ); 178 | const mercCoords = MapGenerator.merc.px( [hashInfo[1], hashInfo[0]], req.params.zoom ); 179 | const torqueX = Math.floor( ( Math.abs( mercCoords[0] - mercMin[0] ) / latdiff ) * 255 ); 180 | const torqueY = Math.floor( ( Math.abs( mercCoords[1] - mercMin[1] ) / londiff ) * 255 ); 181 | returnArray.push( { 182 | x__uint8: torqueX, 183 | y__uint8: torqueY, 184 | vals__uint8: vals, 185 | dates__uint16: dates 186 | } ); 187 | } ); 188 | res.set( "Content-Type", "text/html" ).status( 200 ).json( returnArray ).end( ); 189 | }; 190 | 191 | module.exports = MapGenerator; 192 | -------------------------------------------------------------------------------- /lib/styles.js: -------------------------------------------------------------------------------- 1 | const Styles = { }; 2 | 3 | Styles.points = ( ) => ( 4 | ` 5 | ` 13 | ); 14 | 15 | module.exports = Styles; 16 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "elasticmaps", 3 | "version": "5.3.0", 4 | "description": "Map tileserver based on node-mapnik and elasticsearch", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "mocha --recursive", 8 | "coverage": "nyc --reporter=lcov --reporter=text-summary npm run test" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "https://github.com/inaturalist/elasticmaps.git" 13 | }, 14 | "keywords": [ 15 | "elasticmaps", 16 | "elasticsearch", 17 | "tileserver", 18 | "maps" 19 | ], 20 | "author": "Patrick Leary ", 21 | "license": "MIT", 22 | "dependencies": { 23 | "@mapbox/sphericalmercator": "^1.2.0", 24 | "recacheman": "^2.2.4", 25 | "del": "^6.0.0", 26 | "@elastic/elasticsearch": "^8.7.0", 27 | "es6-promise-pool": "^2.5.0", 28 | "express": "^4.17.2", 29 | "flat": "^5.0.2", 30 | "handlebars": "^4.7.7", 31 | "latlon-geohash": "^1.1.0", 32 | "lodash": "^4.17.21", 33 | "mapnik": "^4.5.9", 34 | "step": "^1.0.0" 35 | }, 36 | "devDependencies": { 37 | "@babel/core": "^7.16.7", 38 | "@babel/eslint-parser": "^7.16.5", 39 | "chai": "^4.3.4", 40 | "chai-as-promised": "^7.1.1", 41 | "eslint": "^8.6.0", 42 | "eslint-config-airbnb": "^19.0.4", 43 | "eslint-plugin-import": "^2.25.4", 44 | "eslint-plugin-jsx-a11y": "^6.5.1", 45 | "eslint-plugin-react": "^7.28.0", 46 | "mocha": "^10.0.0", 47 | "nyc": "*", 48 | "supertest": "^6.2.0" 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /test/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "no-unused-expressions": 0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /test/elastic_mapper.js: -------------------------------------------------------------------------------- 1 | const { expect } = require( "chai" ); 2 | const request = require( "supertest" ); 3 | const _ = require( "lodash" ); 4 | const Mapper = require( "../lib/elastic_mapper" ); 5 | const helpers = require( "./lib/helpers" ); 6 | 7 | let app; 8 | 9 | describe( "ElasticMapper", ( ) => { 10 | before( async function ( ) { 11 | this.timeout( 10000 ); 12 | app = Mapper.server( helpers.testConfig( ) ); 13 | app.get( "/:style/:zoom/:x/:y.:format([a-z.]+)", Mapper.route ); 14 | await helpers.rebuildTestIndex( ); 15 | } ); 16 | 17 | after( async ( ) => { 18 | await helpers.deleteTestIndex( ); 19 | } ); 20 | 21 | describe( "routes", ( ) => { 22 | it( "only knows one route", done => { 23 | request( app ).get( "/" ) 24 | .expect( res => { 25 | expect( res.text ).to.include( "Cannot GET /" ); 26 | } ).expect( 404, done ); 27 | } ); 28 | 29 | it( "allows new routes to be created", done => { 30 | app.get( "/", ( req, res ) => { 31 | res.send( "success" ).status( 200 ).end( ); 32 | } ); 33 | request( app ).get( "/" ).expect( 200 ) 34 | .expect( "success", done ); 35 | } ); 36 | 37 | it( "accepts parameters", done => { 38 | request( app ).get( "/points/1/0/0.png?param=test" ).expect( 200 ) 39 | .expect( "content-type", "image/png", done ); 40 | } ); 41 | } ); 42 | 43 | describe( "validation", ( ) => { 44 | it( "accepts the .png format", done => { 45 | request( app ).get( "/points/1/0/0.png" ).expect( 200 ) 46 | .expect( "content-type", "image/png", done ); 47 | } ); 48 | 49 | it( "accepts the .grid.json format", done => { 50 | request( app ).get( "/points/1/1/1.grid.json" ).expect( 200 ) 51 | .expect( "content-type", "application/json; charset=utf-8", done ); 52 | } ); 53 | 54 | it( "errors on all other formats format", done => { 55 | request( app ).get( "/points/1/0/0.html" ).expect( 404 ) 56 | .expect( "Invalid format", done ); 57 | } ); 58 | 59 | it( "returns an error for an unknown style", done => { 60 | request( app ).get( "/nonsense/1/0/0.png" ).expect( 404 ) 61 | .expect( "unknown style: nonsense", done ); 62 | } ); 63 | 64 | it( "zoom must be 0 or above", done => { 65 | request( app ).get( "/points/-1/0/0.png" ).expect( 404 ) 66 | .expect( "Invalid zoom", done ); 67 | } ); 68 | 69 | it( "zoom must be 21 or below", done => { 70 | request( app ).get( "/points/22/0/0.png" ).expect( 404 ) 71 | .expect( "Invalid zoom", done ); 72 | } ); 73 | 74 | it( "x must be 0 or above", done => { 75 | request( app ).get( "/points/5/-1/0.png" ).expect( 404 ) 76 | .expect( "Invalid x value", done ); 77 | } ); 78 | 79 | it( "x must be within range", done => { 80 | request( app ).get( "/points/5/32/0.png" ).expect( 404 ) 81 | .expect( "Invalid x value", done ); 82 | } ); 83 | 84 | it( "y must be 0 or above", done => { 85 | request( app ).get( "/points/5/0/-1.png" ).expect( 404 ) 86 | .expect( "Invalid y value", done ); 87 | } ); 88 | 89 | it( "y must be within range", done => { 90 | request( app ).get( "/points/5/0/32.png" ).expect( 404 ) 91 | .expect( "Invalid y value", done ); 92 | } ); 93 | 94 | it( "y must be within range", done => { 95 | request( app ).get( "/points/5/0/32.png" ).expect( 404 ) 96 | .expect( "Invalid y value", done ); 97 | } ); 98 | } ); 99 | 100 | describe( "points", ( ) => { 101 | it( "renders .png", done => { 102 | request( app ).get( "/points/1/1/1.png" ).expect( 200 ) 103 | .expect( "content-type", "image/png", done ); 104 | } ); 105 | 106 | it( "renders .grid.json", done => { 107 | request( app ).get( "/points/1/0/0.grid.json" ).expect( 200 ) 108 | .expect( "content-type", "application/json; charset=utf-8", done ); 109 | } ); 110 | 111 | it( "errors on all other formats format", done => { 112 | request( app ).get( "/points/1/0/0.html" ).expect( 404 ) 113 | .expect( "Invalid format", done ); 114 | } ); 115 | } ); 116 | 117 | describe( "geohash", ( ) => { 118 | it( "renders .png", done => { 119 | request( app ).get( "/geohash/1/0/0.png" ).expect( 200 ) 120 | .expect( "content-type", "image/png", done ); 121 | } ); 122 | 123 | it( "renders .grid.json", done => { 124 | request( app ).get( "/geohash/1/1/0.grid.json" ).expect( 200 ) 125 | .expect( "content-type", "application/json; charset=utf-8", done ); 126 | } ); 127 | 128 | it( "renders .torque.json", done => { 129 | request( app ).get( "/geohash/1/0/0.torque.json" ) 130 | .expect( res => { 131 | expect( res.text ).to.include( "x__uint8" ); 132 | expect( res.text ).to.include( "y__uint8" ); 133 | expect( res.text ).to.include( "vals__uint8" ); 134 | } ) 135 | .expect( 200 ) 136 | .expect( "content-type", "text/html; charset=utf-8", done ); 137 | } ); 138 | 139 | it( "errors on all other formats format", done => { 140 | request( app ).get( "/geohash/1/0/0.html" ).expect( 404 ) 141 | .expect( "Invalid format", done ); 142 | } ); 143 | } ); 144 | 145 | describe( "geotile", ( ) => { 146 | it( "renders .png", done => { 147 | request( app ).get( "/geotile/1/0/0.png" ).expect( 200 ) 148 | .expect( "content-type", "image/png", done ); 149 | } ); 150 | 151 | it( "renders .grid.json", done => { 152 | request( app ).get( "/geotile/1/1/0.grid.json" ).expect( 200 ) 153 | .expect( "content-type", "application/json; charset=utf-8", done ); 154 | } ); 155 | 156 | it( "errors on all other formats format", done => { 157 | request( app ).get( "/geotile/1/0/0.html" ).expect( 404 ) 158 | .expect( "Invalid format", done ); 159 | } ); 160 | } ); 161 | 162 | describe( "prepareStyle", ( ) => { 163 | it( "renders errors", done => { 164 | app = Mapper.server( _.assignIn( helpers.testConfig( ), { 165 | prepareStyle: ( ) => { 166 | const e = new Error( ); 167 | e.status = 501; 168 | e.message = "fail"; 169 | throw e; 170 | } 171 | } ) ); 172 | app.get( "/:style/:zoom/:x/:y.:format([a-z.]+)", Mapper.route ); 173 | request( app ).get( "/points/1/0/0.png" ).expect( 501 ) 174 | .expect( "fail", done ); 175 | } ); 176 | 177 | it( "errors on bad styles", done => { 178 | app = Mapper.server( _.assignIn( helpers.testConfig( ), { 179 | prepareStyle: req => { 180 | req.style = "nonsense"; 181 | } 182 | } ) ); 183 | app.get( "/:style/:zoom/:x/:y.:format([a-z.]+)", Mapper.route ); 184 | request( app ).get( "/points/1/0/0.png" ).expect( 500 ) 185 | .expect( "Error", done ); 186 | } ); 187 | } ); 188 | 189 | describe( "renderError", ( ) => { 190 | it( "defaults to 500 Error", done => { 191 | app = Mapper.server( _.assignIn( helpers.testConfig( ), { 192 | prepareQuery: ( ) => { 193 | throw new Error( ); 194 | } 195 | } ) ); 196 | app.get( "/:style/:zoom/:x/:y.:format([a-z.]+)", Mapper.route ); 197 | request( app ).get( "/points/1/0/0.png" ).expect( 500 ) 198 | .expect( "Error", done ); 199 | } ); 200 | } ); 201 | 202 | describe( "defaults", ( ) => { 203 | it( "creates a default config", ( ) => { 204 | app = Mapper.server( ); 205 | expect( global.config.environment ).to.eql( "development" ); 206 | expect( global.config.tileSize ).to.eql( 256 ); 207 | expect( global.config.debug ).to.eql( true ); 208 | } ); 209 | } ); 210 | } ); 211 | -------------------------------------------------------------------------------- /test/elastic_request.js: -------------------------------------------------------------------------------- 1 | /* eslint no-underscore-dangle: 0 */ 2 | 3 | const chai = require( "chai" ); 4 | const chaiAsPromised = require( "chai-as-promised" ); 5 | const _ = require( "lodash" ); 6 | const ElasticRequest = require( "../lib/elastic_request" ); 7 | 8 | const { expect } = chai; 9 | chai.use( chaiAsPromised ); 10 | 11 | describe( "ElasticRequest", ( ) => { 12 | describe( "search", ( ) => { 13 | it( "returns an error with a malformed query", async ( ) => { 14 | await expect( ElasticRequest.search( { made: "up" } ) ).to.be 15 | .rejectedWith( Error, "index_not_found_exception" ); 16 | } ); 17 | } ); 18 | 19 | describe( "boundingBoxFilter", ( ) => { 20 | it( "enlarges the boundary for better tile edges", ( ) => { 21 | expect( ElasticRequest.boundingBoxFilter( { }, [0, 0, 1, 1], true ) ).to.eql( { 22 | geo_bounding_box: { 23 | location: { 24 | bottom_left: [-0.07, -0.07], 25 | top_right: [1.07, 1.07] 26 | }, 27 | type: "indexed" 28 | } 29 | } ); 30 | } ); 31 | 32 | it( "can skip smoothing", ( ) => { 33 | expect( ElasticRequest.boundingBoxFilter( { }, [0, 0, 1, 1], false ) ).to.eql( { 34 | geo_bounding_box: { 35 | location: { 36 | bottom_left: [0, 0], 37 | top_right: [1, 1] 38 | }, 39 | type: "indexed" 40 | } 41 | } ); 42 | } ); 43 | 44 | it( "creates a conditional query for dateline wrapping bboxes", ( ) => { 45 | expect( ElasticRequest.boundingBoxFilter( { }, [179, 1, -179, 2], false ) ).to.eql( { 46 | bool: { 47 | should: [ 48 | { 49 | geo_bounding_box: { 50 | location: { 51 | bottom_left: [179, 1], 52 | top_right: [180, 2] 53 | }, 54 | type: "indexed" 55 | } 56 | }, 57 | { 58 | geo_bounding_box: { 59 | location: { 60 | bottom_left: [-180, 1], 61 | top_right: [-179, 2] 62 | }, 63 | type: "indexed" 64 | } 65 | } 66 | ] 67 | } 68 | } ); 69 | } ); 70 | } ); 71 | 72 | describe( "applyBoundingBoxFilter", ( ) => { 73 | it( "can add to a prefiltered query", ( ) => { 74 | const req = { 75 | params: { 76 | x: 1, 77 | y: 1, 78 | zoom: 1 79 | }, 80 | elastic_query: { 81 | query: { 82 | bool: { 83 | filter: [] 84 | } 85 | } 86 | } 87 | }; 88 | expect( req.elastic_query.query.bool.filter.length ).to.eql( 0 ); 89 | ElasticRequest.applyBoundingBoxFilter( req ); 90 | expect( req.elastic_query.query.bool.filter.length ).to.eql( 1 ); 91 | } ); 92 | 93 | it( "can add to a prefiltered bool", ( ) => { 94 | const req = { 95 | params: { 96 | x: 1, 97 | y: 1, 98 | zoom: 1 99 | }, 100 | elastic_query: { 101 | query: { 102 | bool: { 103 | filter: [ 104 | { something: "different" } 105 | ] 106 | } 107 | } 108 | } 109 | }; 110 | expect( _.size( req.elastic_query.query.bool.filter ) ).to.eql( 1 ); 111 | ElasticRequest.applyBoundingBoxFilter( req ); 112 | expect( _.size( req.elastic_query.query.bool.filter ) ).to.eql( 2 ); 113 | } ); 114 | } ); 115 | 116 | describe( "geohashPrecision", ( ) => { 117 | it( "returns the proper percision for a zoom", ( ) => { 118 | expect( ElasticRequest.geohashPrecision( 1 ) ).to.eql( 3 ); 119 | expect( ElasticRequest.geohashPrecision( 2 ) ).to.eql( 3 ); 120 | expect( ElasticRequest.geohashPrecision( 3 ) ).to.eql( 4 ); 121 | expect( ElasticRequest.geohashPrecision( 4 ) ).to.eql( 4 ); 122 | expect( ElasticRequest.geohashPrecision( 5 ) ).to.eql( 4 ); 123 | expect( ElasticRequest.geohashPrecision( 6 ) ).to.eql( 5 ); 124 | expect( ElasticRequest.geohashPrecision( 7 ) ).to.eql( 5 ); 125 | expect( ElasticRequest.geohashPrecision( 8 ) ).to.eql( 6 ); 126 | expect( ElasticRequest.geohashPrecision( 9 ) ).to.eql( 6 ); 127 | expect( ElasticRequest.geohashPrecision( 10 ) ).to.eql( 6 ); 128 | expect( ElasticRequest.geohashPrecision( 11 ) ).to.eql( 7 ); 129 | expect( ElasticRequest.geohashPrecision( 12 ) ).to.eql( 8 ); 130 | expect( ElasticRequest.geohashPrecision( 13 ) ).to.eql( 9 ); 131 | expect( ElasticRequest.geohashPrecision( 14 ) ).to.eql( 9 ); 132 | expect( ElasticRequest.geohashPrecision( 15 ) ).to.eql( 10 ); 133 | expect( ElasticRequest.geohashPrecision( 16 ) ).to.eql( 12 ); 134 | } ); 135 | } ); 136 | 137 | describe( "geohashAggregation", ( ) => { 138 | it( "returns the proper aggregation hash based on zoom", ( ) => { 139 | expect( ElasticRequest.geohashAggregation( { 140 | params: { zoom: 15 }, 141 | elastic_query: { fields: ElasticRequest.defaultMapFields( ) } 142 | } ) ).to.eql( { 143 | zoom1: { 144 | geohash_grid: { 145 | field: "location", 146 | size: 30000, 147 | precision: 10 148 | }, 149 | aggs: { 150 | geohash: { 151 | top_hits: { 152 | sort: { 153 | id: { 154 | order: "desc" 155 | } 156 | }, 157 | _source: false, 158 | size: 1 159 | } 160 | } 161 | } 162 | } 163 | } ); 164 | } ); 165 | 166 | it( "returns the source if requested", ( ) => { 167 | const agg = ElasticRequest.geohashAggregation( { 168 | query: { source: true }, 169 | params: { zoom: 15 }, 170 | elastic_query: { fields: ElasticRequest.defaultMapFields( ) } 171 | } ); 172 | expect( agg.zoom1.aggs.geohash.top_hits._source ).to.be.true; 173 | } ); 174 | } ); 175 | } ); 176 | -------------------------------------------------------------------------------- /test/lib/helpers.js: -------------------------------------------------------------------------------- 1 | const ElasticRequest = require( "../../lib/elastic_request" ); 2 | 3 | const helpers = { }; 4 | 5 | helpers.testConfig = ( ) => ( 6 | { environment: "test", debug: false } 7 | ); 8 | 9 | helpers.rebuildTestIndex = async ( ) => { 10 | ElasticRequest.createClient( ); 11 | const indexOptions = { index: global.config.elasticsearch.searchIndex }; 12 | if ( await ElasticRequest.esClient.indices.exists( indexOptions ) ) { 13 | await ElasticRequest.esClient.indices.delete( indexOptions ); 14 | await helpers.createTestIndex( ); 15 | } else { 16 | await helpers.createTestIndex( ); 17 | } 18 | }; 19 | 20 | helpers.createTestIndex = async ( ) => { 21 | const body = { 22 | properties: { 23 | id: { type: "integer" }, 24 | user: { 25 | properties: { 26 | name: { type: "text" } 27 | } 28 | }, 29 | location: { type: "geo_point" }, 30 | geojson: { type: "geo_shape" }, 31 | observed_on_details: { 32 | properties: { 33 | month: { type: "byte" } 34 | } 35 | } 36 | } 37 | }; 38 | const indexOptions = { index: global.config.elasticsearch.searchIndex }; 39 | await ElasticRequest.esClient.indices.create( indexOptions ); 40 | await ElasticRequest.esClient.indices.putMapping( { 41 | ...indexOptions, 42 | body 43 | } ); 44 | await ElasticRequest.esClient.create( { 45 | ...indexOptions, 46 | refresh: true, 47 | type: "_doc", 48 | id: "1", 49 | body: { 50 | id: 1, 51 | location: "51.18,-1.83", 52 | geojson: { type: "Point", coordinates: [-1.83, 51.18] }, 53 | observed_on_details: { 54 | month: 1 55 | } 56 | } 57 | } ); 58 | }; 59 | 60 | helpers.deleteTestIndex = async ( ) => { 61 | ElasticRequest.createClient( ); 62 | const indexOptions = { index: global.config.elasticsearch.searchIndex }; 63 | if ( await ElasticRequest.esClient.indices.exists( indexOptions ) ) { 64 | await ElasticRequest.esClient.indices.delete( indexOptions ); 65 | } 66 | }; 67 | 68 | module.exports = helpers; 69 | -------------------------------------------------------------------------------- /test/map_generator.js: -------------------------------------------------------------------------------- 1 | const { expect } = require( "chai" ); 2 | const MapGenerator = require( "../lib/map_generator" ); 3 | 4 | describe( "MapGenerator", ( ) => { 5 | describe( "createMapTemplate", ( ) => { 6 | it( "fails on invalid styles", async ( ) => { 7 | await expect( MapGenerator.createMapTemplate( 8 | { params: { x: 0, y: 0, zoom: 1 }, style: "nonsense" } 9 | ) ).to.be.rejectedWith( "Unable to process some data while parsing" ); 10 | } ); 11 | } ); 12 | 13 | describe( "basicEscape", ( ) => { 14 | it( "removes single quotes", ( ) => { 15 | expect( MapGenerator.basicEscape( "How's things" ) ).to.eq( "\"Hows things\"" ); 16 | } ); 17 | 18 | it( "turn double quotes to single", ( ) => { 19 | expect( MapGenerator.basicEscape( "a \"quoted\" value" ) ).to.eq( "\"a 'quoted' value\"" ); 20 | } ); 21 | } ); 22 | 23 | describe( "createMapTemplate", ( ) => { 24 | it( "returns errors", ( ) => { 25 | MapGenerator.createMapTemplate( { style: "nothing" }, err => { 26 | expect( err.message ).to.eq( 27 | "Cannot read properties of undefined (reading 'format')" 28 | ); 29 | } ); 30 | } ); 31 | } ); 32 | 33 | describe( "geojsonDatasource", ( ) => { 34 | it( "prepares data for GeoJSON responses", ( ) => { 35 | const features = [ 36 | { 37 | type: "Feature", 38 | geometry: { 39 | type: "Point", 40 | coordinates: [12, 11] 41 | }, 42 | properties: { 43 | name: "One" 44 | } 45 | }, 46 | { 47 | type: "Feature", 48 | geometry: { 49 | type: "Point", 50 | coordinates: [22, 21] 51 | }, 52 | properties: { 53 | name: "Two" 54 | } 55 | } 56 | ]; 57 | const d = MapGenerator.geojsonDatasource( features ); 58 | expect( d.extent( ) ).to.deep.eq( [12, 11, 22, 21] ); 59 | } ); 60 | } ); 61 | } ); 62 | -------------------------------------------------------------------------------- /test/styles.js: -------------------------------------------------------------------------------- 1 | const { expect } = require( "chai" ); 2 | const Styles = require( "../lib/styles" ); 3 | 4 | describe( "Styles", ( ) => { 5 | describe( "points", ( ) => { 6 | it( "returns a style with the right name", ( ) => { 7 | expect( Styles.points( ) ).to.include( "Style name='style'" ); 8 | } ); 9 | } ); 10 | } ); 11 | --------------------------------------------------------------------------------