├── .gitignore ├── LICENSE ├── README.md ├── app ├── app.js ├── bin │ └── www ├── conf │ ├── conf.js │ └── schema.sql ├── mapper │ ├── bing-mapper.js │ └── osm-mapper.js ├── model │ ├── bounds.js │ └── tile.js ├── package.json ├── routes │ └── index.js ├── run-debug.sh ├── run.sh ├── service │ ├── mbtiles-generator-service.js │ └── mbtiles-status-service.js ├── start-debug.sh ├── start.sh └── util │ └── projection-utils.js └── docker ├── base ├── Dockerfile ├── README.md ├── build.sh └── push.sh └── server ├── Dockerfile ├── README.md ├── app.sh ├── build.sh ├── conf.js.template └── push.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | npm-debug.log 3 | node_modules 4 | *.iml 5 | .DS_Store 6 | data 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright eBusiness Information 2015 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 4 | 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 7 | 8 | ------------- 9 | 10 | Includes mapbox sqlite3 (https://github.com/mapbox/node-sqlite3) licensed under custom license. 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MBTiles generator 2 | 3 | [Version: 0.3.0] 4 | 5 | A project designed to provide a simple API for generating MBTiles for a bounding box, fetching data from a remote tile-server. 6 | 7 | **Warning: this tool can generate heavy load on tile-servers. Use it responsibly.** 8 | 9 | ## Features 10 | * REST API or command line tool 11 | * Lightweight, easily configurable 12 | * Multiple map providers supported (osm and bing) 13 | * Additional providers can easily be declared 14 | 15 | ## Howto 16 | * Install nodejs 17 | * in the repository root, type ``npm install`` 18 | * To launch the server mode, execute ``./start.sh`` or ``./start-debug.sh`` 19 | * To use the cli, execute ``./run.sh --left=2.31760654 --bottom=48.8243829 --right=2.358607 --top=48.8513625`` 20 | 21 | ## Configuration 22 | To configure your application, edit the conf/Conf.js file. 23 | The file may look like this 24 | ```javascript 25 | module.exports = { 26 | "tileServer": {"type": "...", "endpoint": "..."}, // Tile server specs 27 | "minZoom": 5, // Minimum zoom level to compute for an mbtile. 28 | "maxZoom": 17, // Maximum zoom level to compute for an mbtile 29 | "timeout": 300000, // Http Timeout in milliseconds (Server-mode only) 30 | "maxArea": 16, // Maximum MBTiles covering area in square kilometers. Will reject all oversized requests. 0 to disable. 31 | }; 32 | ``` 33 | Two tile providers are currently supported : 34 | **OpenStreetMap**: 35 | In this case, provide the following tileServer (replace your endpoint): 36 | ```javascript 37 | "tileServer": {"type": "osm", "endpoint": "http://your-tileserver/{z}/{x}/{y}.png"} 38 | ``` 39 | Layers are also supported, using them as: 40 | ```javascript 41 | "tileServer": {"type": "osm", "endpoint": "http://your-tileserver/{layer}/{z}/{x}/{y}.png"} 42 | ``` 43 | **Bing**: 44 | In this case, provide the following tileServer (replace your style and ApiKey): 45 | ```javascript 46 | "tileServer": { 47 | "type": "bing", 48 | "endpoint": "http://dev.virtualearth.net/REST/V1/Imagery/Metadata/Aerial?mapVersion=v1&output=json&key=myApiKey", 49 | "culture": "fr" 50 | } 51 | ``` 52 | 53 | ## Server API 54 | The server runs on port 2999 and listens to the following endpoints: 55 | 56 | ### Synchronous endpoints 57 | **[GET]**``/mbtiles``: 58 | Requires 4 parameters: left, bottom, right, top. 59 | Optional parameter: layer. 60 | Returns: the mbtiles file 61 | **Example**: ``http://localhost:2999/mbtiles?left=2.31760654&bottom=48.8243829&right=2.358607&top=48.8513625`` 62 | Will synchronously prepare your mbtiles file and return it in the http response when done. This process can take up to a few minutes. 63 | 64 | 65 | ### Asynchronous endpoints 66 | **[GET]**``/mbtiles/async``: 67 | Requires 4 parameters: left, bottom, right, top. 68 | Optional parameter: layer. 69 | Returns: a json {"token": "your-token"} 70 | **Example**: ``http://localhost:2999/mbtiles/async?left=2.31760654&bottom=48.8243829&right=2.358607&top=48.8513625`` 71 | Will asynchronously launch your mbtiles computation and return a unique token. 72 | 73 | **[GET]**``/mbtiles/status/:token``: 74 | Requires one path variable: token (your token provided in the previous endpoint) 75 | Returns: {"status": "generating|done|downloaded", "progress":11} the status and the progress percentage. 76 | **Example**: ``http://localhost:2999/mbtiles/status/969396ca-9747-469b-b0a0-32da12dc4ab6`` 77 | Retrieve the status of your current mbtiles computation 78 | 79 | **[GET]**``/mbtiles/download/:token``: 80 | Requires one path variable: token (your token provided in the previous endpoint) 81 | Returns: the mbtiles file 82 | **Example**: ``http://localhost:2999/mbtiles/download/969396ca-9747-469b-b0a0-32da12dc4ab6`` 83 | Retrieve the computed mbtiles file 84 | 85 | 86 | ## CLI 87 | The CLI needs 4 parameters to be provided to work: --left, --bottom, --right, --top. 88 | It supports one optional parameter: --layer. 89 | Use ./run.sh --help for more information. 90 | 91 | ## Docker runtime 92 | To run the MBTiles generator in an even simpler environment, simply execute: 93 | 94 | ```sh 95 | docker run -d -p 2999:2999 -e "APP_MODE=server" -e "TILESERVER_TYPE=osm" -e "TILESERVER_ENDPOINT=http://mytileserver.org/{z}/{x}/{y}.png" -e "APP_TIMEOUT=300" -e "APP_MINZOOM=3" -e "APP_MAXZOOM=17" -e "APP_MAXAREA=16" mapsquare/mbtiles-generator-server 96 | ``` 97 | 98 | ENV variables: 99 | * APP_MODE: the execution mode. Valid values: server, command. *server* will launch a server on port 2999, *command* creates the requested MBTiles and outputs in the container /opt/app/data folder. 100 | * TILESERVER_TYPE: the tile provider. Valid values: osm, bing. 101 | * TILESERVER_ENDPOINT: depends on the provider. see app/mapper files and examples. 102 | * APP_TIMEOUT: timeout in seconds for server mode. 103 | * APP_MAXAREA: max supported area in square kilometers for an MBTiles file. 104 | * APP_MINZOOM: minzoom to compute the MBTiles. 105 | * APP_MAXZOOM: maxzoom to compute the MBTiles. 106 | 107 | ## License 108 | 109 | Copyright 2015 eBusiness Information 110 | 111 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 112 | 113 | http://www.apache.org/licenses/LICENSE-2.0 114 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 115 | -------------------------------------------------------------------------------- /app/app.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 eBusiness Information 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | /** 14 | * Created by Loïc Ortola on 16/10/2015. 15 | * MBTiles main application 16 | */ 17 | // Conf & Utils 18 | var Conf = require('./conf/conf'); 19 | // Classes 20 | var Bounds = require('./model/bounds'); 21 | // Logging 22 | var logger = require('morgan'); 23 | // Imports 24 | var path = require('path'); 25 | var express = require('express'); 26 | var cookieParser = require('cookie-parser'); 27 | var bodyParser = require('body-parser'); 28 | var routes = require('./routes/index'); 29 | var minimist = require('minimist'); 30 | var mbTilesGeneratorService = require('./service/mbtiles-generator-service'); 31 | 32 | var app = express(); 33 | 34 | var argv = minimist(process.argv.slice(2)); 35 | 36 | app.use(logger('dev')); 37 | app.use(bodyParser.json()); 38 | app.use(bodyParser.urlencoded({extended: false})); 39 | app.use(cookieParser()); 40 | app.use('/', routes); 41 | 42 | // catch 404 and forward to error handler 43 | app.use(function (req, res, next) { 44 | var err = new Error('Not Found'); 45 | err.status = 404; 46 | next(err); 47 | }); 48 | 49 | // error handlers 50 | app.use(function (err, req, res, next) { 51 | res.status(err.status || 500); 52 | res.send({ 53 | message: err.message, 54 | error: {} 55 | }); 56 | }); 57 | 58 | if (process.argv.length > 2) { 59 | if (argv.help) { 60 | console.log('Following arguments must be used:'); 61 | console.log('--left=00.000000 (left coordinate in degrees, WGS:84 format valid values: -180.0 to 180.0 .)'); 62 | console.log('--right=00.000000 (right coordinate in degrees, WGS:84 format valid values: -180.0 to 180.0 .)'); 63 | console.log('--top=00.000000 (top coordinate in degrees, WGS:84 format valid values: -85.0 to 85.0 .)'); 64 | console.log('--bottom=00.000000 (bottom coordinate in degrees, WGS:84 format valid values: -85.0 to 85.0 .)'); 65 | console.log('Following arguments can be used:'); 66 | console.log('--min-zoom=z (valid values: 0 to 22.)'); 67 | console.log('The minimum zoom level. Will override the default configuration'); 68 | console.log('--max-zoom=z (valid values: 0 to 22 and > min-zoom.)'); 69 | console.log('The maximum zoom level. Will override the default configuration'); 70 | console.log('--layer=mylayer'); 71 | console.log('The layer to render.'); 72 | console.log('Example:'); 73 | console.log('./run.sh --left=2.31760654 --bottom=48.8243829 --right=2.358607 --top=48.8513625'); 74 | console.log('./run.sh --left=2.31760654 --bottom=48.8243829 --right=2.358607 --top=48.8513625 --max-zoom=12 --layer=roads'); 75 | process.exit(); 76 | } 77 | if (argv['min-zoom']) { 78 | Conf.minZoom = argv['min-zoom']; 79 | } 80 | if (argv['max-zoom']) { 81 | Conf.maxZoom = argv['max-zoom']; 82 | } 83 | var left = argv['left']; 84 | var bottom = argv['bottom']; 85 | var right = argv['right']; 86 | var top = argv['top']; 87 | var layer = argv['layer']; 88 | var bounds = new Bounds(left, bottom, right, top); 89 | // Dirty wait for modules to init. 90 | setTimeout(function() { 91 | mbTilesGeneratorService.requestMBTilesSync(bounds, layer) 92 | .then(function () { 93 | process.exit(); 94 | }, function (result) { 95 | console.error('Process will exit with error: ' + result); 96 | process.exit(-1); 97 | }); 98 | }, 1000); 99 | 100 | } 101 | 102 | module.exports = app; -------------------------------------------------------------------------------- /app/bin/www: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 eBusiness Information 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | /** 14 | * Module dependencies. 15 | */ 16 | 17 | var app = require('../app'); 18 | var Conf = require('../conf/conf'); 19 | var debug = require('debug')('mbt:server'); 20 | var http = require('http'); 21 | var cluster = require('cluster'); 22 | var numCPUs = require('os').cpus().length; 23 | var mbTilesGeneratorService = require('../service/mbtiles-generator-service'); 24 | var mbTilesStatusService = require('../service/mbtiles-status-service'); 25 | var EventEmitter = require('events'); 26 | var emitter = new EventEmitter(); 27 | 28 | var workers = []; 29 | 30 | var FOUR_HOURS = 4 * 3600 * 1000; 31 | 32 | /** 33 | * Get port from environment and store in Express. 34 | */ 35 | 36 | var port = normalizePort(process.env.PORT || '2999'); 37 | var env = process.env.NODE_ENV || 'development'; 38 | app.set('port', port); 39 | app.set('env', env); 40 | 41 | 42 | if (cluster.isMaster) { 43 | 44 | // Fork workers. 45 | for (var i = 0; i < numCPUs; i++) { 46 | var worker = cluster.fork(); 47 | workers[i] = worker; 48 | // Listen to messages on workers 49 | worker.on('message', function (msg) { 50 | // MBTiles status dispatch to workers 51 | if (msg.tag === 'mbtiles-status-broadcast') { 52 | debug('Received message from master: ' + JSON.stringify(msg)); 53 | msg.tag = 'mbtiles-status-push'; 54 | workers.forEach(function (w) { 55 | w.send(msg); 56 | }); 57 | mbTilesStatusService.receiveUpdate(msg); 58 | } 59 | }); 60 | } 61 | 62 | // We are in server-mode. Therefore, we will clear the data directory every 24 hours 63 | setInterval(function() { 64 | mbTilesGeneratorService.removeOldMBTiles(); 65 | }, FOUR_HOURS); 66 | 67 | cluster.on('listening', function (worker, address) { 68 | debug('PID(' + worker.process.pid + ') Cluster worker now connected'); 69 | }); 70 | 71 | cluster.on('exit', function (worker, code, signal) { 72 | console.log('worker ' + worker.process.pid + ' died'); 73 | }); 74 | 75 | console.log('Application is running, connected to endpoint ' + Conf.tileServer.endpoint + '. Pretty cool huh?'); 76 | 77 | } else { 78 | /** 79 | * Create HTTP server. 80 | */ 81 | 82 | var server = http.createServer(app); 83 | 84 | /** 85 | * Listen on provided port, on all network interfaces. 86 | */ 87 | 88 | server.listen(port); 89 | server.on('error', onError); 90 | server.on('listening', onListening); 91 | server.setTimeout(Conf.timeout); 92 | 93 | } 94 | 95 | 96 | /** 97 | * Normalize a port into a number, string, or false. 98 | */ 99 | 100 | function normalizePort(val) { 101 | var port = parseInt(val, 10); 102 | 103 | if (isNaN(port)) { 104 | // named pipe 105 | return val; 106 | } 107 | 108 | if (port >= 0) { 109 | // port number 110 | return port; 111 | } 112 | 113 | return false; 114 | } 115 | 116 | /** 117 | * Event listener for HTTP server "error" event. 118 | */ 119 | 120 | function onError(error) { 121 | if (error.syscall !== 'listen') { 122 | throw error; 123 | } 124 | 125 | var bind = typeof port === 'string' 126 | ? 'Pipe ' + port 127 | : 'Port ' + port; 128 | 129 | // handle specific listen errors with friendly messages 130 | switch (error.code) { 131 | case 'EACCES': 132 | console.error(bind + ' requires elevated privileges'); 133 | process.exit(1); 134 | break; 135 | case 'EADDRINUSE': 136 | console.error(bind + ' is already in use'); 137 | process.exit(1); 138 | break; 139 | default: 140 | console.error('Error occured: ' + error); 141 | //throw error; 142 | } 143 | } 144 | 145 | /** 146 | * Event listener for HTTP server "listening" event. 147 | */ 148 | 149 | function onListening() { 150 | 151 | } 152 | -------------------------------------------------------------------------------- /app/conf/conf.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 eBusiness Information 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | /** 14 | * Created by Loïc Ortola on 16/10/2015. 15 | * MBTiles Generator Configuration 16 | */ 17 | module.exports = { 18 | "tileServer": {"type": "osm", "endpoint": "http://a.tile.openstreetmap.fr/{layer}/{z}/{x}/{y}.png"}, 19 | "minZoom": 4, 20 | "maxZoom": 17, 21 | "timeout": 500000, // Timeout in milliseconds 22 | "maxArea": 12 // Maximum area, in square kilometers 23 | }; -------------------------------------------------------------------------------- /app/conf/schema.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 eBusiness Information 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob); 14 | CREATE TABLE metadata (name text, value text); 15 | -------------------------------------------------------------------------------- /app/mapper/bing-mapper.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 eBusiness Information 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | /** 14 | * Created by Loïc Ortola on 19/10/2015. 15 | * Bing mapper file 16 | */ 17 | // Conf & Utils 18 | var Conf = require('../conf/conf'); 19 | // Imports 20 | var http = require('http'); 21 | 22 | // Variables 23 | var tileServer; 24 | 25 | /** 26 | * OSM Mapper computes a bing tile request URL from a tile metadata 27 | * http://tile-server.net/h{quadkey}.jpeg?g=4258&mkt=en 28 | * @param t the requested tile 29 | * @returns {string} 30 | */ 31 | var getTileUrl = function (t) { 32 | return tileServer.replace('{quadkey}', tileXYToQuadKey(t)); 33 | }; 34 | 35 | /** 36 | * Returns the tile extension type. Can be .png or .jpg according to MBTiles spec 37 | * @returns {string} 38 | */ 39 | var getExtension = function () { 40 | return 'jpeg'; 41 | }; 42 | 43 | /** 44 | * Compute quadKey from tileXY 45 | * @param t 46 | * @returns {string} 47 | */ 48 | var tileXYToQuadKey = function (t) { 49 | var quadKey = ''; 50 | for (var i = t.z; i > 0; i--) { 51 | var digit = 0; 52 | var mask = 1 << (i - 1); 53 | if ((t.x & mask)) { 54 | digit++; 55 | } 56 | if ((t.y & mask)) { 57 | digit += 2; 58 | } 59 | quadKey += digit; 60 | } 61 | return quadKey; 62 | }; 63 | 64 | /** 65 | * Compute tileServerUrl from bing endpoint. 66 | * A valid bing endpoint looks like this: 67 | http://dev.virtualearth.net/REST/V1/Imagery/Metadata/{yourStyle}?mapVersion=v1&output=json&key={yourAPIKey} 68 | replacing endpoint implies that you replace {yourStyle} and {yourAPIKey} by appropriate values. 69 | External documentation available on https://msdn.microsoft.com/en-us/library/ff701716.aspx 70 | @returns {string} the bing tileServerUrl 71 | looks like this: http://ecn.t1.tiles.virtualearth.net/tiles/h{quadkey}.jpeg?g=4258&mkt=en 72 | */ 73 | var getTileServerUrl = function () { 74 | http.get(Conf.tileServer.endpoint, function (res) { 75 | if (res.statusCode != 200) { 76 | console.error('Ooops'); 77 | throw new Error('Cannot retrieve tileServerUrl. Please check that your bing endpoint is correct.'); 78 | } 79 | res.setEncoding('utf-8'); 80 | var body = ''; 81 | res.on('data', function (chunk) { 82 | body += chunk; 83 | }); 84 | res.on('end', function () { 85 | body = JSON.parse(body); 86 | var tileServerUrl = body.resourceSets[0].resources[0].imageUrl; 87 | var subDomains = body.resourceSets[0].resources[0].imageUrlSubdomains; 88 | tileServer = tileServerUrl 89 | .replace('{culture}', Conf.tileServer.culture || 'en') 90 | .replace('{subdomain}', subDomains[Math.floor(Math.random() * subDomains.length)]); 91 | }) 92 | }).on('error', function(err) { 93 | throw err; 94 | }); 95 | }; 96 | 97 | 98 | // Init 99 | getTileServerUrl(); 100 | 101 | // Exports 102 | module.exports = { 103 | "getTileUrl": getTileUrl, 104 | "getExtension": getExtension 105 | }; -------------------------------------------------------------------------------- /app/mapper/osm-mapper.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 eBusiness Information 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | /** 14 | * Created by Loïc Ortola on 19/10/2015. 15 | * OSM Mapper file 16 | */ 17 | // Conf & Utils 18 | var Conf = require('../conf/conf'); 19 | 20 | var extension = Conf.tileServer.endpoint.substr(Conf.tileServer.endpoint.lastIndexOf('.')); 21 | 22 | /** 23 | * OSM Mapper computes an OSM tile request URL from a tile metadata 24 | * http://tile-server.com/{z}/{x}/{y}.png 25 | * OR http://tile-server.com/{layer}/{z}/{x}/{y}.png 26 | * @param t the requested tile 27 | * @returns {string} 28 | */ 29 | var getTileUrl = function(t) { 30 | if (Conf.tileServer.endpoint.indexOf('{layer}') == -1) { 31 | return Conf.tileServer.endpoint.replace('{z}',t.z).replace('{x}', t.x).replace('{y}', t.y); 32 | } else { 33 | return Conf.tileServer.endpoint.replace('{layer}',t.layer).replace('{z}',t.z).replace('{x}', t.x).replace('{y}', t.y); 34 | } 35 | }; 36 | 37 | /** 38 | * Returns the tile extension type. Can be .png or .jpg according to MBTiles spec 39 | * @returns {string} 40 | */ 41 | var getExtension = function() { 42 | return extension; 43 | }; 44 | 45 | // Exports 46 | module.exports = { 47 | "getTileUrl": getTileUrl, 48 | "getExtension": getExtension 49 | }; -------------------------------------------------------------------------------- /app/model/bounds.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 eBusiness Information 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | /** 14 | * Created by Loïc Ortola on 16/10/2015. 15 | * Bounds model 16 | */ 17 | // Constructor 18 | function Bounds(left, bottom, right, top) { 19 | this.left = parseFloat(left); 20 | this.bottom = parseFloat(bottom); 21 | this.right = parseFloat(right); 22 | this.top = parseFloat(top); 23 | } 24 | 25 | // Export class 26 | module.exports = Bounds; -------------------------------------------------------------------------------- /app/model/tile.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 eBusiness Information 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | /** 14 | * Created by Loïc Ortola on 16/10/2015. 15 | * Tile model 16 | */ 17 | // Constructor 18 | function Tile(x, y, z, layer, ext) { 19 | this.x = x; 20 | this.y = y; 21 | this.z = z; 22 | this.layer = layer || 'default'; 23 | this.ext = ext; 24 | var tileCountXY = Math.pow(2, z); 25 | if (this.x >= tileCountXY || this.y >= tileCountXY) { 26 | throw new Error('Illegal parameters for tile'); 27 | } 28 | } 29 | 30 | // Static Methods 31 | Tile.getId = function (x, y, z, layer, ext) { 32 | return x + '-' + y + '-' + z + '-' + layer + '-' + ext; 33 | }; 34 | 35 | Tile.fromId = function (id) { 36 | //id <=> number-number-number-layer-ext 37 | var idArray = id.split('-'); 38 | return new Tile(idArray[0], idArray[1], idArray[2], idArray[3], idArray[4]); 39 | }; 40 | 41 | Tile.copy = function (t) { 42 | return new Tile(t.x, t.y, t.z, t.layer, t.ext); 43 | }; 44 | 45 | // Methods 46 | Tile.prototype.getId = function () { 47 | return this.x + '-' + this.y + '-' + this.z + '-' + this.layer + '-' + this.ext; 48 | }; 49 | 50 | // Export class 51 | module.exports = Tile; -------------------------------------------------------------------------------- /app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mbtile-generator", 3 | "version": "0.2.0", 4 | "private": false, 5 | "scripts": { 6 | "start": "node ./bin/www" 7 | }, 8 | "dependencies": { 9 | "body-parser": "~1.13.2", 10 | "cookie-parser": "~1.3.5", 11 | "debug": "~2.2.0", 12 | "express": "~4.13.1", 13 | "minimist": "^1.2.0", 14 | "morgan": "~1.6.1", 15 | "sqlite3": "^3.1.0", 16 | "step": "0.0.6", 17 | "hashmap": "^2.0.4", 18 | "node-uuid": "^1.4.3" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /app/routes/index.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 eBusiness Information 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | /** 14 | * Created by Loïc Ortola on 16/10/2015. 15 | * Routes 16 | */ 17 | // Conf & Utils 18 | var Conf = require('../conf/conf'); 19 | var ProjectionUtils = require('../util/projection-utils'); 20 | // Classes 21 | var Bounds = require('../model/bounds'); 22 | // Logging 23 | var debug = require('debug')('mbt:server'); 24 | // Imports 25 | var express = require('express'); 26 | var mbTilesGeneratorService = require('../service/mbtiles-generator-service'); 27 | var mbTilesStatusService = require('../service/mbtiles-status-service'); 28 | var router = express.Router(); 29 | 30 | /** 31 | * Main route 32 | */ 33 | router.get('/mbtiles', function (req, res, next) { 34 | var left = req.query.left; 35 | var bottom = req.query.bottom; 36 | var right = req.query.right; 37 | var top = req.query.top; 38 | var bounds = new Bounds(left, bottom, right, top); 39 | 40 | // Validate Bounds 41 | if (!ProjectionUtils.isValidBounds(bounds)) { 42 | res.set('Content-Type', 'application/json'); 43 | // Bad request 44 | res.statusCode = 400; 45 | res.send({"message": 'Wrong bounds provided. Please provide a valid set of bounds to complete request.'}); 46 | return; 47 | } 48 | 49 | // Validate Request Area 50 | var requestArea = ProjectionUtils.boundsToArea(bounds); 51 | if (Conf.maxArea && Conf.maxArea > 0 && requestArea > Conf.maxArea) { 52 | var msg = 'Requested area (' + Math.floor(requestArea) + ' km2) is superior to the application maxArea (' + Conf.maxArea + ' km2)'; 53 | console.error(msg); 54 | res.set('Content-Type', 'application/json'); 55 | // Bad request 56 | res.statusCode = 400; 57 | res.send({"message": msg}); 58 | return; 59 | } 60 | 61 | // If a layer have been requested, add it to request 62 | var layer = req.query.layer; 63 | 64 | // Wait for promise to be resolved before returning response (synchronous behaviour) 65 | mbTilesGeneratorService.requestMBTilesSync(bounds, layer) 66 | .then(function (result) { 67 | var content = new Buffer(result, 'binary'); 68 | res.set('Content-Type', 'application/x-sqlite3'); 69 | res.set('Content-Disposition', 'inline; filename="mapsquare.mbtiles"'); 70 | res.send(content); 71 | 72 | }, function (result) { 73 | res.send(result.message); 74 | }); 75 | }); 76 | 77 | /** 78 | * Main route 79 | */ 80 | router.get('/mbtiles/async', function (req, res, next) { 81 | var left = req.query.left; 82 | var bottom = req.query.bottom; 83 | var right = req.query.right; 84 | var top = req.query.top; 85 | var bounds = new Bounds(left, bottom, right, top); 86 | 87 | // Validate Bounds 88 | if (!ProjectionUtils.isValidBounds(bounds)) { 89 | res.set('Content-Type', 'application/json'); 90 | // Bad request 91 | res.statusCode = 400; 92 | res.send({"message": 'Wrong bounds provided. Please provide a valid set of bounds to complete request.'}); 93 | return; 94 | } 95 | 96 | // Validate Request Area 97 | var requestArea = ProjectionUtils.boundsToArea(bounds); 98 | if (Conf.maxArea && Conf.maxArea > 0 && requestArea > Conf.maxArea) { 99 | var msg = 'Requested area (' + Math.floor(requestArea) + ' km2) is superior to the application maxArea (' + Conf.maxArea + ' km2)'; 100 | console.error(msg); 101 | res.set('Content-Type', 'application/json'); 102 | // Bad request 103 | res.statusCode = 400; 104 | res.send({"message": msg}); 105 | return; 106 | } 107 | 108 | // If a layer have been requested, add it to request 109 | var layer = req.query.layer; 110 | 111 | // Return token 112 | var token = mbTilesGeneratorService.requestMBTiles(bounds, layer); 113 | res.set('Content-Type', 'application/json'); 114 | res.send({"token": token}); 115 | }); 116 | 117 | /** 118 | * Main route 119 | */ 120 | router.get('/mbtiles/status/:token', function (req, res, next) { 121 | var token = req.params.token; 122 | var status = mbTilesStatusService.get(token); 123 | res.set('Content-Type', 'application/json'); 124 | if (status) { 125 | // If generating, set to 202-ACCEPTED 126 | res.statusCode = 202; 127 | res.send(status); 128 | } else { 129 | // Else, not found. 130 | res.statusCode = 404; 131 | res.send({"message": "Token does not exist. please refer to a valid mbtiles token."}); 132 | } 133 | }); 134 | 135 | /** 136 | * Main route 137 | */ 138 | router.get('/mbtiles/download/:token', function (req, res, next) { 139 | var token = req.params.token; 140 | mbTilesGeneratorService.getMBTiles(token, function(result) { 141 | if (!result) { 142 | // Not ready yet. 143 | res.statusCode = 202; 144 | res.send(); 145 | return; 146 | } 147 | var content = new Buffer(result, 'binary'); 148 | res.set('Content-Type', 'application/x-sqlite3'); 149 | res.set('Content-Disposition', 'inline; filename="mapsquare.mbtiles"'); 150 | res.send(content); 151 | 152 | }); 153 | }); 154 | 155 | // Exports 156 | module.exports = router; -------------------------------------------------------------------------------- /app/run-debug.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | NODE_DEBUG=cluster NODE_ENV=development DEBUG=mbt* node app $@ 3 | -------------------------------------------------------------------------------- /app/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | NODE_ENV=production node app $@ 3 | -------------------------------------------------------------------------------- /app/service/mbtiles-generator-service.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 eBusiness Information 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | /** 14 | * Created by Loïc Ortola on 16/10/2015. 15 | * MBTiles Generator service 16 | */ 17 | // Conf & Utils 18 | var Conf = require('../conf/conf'); 19 | var ProjectionUtils = require('../util/projection-utils'); 20 | // Classes 21 | var Tile = require('../model/tile'); 22 | var Step = require('step'); 23 | // Logging 24 | var debug = require('debug')('mbt:service:mbtile-generator-service'); 25 | // Imports 26 | var fs = require('fs'); 27 | var path = require('path'); 28 | var sqlite3 = require('sqlite3').verbose(); 29 | var http = require('http'); 30 | var uuid = require('node-uuid'); 31 | var mapper = getMapper(Conf.tileServer.type); 32 | var mbTilesStatusService = require('./mbtiles-status-service'); 33 | 34 | // Number of requests for each step set. 35 | var STEP_REQUEST_SIZE = 96; 36 | 37 | 38 | /** 39 | * Get the appropriate mapper 40 | * @param type the tile server type 41 | */ 42 | function getMapper(type) { 43 | if (type === 'osm') { 44 | return require('../mapper/osm-mapper'); 45 | } else if (type === 'bing') { 46 | return require('../mapper/bing-mapper'); 47 | } 48 | throw new Error('Wrong tile mapper specified. Please specify an appropriate mapper type in Conf'); 49 | } 50 | 51 | var initMBTilesRequest = function() { 52 | // Create temporary sqlite DB file 53 | var token = uuid.v4(); 54 | var file = 'data/' + token + '.sqlite'; 55 | var db = new sqlite3.Database(file); 56 | mbTilesStatusService.create(token); 57 | return {"db": db, "token": token, "file": file}; 58 | }; 59 | 60 | /** 61 | * Returns the future token to retrieve MBTiles once ready. 62 | * @param bounds the MBTiles bounds 63 | * @param layer the requested layer 64 | * @returns a string value 65 | */ 66 | var requestMBTiles = function (bounds, layer) { 67 | var request = initMBTilesRequest(); 68 | processMBTiles(request, bounds, layer); 69 | return request.token; 70 | }; 71 | 72 | /** 73 | * Returns a promise whose resolution will return an mbtile with the requested bounds. 74 | * @param bounds 75 | * @param layer the requested layer 76 | * @returns {Promise} the MBTile 77 | */ 78 | var requestMBTilesSync = function (bounds, layer) { 79 | var request = initMBTilesRequest(); 80 | return processMBTiles(request, bounds, layer); 81 | }; 82 | 83 | /** 84 | * Returns a promise whose resolution will return an mbtile with the requested bounds. 85 | * @param request the mbtiles request 86 | * @param bounds 87 | * @param layer the requested layer 88 | * @returns {Promise} the MBTile bounds 89 | */ 90 | var processMBTiles = function (request, bounds, layer) { 91 | 92 | return new Promise(function (resolve, reject) { 93 | console.log("Processing MBTiles for bounds:" + JSON.stringify(bounds)); 94 | 95 | fs.readFile('conf/schema.sql', 'utf8', function (err, data) { 96 | if (err) { 97 | console.error('Error while loading schema: ' + err); 98 | throw err; 99 | } 100 | createTables(request.db, data) 101 | .then(function () { 102 | // required MetaData for mbtiles spec 103 | var metaData = { 104 | "name": "Mapsquare MBTile", 105 | "type": "baselayer", 106 | "version": 1, 107 | "description": "Base Layer generated by Mapsquare", 108 | "format": "png", 109 | "bounds": bounds.left + ',' + bounds.bottom + ',' + bounds.right + ',' + bounds.top 110 | }; 111 | 112 | if (layer) { 113 | metaData.type = "overlay"; 114 | } 115 | // Insert metadata into db 116 | return insertMetadata(request.db, metaData); 117 | 118 | }) 119 | .then(function () { 120 | // Fetch then store tiles 121 | return fetchAndStoreTiles(request.token, bounds, layer, request.db); 122 | }) 123 | .then(function () { 124 | // All tiles have been stored. Close db. 125 | request.db.close(function () { 126 | console.log('MBTile computed successfully. File output is available in ' + request.file); 127 | // Persist tile state 128 | mbTilesStatusService.update(request.token, "done", 100); 129 | // Open file, send binary data to client, and remove file. 130 | fs.readFile(request.file, function (err, data) { 131 | resolve(data); 132 | }); 133 | }); 134 | 135 | }) 136 | }); 137 | }); 138 | }; 139 | 140 | /** 141 | * Create mbtiles tables 142 | * @param db the database 143 | * @param data the queries to execute 144 | * @returns {Promise} 145 | */ 146 | var createTables = function (db, data) { 147 | return new Promise(function (resolve, reject) { 148 | db.exec(data, function () { 149 | resolve(); 150 | }); 151 | }); 152 | }; 153 | 154 | /** 155 | * Insert metadata into mbtile 156 | * @param db the database 157 | * @param metaData the metadata object 158 | * @returns {Promise} 159 | */ 160 | var insertMetadata = function (db, metaData) { 161 | return new Promise(function (resolve, reject) { 162 | debug('Inserting Metadata'); 163 | db.serialize(function () { 164 | var stmt = db.prepare('INSERT INTO metadata VALUES (?, ?)'); 165 | for (var key in metaData) { 166 | stmt.run(key, metaData[key]); 167 | } 168 | stmt.finalize(); 169 | debug('Metadata inserted successfully.'); 170 | }); 171 | resolve(); 172 | }); 173 | }; 174 | 175 | /** 176 | * Insert a tile using an sqlite statement 177 | * @param stmt the statement 178 | * @param tile the tile metadata 179 | * @param data the tile blob 180 | * @param callback once it is done 181 | */ 182 | var insertTile = function (stmt, tile, data, callback) { 183 | debug('Inserting Tile'); 184 | stmt.run(tile.z, tile.x, Math.pow(2, tile.z) - tile.y - 1, data); 185 | callback(); 186 | }; 187 | 188 | /** 189 | * Process step by step fetch of tiles and parallel store into db 190 | * @param token the generation token 191 | * @param bounds the requested bounds 192 | * @param layer the requested layer 193 | * @param db the database 194 | * @returns {Promise} a promise resolved when finished. 195 | */ 196 | var fetchAndStoreTiles = function (token, bounds, layer, db) { 197 | return new Promise(function (resolve, reject) { 198 | // List tiles 199 | var tiles = listTiles(bounds, layer); 200 | console.log(tiles.length + " tiles to process."); 201 | 202 | // Prepare steps 203 | var steps = []; 204 | var stepCount = Math.floor(1 + tiles.length / STEP_REQUEST_SIZE); 205 | for (var s = 0; s < stepCount; s++) { 206 | var stmt = db.prepare('INSERT INTO tiles VALUES (?, ?, ?, ?)'); 207 | // Use closures to split the tile fetch into sets, to prevent overflows (10000s of http requests at the same time). 208 | steps.push(fetchTilesFunction(tiles.slice(s * STEP_REQUEST_SIZE, Math.min((s + 1) * STEP_REQUEST_SIZE, tiles.length)), stmt)); 209 | steps.push(finalizeStepFunction(stmt, token, s, stepCount)); 210 | } 211 | // Last step is resolution 212 | steps.push(function () { 213 | debug('Rendering done'); 214 | resolve(); 215 | }); 216 | 217 | // Launch processing 218 | Step.apply(this, steps); 219 | 220 | }); 221 | }; 222 | 223 | /** 224 | * Make a list of the necessary tiles to compute and embed into the mbtile according to bounds. 225 | * @param bounds the requested bounds 226 | * @param layer the requested layer 227 | * @returns {Array} the array of tiles 228 | */ 229 | var listTiles = function (bounds, layer) { 230 | debug('Listing tiles for bounds' + JSON.stringify(bounds)); 231 | var tiles = []; 232 | for (var z = Conf.minZoom; z <= Conf.maxZoom; z++) { 233 | var coords1 = ProjectionUtils.latLngToTileXYForZoom(bounds.top, bounds.left, z); 234 | var coords2 = ProjectionUtils.latLngToTileXYForZoom(bounds.bottom, bounds.right, z); 235 | // Adjust to process at least one tile for each zoom (lower zoom levels) 236 | if (coords1[0] === coords2[0]) { 237 | coords2[0] += 1; 238 | } 239 | if (coords1[1] === coords2[1]) { 240 | coords2[1] += 1; 241 | } 242 | 243 | for (var x = Math.min(coords1[0], coords2[0]); x <= Math.max(coords1[0], coords2[0]); x++) { 244 | for (var y = Math.min(coords1[1], coords2[1]); y <= Math.max(coords1[1], coords2[1]); y++) { 245 | var t = new Tile(x, y, z, layer, mapper.getExtension()); 246 | tiles.push(t); 247 | } 248 | } 249 | } 250 | return tiles; 251 | 252 | }; 253 | 254 | /** 255 | * Closure for tile fetch 256 | * @param tiles the set of tiles to fetch 257 | * @param stmt the statement to run queries in 258 | * @returns {Function} the function 259 | */ 260 | var fetchTilesFunction = function (tiles, stmt) { 261 | return function () { 262 | var group = this.group(); 263 | tiles.forEach(function (t) { 264 | var next = group(); 265 | fetchTile(t, 0, function (data) { 266 | // Once fetch is done, store tile 267 | insertTile(stmt, t, data, next); 268 | }); 269 | }); 270 | }; 271 | 272 | }; 273 | 274 | /** 275 | * Closure for step finalization 276 | * @param stmt the statement to close 277 | * @param token the current token 278 | * @param s the current step 279 | * @param stepCount the total number of steps 280 | * @returns {Function} the function 281 | */ 282 | var finalizeStepFunction = function (stmt, token, s, stepCount) { 283 | return function () { 284 | mbTilesStatusService.update(token, "generating", Math.floor((s + 1) * 100 / (stepCount + 1))); 285 | stmt.finalize(); 286 | this(); 287 | } 288 | }; 289 | 290 | /** 291 | * Make an http request to get a particular tile from the tileserver. Will attempt 3 times if request fails. 292 | * @param t the requested tile 293 | * @param attempts the number of failed attempts (< 3) 294 | * @param callback the callback once tile is fetched 295 | */ 296 | var fetchTile = function (t, attempts, callback) { 297 | var url = mapper.getTileUrl(t); 298 | debug("Getting " + url); 299 | http.get(url, function (res) { 300 | if (res.statusCode == 200) { 301 | res.setEncoding('base64'); 302 | var body = ''; 303 | res.on('data', function (chunk) { 304 | body += chunk; 305 | }); 306 | res.on('end', function () { 307 | var blob = new Buffer(body, 'base64'); 308 | // Store tile in sqlite db 309 | callback(blob); 310 | }) 311 | } else { 312 | var error = {statusCode: res.statusCode, message: res.body}; 313 | console.error('Received error from server' + JSON.stringify(error)); 314 | // Will retry 3 times if error occured (maybe tile is not ready yet), fail beyond. 315 | if (attempts < 3) { 316 | fetchTile(t, attempts++, callback); 317 | } 318 | } 319 | }).on('error', function (err) { 320 | console.error('Received error from server' + JSON.stringify(err)); 321 | // Will retry 3 times if error occured (maybe tile is not ready yet), fail beyond. 322 | if (attempts < 3) { 323 | fetchTile(t, attempts++, callback); 324 | } 325 | }); 326 | }; 327 | 328 | 329 | /** 330 | * Retrieve MBTiles data for current token. 331 | * @param token 332 | */ 333 | var getMBTiles = function (token, callback) { 334 | var status = mbTilesStatusService.get(token); 335 | // Return if not finished 336 | if (!status || status.status === "generating") { 337 | callback(); 338 | return; 339 | } 340 | 341 | var dbFile = 'data/' + token + '.sqlite'; 342 | 343 | // Open file, send binary data to client. 344 | fs.readFile(dbFile, function (err, data) { 345 | callback(data); 346 | // Tile has been downloaded. Notify 347 | mbTilesStatusService.update(token, "downloaded", 100); 348 | }); 349 | }; 350 | 351 | /** 352 | * Remove all MBTiles which have no reference in the app or which have been downloaded already. 353 | */ 354 | var removeOldMBTiles = function () { 355 | fs.readdir('data', function (err, files) { 356 | for (var i in files) { 357 | var token = path.basename(files[i], '.sqlite'); 358 | var status = mbTilesStatusService.get(token); 359 | if (!status || status.status === "downloaded") { 360 | fs.unlink('data/' + token + '.sqlite', function(err) { 361 | debug('Deleted file. Error? ' + err); 362 | }); 363 | } 364 | } 365 | console.log('MBTiles Cleaning in progress.'); 366 | }); 367 | }; 368 | 369 | // Init 370 | fs.mkdir('data', function (err) { 371 | if (err && err.code != 'EEXIST') { 372 | console.error(err); 373 | } 374 | }); 375 | 376 | 377 | // Exports 378 | module.exports = { 379 | requestMBTilesSync: requestMBTilesSync, 380 | requestMBTiles: requestMBTiles, 381 | getMBTiles: getMBTiles, 382 | removeOldMBTiles: removeOldMBTiles 383 | }; -------------------------------------------------------------------------------- /app/service/mbtiles-status-service.js: -------------------------------------------------------------------------------- 1 | 2 | // Logging 3 | var debug = require('debug')('mbt:service:mbtile-status-service'); 4 | // Imports 5 | var HashMap = require('hashmap'); 6 | 7 | var mbtilesStatus = new HashMap(); 8 | 9 | /** 10 | * Add new MBTiles to map 11 | * @param token 12 | */ 13 | var create = function(token) { 14 | var value = {"status": "generating", "progress": 0}; 15 | mbtilesStatus.set(token, value); 16 | }; 17 | 18 | /** 19 | * Update MBTiles status in map 20 | * @param token 21 | * @param status 22 | * @param progress 23 | */ 24 | var update = function(token, status, progress) { 25 | var value = {"status": status, "progress": progress}; 26 | mbtilesStatus.set(token, value); 27 | forwardUpdate(token, value); 28 | }; 29 | 30 | /** 31 | * Broadcast update to master 32 | * @param token 33 | * @param value 34 | */ 35 | var forwardUpdate = function(token, value) { 36 | debug('Forwarding update to master :' + JSON.stringify(value)); 37 | process.send({"tag": "mbtiles-status-broadcast", "key": token, "value": value}); 38 | debug('Forwarded'); 39 | }; 40 | 41 | /** 42 | * Received push update from paster 43 | */ 44 | process.on('message', function(msg) { 45 | if (msg.tag === 'mbtiles-status-push') { 46 | receiveUpdate(msg); 47 | } 48 | }); 49 | 50 | /** 51 | * Apply update to worker 52 | * @param data 53 | */ 54 | var receiveUpdate = function(data) { 55 | mbtilesStatus.set(data.key, data.value); 56 | }; 57 | 58 | /** 59 | * Retrieve status of current mbtiles generation. 60 | * @param token 61 | * @returns {"status": "generating|done|downloaded", "progress": 11} 62 | */ 63 | var get = function (token) { 64 | return mbtilesStatus.get(token); 65 | }; 66 | 67 | // Exports 68 | module.exports = { 69 | "get": get, 70 | "update": update, 71 | "create": create, 72 | "dispatchUpdate": forwardUpdate, 73 | "receiveUpdate": receiveUpdate 74 | }; -------------------------------------------------------------------------------- /app/start-debug.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | NODE_DEBUG=cluster NODE_ENV=development DEBUG=mbt* node bin/www $@ 3 | -------------------------------------------------------------------------------- /app/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | NODE_ENV=production node bin/www $@ 3 | -------------------------------------------------------------------------------- /app/util/projection-utils.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 eBusiness Information 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | /** 14 | * Created by Loïc Ortola on 16/10/2015. 15 | * MBTiles main application 16 | */ 17 | var EARTH_RADIUS = 6378.137; 18 | 19 | /** 20 | * Get Tile x y from Latitude, Longitude and tile numbers 21 | * @param lat in degrees 22 | * @param lng in degrees 23 | * @param z 24 | * @returns {*[]} 25 | */ 26 | var latLngToTileXYForZoom = function(lat, lng, z) { 27 | var n = Math.pow(2, z); 28 | var x = n * ((lng + 180) / 360); 29 | var latRad = lat * 2 * Math.PI /360; 30 | var y = n * (1 - (Math.log(Math.tan(latRad) + 1 / Math.cos(latRad)) / Math.PI)) / 2; 31 | return [Math.floor(x), Math.floor(y)]; 32 | }; 33 | 34 | /** 35 | * Compute the area in square kilometers of a lat lng quad. 36 | * @param b the bounds {left:, bottom:, right:, top:} 37 | */ 38 | var boundsToArea = function(b) { 39 | var r2 = Math.pow(EARTH_RADIUS, 2); 40 | console.log('Earth radius is ' + r2); 41 | // Area of lat bottom to the north-pole 42 | var alat1 = 2 * Math.PI * r2 * (1 - Math.sin(b.bottom * Math.PI / 180)); 43 | // Area of lat top to the north-pole 44 | var alat2 = 2 * Math.PI * r2 * (1 - Math.sin(b.top * Math.PI / 180)); 45 | // Area of lat portion strip 46 | var alat = alat1 - alat2; 47 | // Area of lat portion between left and right lngs. 48 | var a = alat * (Math.abs(b.left - b.right) / 360); 49 | return a; 50 | }; 51 | 52 | /** 53 | * Validate bounds 54 | * @param b the bounds {left:, bottom:, right:, top:} 55 | */ 56 | var isValidBounds = function(b) { 57 | return b.left >= -180 && b.left <= 180 && b.right >= -180 && b.right <= 180 && b.bottom >= -85 && b.bottom <= 85 && b.top >= -85 && b.top <= 85 58 | }; 59 | 60 | // Exports 61 | module.exports = { 62 | latLngToTileXYForZoom: latLngToTileXYForZoom, 63 | boundsToArea: boundsToArea, 64 | isValidBounds: isValidBounds 65 | }; -------------------------------------------------------------------------------- /docker/base/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 eBusiness Information 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | FROM node:4.2.1-slim 15 | 16 | MAINTAINER Loic Ortola 17 | 18 | RUN apt-get update && apt-get install -y git 19 | RUN mkdir /opt/app 20 | WORKDIR /tmp 21 | RUN git clone https://github.com/mapsquare/mbtiles-generator.git \ 22 | && mv mbtiles-generator/app/* /opt/app 23 | WORKDIR /opt/app 24 | RUN npm install 25 | 26 | # Remove configuration 27 | RUN rm conf/conf.js 28 | 29 | # Port exposed 30 | EXPOSE 2999 31 | 32 | # Launch server script 33 | CMD ./start.sh -------------------------------------------------------------------------------- /docker/base/README.md: -------------------------------------------------------------------------------- 1 | ## Base Docker image for MBTiles-Generator 2 | 3 | ### Supported tags and respective `Dockerfile` links 4 | * [`0.3.0`, `latest` (Dockerfile)](https://github.com/mapsquare/mbtiles-generator/tree/master/docker/base) 5 | * [`0.2.0`, (Dockerfile)](https://github.com/mapsquare/mbtiles-generator/tree/master/docker/base) 6 | 7 | ### What are MBTiles ? 8 | MBTiles is a specification allowing to store image map tiles into an sqlite database. 9 | The spec is available on [GitHub](https://github.com/mapbox/mbtiles-spec) 10 | 11 | ### What is MBTiles Generator ? 12 | Although MBTiles seem very convenient for offline purposes, it has always been tricky to create. 13 | Now, with this generator, you can easily make your own MBTiles using a remote tile-server (bing or OpenStreetMap). 14 | 15 | ### Usage 16 | Two images are available: mapsquare/mbtiles-generator-base and mapsquare/mbtiles-generator-server. 17 | This image (**mapsquare/mbtiles-generator-base**), is the bare generator: the application without any configuration file. 18 | The purpose of this image is to serve as a base-image for your own derivated images, so you have more control over the configuration and output data. 19 | If you are just looking for an easy bootstrap of the app, consider using the mapsquare/mbtiles-generator-server image instead. 20 | 21 | Here is a typical usage of this base-image: 22 | ```sh 23 | #replace here! 24 | export MBTG_CONF=/your/path/to/etc/mbtiles-generator 25 | #replace here! 26 | export MBTG_DATA=/your/path/to/tmp/mbtiles-generated 27 | docker run -d -p 2999:2999 -v $MBTG_CONF:/opt/app/conf -v $MBTG_DATA/data:/opt/app/data --name=mbtiles-generator mapsquare/mbtiles-generator-base:latest 28 | ``` -------------------------------------------------------------------------------- /docker/base/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker build -t mapsquare/mbtiles-generator-base:0.3.0 . 3 | docker build -t mapsquare/mbtiles-generator-base:latest . 4 | -------------------------------------------------------------------------------- /docker/base/push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker push mapsquare/mbtiles-generator-base:0.3.0 3 | docker push mapsquare/mbtiles-generator-base:latest 4 | -------------------------------------------------------------------------------- /docker/server/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 eBusiness Information 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM mapsquare/mbtiles-generator-base:0.3.0 16 | 17 | # The difference between this mbtiles-generator and the base image, is that this one allows an easy bootstrap of the mbtiles-generator server. 18 | # That means user only needs to provide env variables, which will configure the docker at runtime. 19 | 20 | MAINTAINER Loic Ortola 21 | 22 | # Add template conf and app script 23 | ADD *.template ./conf/ 24 | ADD app.sh ./ 25 | 26 | # Environment defaults 27 | ENV APP_MODE=server 28 | ENV TILESERVER_TYPE=osm 29 | ENV TILESERVER_ENDPOINT=http://a.tile.openstreetmap.org/{z}/{x}/{y}.png 30 | ENV APP_MINZOOM=1 31 | ENV APP_MAXZOOM=17 32 | ENV APP_MAXAREA=16 33 | # Only for server mode 34 | ENV APP_TIMEOUT=300 35 | 36 | # Launch script 37 | CMD ./app.sh -------------------------------------------------------------------------------- /docker/server/README.md: -------------------------------------------------------------------------------- 1 | ## Base Docker image for MBTiles-Generator 2 | 3 | ### Supported tags and respective `Dockerfile` links 4 | * [`0.3.0`, `latest` (Dockerfile)](https://github.com/mapsquare/mbtiles-generator/tree/master/docker/base) 5 | * [`0.2.0` (Dockerfile)](https://github.com/mapsquare/mbtiles-generator/tree/master/docker/server) 6 | 7 | ### What are MBTiles ? 8 | MBTiles is a specification allowing to store image map tiles into an sqlite database. 9 | The spec is available on [GitHub](https://github.com/mapbox/mbtiles-spec) 10 | 11 | ### What is MBTiles Generator ? 12 | Although MBTiles seem very convenient for offline purposes, it has always been tricky to create. 13 | Now, with this generator, you can easily make your own MBTiles using a remote tile-server (bing or OpenStreetMap). 14 | 15 | ### Usage 16 | Two images are available: mapsquare/mbtiles-generator-base and mapsquare/mbtiles-generator-server. 17 | This image (**mapsquare/mbtiles-generator-server**), is an all-in-one server you can launch easily, only providing env variables. 18 | 19 | Example of use: 20 | ```sh 21 | docker run -p 2999:2999 -e "APP_MODE=server" -e "TILESERVER_TYPE=osm" -e "TILESERVER_ENDPOINT=http://mytileserver.org/{z}/{x}/{y}.png" -e "APP_TIMEOUT=300" -e "APP_MINZOOM=3" -e "APP_MAXZOOM=16" mapsquare/mbtiles-generator 22 | ``` 23 | ENV variables: 24 | * APP_MODE: the execution mode. Valid values: server, command. server will launch a server on port 2999, command creates the requested MBTiles and outputs in the container /opt/app/data folder. 25 | * TILESERVER_TYPE: the tile provider. Valid values: osm, bing. 26 | * TILESERVER_ENDPOINT: depends on the provider. see app/mapper files and examples. 27 | * APP_TIMEOUT: timeout in seconds for server mode. 28 | * APP_MINZOOM: minzoom to compute the MBTiles. 29 | * APP_MAXZOOM: maxzoom to compute the MBTiles. 30 | -------------------------------------------------------------------------------- /docker/server/app.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Replace conf by env variables conf 3 | sed -e "s/\${tileServer.type}/$TILESERVER_TYPE/" \ 4 | -e "s!\${tileServer.endpoint}!$TILESERVER_ENDPOINT!" \ 5 | -e "s/\${maxZoom}/$APP_MAXZOOM/" \ 6 | -e "s/\${minZoom}/$APP_MINZOOM/" \ 7 | -e "s/\${maxArea}/$APP_MAXAREA/" \ 8 | -e "s/\${timeout}/$APP_TIMEOUT/" \ 9 | conf/conf.js.template > conf/conf.js 10 | 11 | # Launch server or CLI 12 | case "$APP_MODE" in 13 | server) 14 | ./start.sh $@ 15 | ;; 16 | *) 17 | ./run.sh $@ 18 | ;; 19 | esac -------------------------------------------------------------------------------- /docker/server/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker build -t mapsquare/mbtiles-generator-server:0.3.0 . 3 | docker build -t mapsquare/mbtiles-generator-server:latest . -------------------------------------------------------------------------------- /docker/server/conf.js.template: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 eBusiness Information 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | /** 14 | * Created by Loïc Ortola on 16/10/2015. 15 | * MBTiles Generator Configuration 16 | */ 17 | module.exports = { 18 | "tileServer": {"type": "${tileServer.type}", "endpoint": "${tileServer.endpoint}"}, 19 | "minZoom": ${minZoom}, 20 | "maxZoom": ${maxZoom}, 21 | "timeout": ${timeout}000, // Timeout in milliseconds 22 | "maxArea": ${maxArea}, 23 | }; -------------------------------------------------------------------------------- /docker/server/push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker push mapsquare/mbtiles-generator-server:0.3.0 3 | docker push mapsquare/mbtiles-generator-server:latest 4 | --------------------------------------------------------------------------------