├── FUNDING.yml ├── .gitignore ├── CHANGES.md ├── bin └── mbtiles-extractor.js ├── .eslintrc ├── LICENSE ├── package.json ├── README.md └── src └── main.js /FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [rowanwins] 2 | custom: ["https://paypal.me/rowanwinsemius"] -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules/ 3 | npm-debug.log* 4 | yarn-debug.log* 5 | yarn-error.log* 6 | package-lock.json -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | ### v0.1.1 2 | - add new option for force 3 | - add support for webp 4 | - improve handling of file extension dot -------------------------------------------------------------------------------- /bin/mbtiles-extractor.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | require = require('esm')(module); 3 | require = require('../src/main').cli(process.argv); 4 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "mourner", 3 | "parserOptions": { 4 | "ecmaVersion": 2017, 5 | "sourceType": "module" 6 | }, 7 | "rules": { 8 | "space-before-function-paren": 0, 9 | "semi": 0, 10 | "prefer-arrow-callback": 0, 11 | "no-invalid-this": 0 12 | }, 13 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Rowan Winsemius 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mbtiles-extractor", 3 | "version": "0.1.1", 4 | "description": "Takes tiles out of an mbtiles file and puts them somewhere in a directory structure.", 5 | "main": "src/index.js", 6 | "bin": { 7 | "mbtiles-extractor": "bin/mbtiles-extractor.js" 8 | }, 9 | "keywords": [ 10 | "mbtiles", 11 | "tiles", 12 | "raster-tiles", 13 | "vector-tiles", 14 | "AWS" 15 | ], 16 | "scripts": { 17 | "postpublish": "PACKAGE_VERSION=$(cat package.json | grep \\\"version\\\" | head -1 | awk -F: '{ print $2 }' | sed 's/[\",]//g' | tr -d '[[:space:]]') && git tag v$PACKAGE_VERSION && git push --tags" 18 | }, 19 | "author": "Rowan Winsemius", 20 | "license": "MIT", 21 | "dependencies": { 22 | "agentkeepalive": "^4.1.2", 23 | "aws-sdk": "^2.652.0", 24 | "cli-progress": "^3.6.1", 25 | "colors": "^1.4.0", 26 | "esm": "^3.2.25", 27 | "load-json-file": "^6.2.0", 28 | "p-queue": "^6.3.0", 29 | "p-throttle": "^3.1.0", 30 | "prompt": "^1.0.0", 31 | "sqlite3": "^4.1.1", 32 | "yargs": "^15.3.1" 33 | }, 34 | "devDependencies": { 35 | "eslint": "^6.8.0", 36 | "eslint-config-mourner": "^3.0.0" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Take tiles from an mbtiles file locally (eg vector tiles) and extracts them to ZXY structure in an S3 bucket or locally. Also works for mbtiles files containing raster tiles. 2 | 3 | ```` 4 | npm install -g mbtiles-extractor 5 | 6 | mbtiles-extractor --input=some.mbtiles --bucket=myTiles --maxZoom=10 7 | 8 | Uploading [========================================] 100% | ETA: 0s | Duration: 11s 9 | 10 | 🎉 All tiles written to AWS S3 myTiles/tiles/ 11 | 12 | // or using locally 13 | 14 | mbtiles-extractor --input=some.mbtiles --outputType=local --localOutDir=Data --tileDir=mytiles 15 | 16 | 🎉 All tiles written to /Data/mytiles 17 | 18 | ```` 19 | 20 | ### Features 21 | - Supports vector and raster tiles 22 | - Throws a prompt if you're about to override existing content in a bucket or local directory 23 | - Uses concurrent PUT requests to AWS S3 and file writes locally 24 | 25 | ### Options 26 | `--input` **Required** The filepath of an mbtiles file. Eg `--input=some.mbtiles` 27 | 28 | `--outputType=S3` Where you want to store the tiles, either `S3` or `local`. Defaults to S3. 29 | 30 | `--inRoot=false` If you want to place the tiles in the root of the output location (using no `tileDir`) 31 | 32 | `--tileDir=tiles` A directory to place the tiles in within the output dir or bucket. Quite handy using with S3. 33 | 34 | `--minZoom=0` The minimum zoom level of tiles to transfer. Eg if `minZoom=3` then levels 1 & 2 will not be transfered 35 | 36 | `--maxZoom` The maximum zoom level of tiles to transfer. Eg if `maxZoom=4` then levels 5 and above won't be transfered. If not specified this is calculated from the mbtiles file. 37 | 38 | `--fileExtension` Overrides the file extension contained in the `metadata` table. 39 | 40 | `--maxOperations` The maximum number of requests or files to write, defaults to 1000. [AWS advises](https://aws.amazon.com/about-aws/whats-new/2018/07/amazon-s3-announces-increased-request-rate-performance/) it's possible to send as many as 3500 per second. 41 | 42 | #### AWS S3 Related Options 43 | 44 | `--bucket` The name of a pre-existing bucket to put the tiles in. 45 | 46 | `--awsProfile` The name of an AWS profile to use. 47 | 48 | `--acl=public-read` The access control level of the tile. 49 | 50 | `--force=false` Force replace any existing files without asking. 51 | 52 | #### Local storage options 53 | 54 | `--localOutDir` **Required** The name of a folder to place the tiles in. 55 | 56 | 57 | ### Handling AWS roles & profiles 58 | If you need to use a named profile pass in the `awsProfile` option. 59 | 60 | For example 61 | ```` 62 | export AWS_SDK_LOAD_CONFIG=1 63 | export AWS_SHARED_CREDENTIALS_FILE=$HOME/.aws/credentials 64 | export AWS_CONFIG_FILE=$HOME/.aws/config 65 | // Followed by 66 | 67 | mbtiles-extractor --input=EEZ.mbtiles --bucket=someBucket --tileDir=EEZ --maxZoom=10 --awsProfile=myNamedProfile 68 | 69 | ```` 70 | 71 | ### Motivation 72 | I ran into lots of grief with `mapbox-tile-copy` across various node versions. It has a huge nesting of dependencies which seemed to get confused around various binding versions for sqllite3 and mapnik etc. This library is fair bit simpler in it's dependencies. 73 | 74 | This cli is also a bit faster 75 | - node-mbtiles [as described here](https://github.com/mapbox/node-mbtiles#hook-up-to-tilelive) 76 | 148 seconds 77 | - mbtiles-extractor 117 seconds 78 | 79 | ### To Do 80 | - Investigate storing in Azure or Google Cloud 81 | -------------------------------------------------------------------------------- /src/main.js: -------------------------------------------------------------------------------- 1 | import yargs from 'yargs' 2 | import prompt from 'prompt' 3 | import cliProgress from 'cli-progress' 4 | import sqlite3 from 'sqlite3' 5 | import AWS from 'aws-sdk' 6 | import colors from 'colors' 7 | import path from 'path' 8 | import fs from 'fs' 9 | import pThrottle from 'p-throttle' 10 | import PQueue from 'p-queue' 11 | import AgentKeepAlive from 'agentkeepalive' 12 | 13 | const options = yargs 14 | .option('input', {describe: 'The mbtiles file', type: 'string', demandOption: true}) 15 | .option('outputType', {describe: 'Where you want to store the tiles', type: 'string', default: 'S3', choices: ['S3', 'local']}) 16 | .option('maxOperations', {describe: 'The maximum number of requests or files to write', type: 'integer', default: 2000}) 17 | .option('inRoot', {describe: 'Should the tiles be put in the root of the location', type: 'boolean', default: false}) 18 | .option('tileDir', {describe: 'A directory to place the tiles in within the output dir or bucket', type: 'string', default: 'tiles'}) 19 | .option('minZoom', {describe: 'The minimum zoom level of tiles to transfer.', type: 'integer', default: 0}) 20 | .option('maxZoom', {describe: 'The max zoom level of tiles to transfer, otherwise uses the max level available.', type: 'integer'}) 21 | .option('fileExtension', {describe: 'Overrides the file extension contained in the metadata table.', type: 'string'}) 22 | .option('force', {describe: 'Force replace any existing files without asking.', type: 'boolean', default: false}) 23 | 24 | // AWS S3 related options 25 | .option('bucket', {describe: 'The name of the bucket', type: 'string'}) 26 | .option('awsProfile', {describe: 'A named profile to use', type: 'string'}) 27 | .option('acl', {describe: 'ACL of the uploaded file', type: 'string', default: 'public-read'}) 28 | 29 | // Local storage options 30 | .option('localOutDir', {describe: 'A directory to place the files in locally', type: 'string'}) 31 | .argv; 32 | 33 | 34 | const defaultAgent = new AgentKeepAlive.HttpsAgent({ 35 | keepAlive: true, 36 | maxSockets: 128, 37 | freeSocketTimeout: 60000 38 | }); 39 | 40 | const cliBar = new cliProgress.SingleBar({ 41 | format: `Uploading ${colors.cyan('[{bar}]')} {percentage}% | Duration: {duration_formatted}`, 42 | }, cliProgress.Presets.legecy) 43 | 44 | 45 | function getOutputFilename (row) { 46 | // Flip Y coordinate because MBTiles files are TMS. 47 | // tip from https://github.com/mapbox/node-mbtiles/blob/master/lib/mbtiles.js#L158-L159 48 | const y = (1 << row.zoom_level) - 1 - row.tile_row; 49 | return `${options.basePath}${row.zoom_level}/${row.tile_column}/${y}.${options.fileExtension}` 50 | } 51 | 52 | async function adoptProfile () { 53 | const getMfa = (serial, cb) => { 54 | prompt.message = '' 55 | const schema2 = { 56 | properties: { 57 | MFA: { 58 | description: colors.white('Enter the AWS MFA'), 59 | type: 'string', 60 | required: true, 61 | hidden: true 62 | } 63 | } 64 | } 65 | prompt.start() 66 | prompt.get(schema2, (err, r) => { 67 | if (err) errorEncounted(err) 68 | cb(null, r.MFA); 69 | }); 70 | }; 71 | 72 | const creds = new AWS.SharedIniFileCredentials({ 73 | profile: options.awsProfile, 74 | tokenCodeFn: getMfa 75 | }); 76 | await creds.getPromise() 77 | 78 | AWS.config.credentials = creds //eslint-disable-line 79 | const sts = new AWS.STS() 80 | return new Promise((resolve) => { 81 | sts.assumeRole({ 82 | RoleSessionName: 'tile-upload', 83 | RoleArn: creds.roleArn 84 | }, function (err, data) { 85 | if (err) errorEncounted(err) 86 | resolve({ 87 | accessKeyId: data.Credentials.AccessKeyId, 88 | secretAccessKey: data.Credentials.SecretAccessKey, 89 | sessionToken: data.Credentials.SessionToken, 90 | }) 91 | }); 92 | }) 93 | } 94 | 95 | let totalTileCount = 0 96 | let processed = 0 97 | 98 | function tileAdded () { 99 | processed++ 100 | } 101 | 102 | let s3 = null 103 | let contentType = null 104 | let contentEncoding = null 105 | 106 | const putInBucket = pThrottle(row => new Promise((resolve) => { 107 | const tileOptions = { 108 | ACL: options.acl, 109 | Body: row.tile_data, 110 | Bucket: options.bucket, 111 | Key: getOutputFilename(row), 112 | ContentType: contentType, 113 | ContentEncoding: contentEncoding 114 | } 115 | 116 | s3.putObject(tileOptions, function(err) { 117 | if (err) errorEncounted(err) 118 | tileAdded() 119 | resolve() 120 | }) 121 | }), options.maxOperations, 1000) 122 | 123 | const storeLocally = pThrottle(row => new Promise((resolve) => { 124 | const outName = `${path.join(options.localOutDir, getOutputFilename(row))}` 125 | if (!fs.existsSync(outName)) fs.mkdirSync(path.dirname(outName), {recursive: true}) 126 | fs.writeFile(outName, row.tile_data, function(err) { 127 | if (err) errorEncounted(err) 128 | tileAdded() 129 | resolve() 130 | }) 131 | }), options.maxOperations, 1000) 132 | 133 | const queue = new PQueue({concurrency: options.maxOperations}); 134 | 135 | function result (err, row) { 136 | if (err) errorEncounted(err); 137 | if (options.outputType === 'S3') queue.add(() => putInBucket(row)) 138 | if (options.outputType === 'local') queue.add(() => storeLocally(row)) 139 | } 140 | 141 | function processMetadata (rows) { 142 | rows.forEach(function (r) { 143 | if (r.name === 'minzoom') { 144 | options.minZoom = options.minZoom ? options.minZoom : r.value 145 | } 146 | if (r.name === 'maxzoom') { 147 | options.maxZoom = options.maxZoom ? options.maxZoom : r.value 148 | } 149 | if (r.name === 'format') { 150 | let ext = null 151 | if (r.value === 'image/png') { 152 | contentType = r.value 153 | ext = 'png' 154 | } else if (r.value === 'image/jpeg') { 155 | contentType = r.value 156 | ext = 'jpg' 157 | } else if (r.value === 'image/webp') { 158 | contentType = r.value 159 | ext = 'webp' 160 | } else if (r.value === 'pbf') { 161 | contentType = 'application/x-protobuf' 162 | ext = 'pbf' 163 | contentEncoding = 'gzip' 164 | } 165 | options.fileExtension = options.fileExtension ? options.fileExtension : ext 166 | if (options.fileExtension.startsWith('.')) { 167 | options.fileExtension = options.fileExtension.substr(1) 168 | } 169 | } 170 | }) 171 | } 172 | 173 | let chunk = 0 174 | async function onChunkCompletion () { 175 | chunk++ 176 | await queue.onIdle(); 177 | cliBar.update((processed / totalTileCount) * 100); 178 | if (processed < totalTileCount) { 179 | processChunk() 180 | } else if (processed === totalTileCount) { 181 | completed() 182 | } else if (processed < totalTileCount) { 183 | errorEncounted('This shouldnt happen') 184 | } 185 | } 186 | 187 | let db = null 188 | function processChunk () { 189 | db.each(`SELECT * FROM tiles WHERE zoom_level BETWEEN ${options.minZoom} AND ${options.maxZoom} LIMIT ${options.maxOperations} OFFSET ${(chunk * options.maxOperations)}`, [], result, onChunkCompletion) 190 | } 191 | 192 | async function processMbTiles () { 193 | db = new sqlite3.Database(`${options.input}`, sqlite3.OPEN_READONLY, async function (err) { // eslint-disable-line 194 | if (err) errorEncounted(err) 195 | this.all('SELECT * FROM metadata', [], (err, rows) => { 196 | if (err) errorEncounted(err) 197 | processMetadata(rows) 198 | this.all(`SELECT COUNT(*) FROM tiles WHERE zoom_level BETWEEN ${options.minZoom} AND ${options.maxZoom}`, [], (err, rows) => { 199 | if (err) errorEncounted(err) 200 | totalTileCount = rows[0]['COUNT(*)'] 201 | cliBar.start(100, 0); 202 | processChunk() 203 | }) 204 | }) 205 | }) 206 | } 207 | 208 | prompt.message = '' 209 | const schema = { 210 | properties: { 211 | agree: { 212 | description: colors.cyan('Files already exist in that location, are you sure you want to continue? t/rue or f/alse'), 213 | type: 'boolean', 214 | message: 'Must respond true/t or false/f', 215 | required: true 216 | } 217 | } 218 | } 219 | 220 | export async function cli () { 221 | // remove a traiing slash from the tileDir 222 | options.tileDir = options.tileDir.replace(/\/$/, '') 223 | options.basePath = options.inRoot ? '' : `${options.tileDir}/` 224 | 225 | if (options.outputType === 'S3') { 226 | if (options.bucket === undefined) errorEncounted('For outputType=S3 you must specify the bucket option') 227 | const awsOptions = { 228 | httpOptions: { 229 | timeout: 2000, 230 | agent: defaultAgent 231 | } 232 | } 233 | if (options.awsProfile) { 234 | const profile = await adoptProfile() 235 | Object.assign(awsOptions, profile); 236 | } 237 | s3 = new AWS.S3(awsOptions) 238 | const basePath = { 239 | Bucket: options.bucket, 240 | Prefix: options.basePath, 241 | MaxKeys: 1 242 | } 243 | 244 | if (options.force) { 245 | processMbTiles() 246 | } else { 247 | // Check if there are already any files in that dir 248 | s3.listObjects(basePath, function (err, data) { 249 | if (err) errorEncounted(err) 250 | if (data.Contents.length > 0) { 251 | prompt.start() 252 | prompt.get(schema, (err, result) => { 253 | if (err) errorEncounted(err) 254 | if (result.agree) processMbTiles() 255 | }); 256 | } else { 257 | processMbTiles() 258 | } 259 | }) 260 | } 261 | } else if (options.outputType === 'local') { 262 | if (options.localOutDir === undefined) errorEncounted('For outputType=local you must specify the localOutDir option.') 263 | const dest = path.join(options.localOutDir, options.basePath) 264 | if (!fs.existsSync(dest)) { 265 | fs.mkdirSync(dest) 266 | processMbTiles() 267 | } else { 268 | prompt.start() 269 | prompt.get(schema, (err, result) => { 270 | if (err) errorEncounted(err) 271 | if (result.agree) processMbTiles() 272 | }); 273 | } 274 | } 275 | } 276 | 277 | function errorEncounted (err) { 278 | console.log(` 279 | ❌ Sorry but we couldn't complete the operation - check the error message below. 280 | 281 | ${err} 282 | `) 283 | process.exit(0) 284 | } 285 | 286 | function completed () { 287 | cliBar.stop(); 288 | const out = options.outputType === 'local' ? path.join(options.localOutDir, options.basePath) : `AWS S3 ${options.bucket}/${options.basePath}` 289 | console.log(` 290 | 🎉 All tiles written to ${out} 291 | `) 292 | } 293 | 294 | --------------------------------------------------------------------------------