├── .eslintignore ├── .eslintrc ├── .gitignore ├── .travis.yml ├── .vscode └── launch.json ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── appveyor.yml ├── files ├── style.json ├── styleCatOff.json ├── styleMultipleLayers.json └── trails.mbtiles ├── index.js ├── media ├── VTSummary.PNG ├── example.gif ├── optimization.gif ├── optimizationResults.PNG ├── simplify.PNG ├── sizeDistribution.PNG ├── tileInfo.PNG └── tilesInBucket.PNG ├── package.json ├── src ├── PaintPropertiesToCheck.js ├── UI.js ├── command-line-options.js ├── core │ ├── ColoredString.js │ ├── DataConverter.js │ ├── IO.js │ ├── Log.js │ ├── MapboxStyle.js │ ├── MapboxStyleLayer.js │ ├── SQLite.js │ ├── Simplifier.js │ ├── Utils.js │ ├── VTProcessor.js │ ├── VTReader.js │ ├── VTWriter.js │ └── vector-tile.js └── usage-sections.js └── test └── unit ├── ColoredString.test.js ├── IO.test.js ├── MapboxStyle.test.js ├── MapboxStyleLayer.test.js └── Utils.test.js /.eslintignore: -------------------------------------------------------------------------------- 1 | index.js -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "parserOptions": { 3 | "ecmaVersion": 2017 4 | }, 5 | 6 | "env": { 7 | "es6": true 8 | } 9 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | cache: 3 | directories: 4 | - node_modules 5 | node_js: 6 | - 8 7 | before_script: 8 | - npm install 9 | script: 10 | - npm test -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Examiner", 11 | "program": "${workspaceFolder}/index.js", 12 | "args": [ 13 | "-m", 14 | "files/trails.mbtiles" 15 | ] 16 | }, 17 | { 18 | "type": "node", 19 | "request": "launch", 20 | "name": "Optimizer", 21 | "program": "${workspaceFolder}/index.js", 22 | "args": [ 23 | "-m", 24 | "files/0.mbtiles", 25 | "-s", 26 | "files/styleCatOff.json" 27 | ] 28 | }, 29 | { 30 | "type": "node", 31 | "request": "launch", 32 | "name": "Simplifier", 33 | "program": "${workspaceFolder}/index.js", 34 | "args": [ 35 | "-m", 36 | "files/0.mbtiles", 37 | "-x", 38 | "80", 39 | "-y", 40 | "64", 41 | "-z", 42 | "7", 43 | "-l", 44 | "pein_1_lin", 45 | "-t", 46 | "0.01" 47 | ] 48 | }, 49 | { 50 | "type": "node", 51 | "request": "launch", 52 | "name": "Inspect URL", 53 | "program": "${workspaceFolder}/index.js", 54 | "args": [ 55 | "-u", 56 | "http://vladivostok.icgc.local:3000/14/8292/6112.pbf", 57 | "-x", 58 | "6112", 59 | "-y", 60 | "8292", 61 | "-z", 62 | "14" 63 | ] 64 | } 65 | ] 66 | } -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contbitute 2 | We are glad you are reading this, because we need volunteer works to improve vt-optimizer. 3 | 4 | ## Testing 5 | Please write tests for every functionality. 6 | 7 | ## Submitting changes 8 | Please send a [GitHub Pull Request to the dev branch](https://github.com/ibesora/vt-optimizer/pull/new/dev), so that @ibesora can merge to master when everything is documented and is ready to go after passing all tests. 9 | 10 | ## Coding conventions 11 | We use [eslint-config-geostart](https://github.com/geostarters/eslint-config-geostart). 12 | 13 | Thank you. 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Isaac Besora Vilardaga 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Vector Tile optimizer 2 | 3 | [![Build Status Linux](https://travis-ci.org/ibesora/vt-optimizer.svg?branch=master)](https://travis-ci.org/ibesora/vt-optimizer) 4 | [![Build status](https://ci.appveyor.com/api/projects/status/7c6haos5thbxrnu4/branch/master?svg=true)](https://ci.appveyor.com/project/ibesora/vt-optimizer/branch/master) 5 | ![Dependency Status](https://david-dm.org/ibesora/vt-optimizer.svg) 6 | ![License](https://img.shields.io/badge/license-MIT-blue.svg) 7 | > A small NodeJS cli tool to inspect and optimize [Mapbox MBTiles files](https://github.com/mapbox/mbtiles-spec) used to encode [Mapbox Vector Tiles](https://www.mapbox.com/vector-tiles/). You can find an explanation on what it does and how it works in this [blog post](https://medium.com/@ibesora/a-data-driven-journey-through-vector-tile-optimization-4a1dbd4f3a27) 8 | 9 | 10 | 11 | ## Installation 12 | 13 | Clone this repository and run `npm install` 14 | 15 | ## Usage 16 | This tool has two main functionalities: **inspecting** a vector tile to see if it conforms to Mapbox's recommendations of an average tile size of 50KB and a maximum tile size of 500KB and **optimizing** a vector tile to be used with a fixed style. 17 | 18 | ### Vector Tile inspection 19 | When running the tool with the *-m file.mbtiles* argument, the inspection mode will be started. This mode is used for interactively inspecting the contents of a Vector Tile. 20 | 21 | After the file is read, some metadata about the file, the layers list and a summary table are shown. With this table you can get a first glimpse of how well your Vector Tile is doing. 22 | 23 | Where: 24 | * **Zoom level** is the zoom level index 25 | * **Tiles** is the total number of tiles found in this level 26 | * **Total level size** is the sum of all the data found in this level in kilobytes 27 | * **Average tile size** is the average tile size of the tiles found in this level in kilobytes 28 | * **Max tile size** is the maximum tile size found in this level 29 | 30 | We can then get more information about a given level to see the distribution of its tiles by size on a 10-bucket histogram. 31 | 32 | Where: 33 | * **Bucket min** and **Bucket max** are the minimum and maximum size of this bucket respectively 34 | * **Nº of tiles** is the number of tiles that have a size between the min and max values defined by the bucket 35 | * **Running avg size** is the tile average size in kilobytes that this level would have if only the buckets up to and including this one were present. Average sizes that are almost at the limit of the recommendation are drawn in yellow and the ones that go above the limit are drawn in red. If the average size is above limit you'd want, it, ideally, as lower as possible on the list. 36 | * **% of tiles in this level** is the percentage of tiles found in this bucket. Ideally you'd want an almost even distribution of tiles between bucket sizes. 37 | * **% of level size** is the percentage of level size that this bucket brings to the level total 38 | * **Accum % of tiles** is the accumulated percentage number of tiles from the lower size bucket up to the one we are looking at 39 | * **Accum % size** is the accumulated percentage of level size from the lower size bucket up to the one we are looking at 40 | 41 | Afterwards we can see which are the specific tiles that fall inside a given bucket 42 | 43 | and selecting one of them we can see a summary of the tile contents 44 | 45 | where the number of layers and the number of features and properties of each layer are shown. 46 | 47 | #### Single tile inspection 48 | 49 | ##### From a local file 50 | When running the tool with the *-m file.pbf* and *-x X -y Y -z Z* arguments, each layer of the tile contents will be converted to geojson and printed on the console 51 | 52 | ##### From an url 53 | When running the tool with the *-u url.pbf* and *-x X -y Y -z Z* arguments, the pbf will be downloaded and each layer of the single tile will be converted to geojson and printed on the console 54 | 55 | ### Vector Tile optimization 56 | When running the tool with the *-m file.mbtiles* and *-s style.json* arguments, the optimization mode will be started. This mode is used for optimizing a Vector Tile when used in conjunction with a style that follows [Mapbox Style Specification](https://www.mapbox.com/mapbox-gl-js/style-spec/). 57 | 58 | It reads both the Vector Tile and style and removes all the layers and features that are not visible, either because they are not used or because the style configuration makes them not renderable. When the process is finished the number of features removed in each zoom level and the levels where each layer has been removed from are shown. 59 | 60 | **Note:** To use the optimization tool it's better to run the `--max-old-space-size` NodeJS argument to increase NodeJS process heap space as the entire Vector Tile is loaded and decompressed when working. 61 | ``` 62 | node --max-old-space-size=16386 index.js -m files/input.mbtiles -s files/style.json -o files/output.mbtiles 63 | ``` 64 | ### Vector Tile layer simplification 65 | When running the tool with the *-x X -y Y -z Z -l layerName -t tolerance* arguments, the simplification mode will be started. This mode is used for simplifying a layer on a given tile. **Note:** The tolerance value is in degrees (1º is aproximately 110 meters) and tells the algorithm that two points with a distance lower than the tolerance value should be merged. 66 | 67 | When the process is finished, the results of the simplification are shown. 68 | 69 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | image: Visual Studio 2017 2 | environment: 3 | matrix: 4 | - nodejs_version: "8" 5 | - nodejs_version: "9" 6 | - nodejs_version: "10" 7 | platform: 8 | - x86 9 | - x64 10 | install: 11 | - ps: Install-Product node $env:nodejs_version 12 | - md public 13 | - npm install --global --production windows-build-tools 14 | - npm install 15 | build: off 16 | test_script: 17 | - npm run lint -------------------------------------------------------------------------------- /files/style.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 8, 3 | "name": "Trails test", 4 | "sources": {}, 5 | "layers": [ 6 | { 7 | "id": "water", 8 | "source": "mapbox-streets", 9 | "source-layer": "water", 10 | "type": "fill", 11 | "paint": { 12 | "fill-color": "#00ffff" 13 | }, 14 | "minzoom": 1, 15 | "maxzoom": 21 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /files/styleMultipleLayers.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 8, 3 | "name": "munich test", 4 | "sources": { 5 | "munich": { 6 | "type": "vector", 7 | "url": "tileserver/munich.json" 8 | } 9 | }, 10 | "layers": [ 11 | { 12 | "id": "circle1", 13 | "type": "circle", 14 | "source": "munich", 15 | "source-layer": "munich", 16 | "minzoom": 0, 17 | "maxzoom": 2, 18 | "paint": { 19 | "circle-radius": 6, 20 | "circle-color": "#B42222" 21 | } 22 | }, 23 | { 24 | "id": "circle2", 25 | "type": "circle", 26 | "source": "munich", 27 | "source-layer": "munich", 28 | "minzoom": 2, 29 | "paint": { 30 | "circle-radius": 6, 31 | "circle-color": "#2222B4" 32 | } 33 | } 34 | ] 35 | } 36 | -------------------------------------------------------------------------------- /files/trails.mbtiles: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ibesora/vt-optimizer/90fbc28e16975c1f3d6a24b3a383f9a08fbea72b/files/trails.mbtiles -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const CommandLineArgs = require("command-line-args"); 4 | const CommandLineUsage = require("command-line-usage"); 5 | const commandLineOptions = require("./src/command-line-options"); 6 | const usageSections = require("./src/usage-sections"); 7 | const VTProcessor = require("./src/core/VTProcessor"); 8 | 9 | 10 | const Log = require("./src/core/Log"); 11 | 12 | try { 13 | 14 | const options = CommandLineArgs(commandLineOptions); 15 | 16 | if (options.help) { 17 | 18 | const usage = CommandLineUsage(usageSections); 19 | console.log(usage); 20 | 21 | } else if (options.mbtiles) { 22 | 23 | if (options.style) { 24 | 25 | VTProcessor.slim(options.mbtiles, options.style, options.out); 26 | 27 | } else if (options.tolerance) { 28 | 29 | VTProcessor.simplifyTileLayer(options.mbtiles, parseInt(options.zoom), 30 | parseInt(options.column), parseInt(options.row), 31 | options.layer, parseFloat(options.tolerance)); 32 | 33 | } else if (options.row) { 34 | 35 | VTProcessor.showTileInfo(options.mbtiles, parseInt(options.zoom), 36 | parseInt(options.column), parseInt(options.row)); 37 | 38 | } else { 39 | 40 | // Examination mode 41 | VTProcessor.showInfo(options.mbtiles); 42 | 43 | } 44 | 45 | } else if (options.PBFUrl) { 46 | 47 | VTProcessor.showURLTileInfo(options.PBFUrl, parseInt(options.zoom), 48 | parseInt(options.column), parseInt(options.row)); 49 | 50 | } else { 51 | 52 | Log.info("Wrong usage. Use -h to see the valid arguments."); 53 | 54 | } 55 | 56 | } catch (err) { 57 | 58 | Log.error(err); 59 | 60 | } 61 | -------------------------------------------------------------------------------- /media/VTSummary.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ibesora/vt-optimizer/90fbc28e16975c1f3d6a24b3a383f9a08fbea72b/media/VTSummary.PNG -------------------------------------------------------------------------------- /media/example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ibesora/vt-optimizer/90fbc28e16975c1f3d6a24b3a383f9a08fbea72b/media/example.gif -------------------------------------------------------------------------------- /media/optimization.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ibesora/vt-optimizer/90fbc28e16975c1f3d6a24b3a383f9a08fbea72b/media/optimization.gif -------------------------------------------------------------------------------- /media/optimizationResults.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ibesora/vt-optimizer/90fbc28e16975c1f3d6a24b3a383f9a08fbea72b/media/optimizationResults.PNG -------------------------------------------------------------------------------- /media/simplify.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ibesora/vt-optimizer/90fbc28e16975c1f3d6a24b3a383f9a08fbea72b/media/simplify.PNG -------------------------------------------------------------------------------- /media/sizeDistribution.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ibesora/vt-optimizer/90fbc28e16975c1f3d6a24b3a383f9a08fbea72b/media/sizeDistribution.PNG -------------------------------------------------------------------------------- /media/tileInfo.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ibesora/vt-optimizer/90fbc28e16975c1f3d6a24b3a383f9a08fbea72b/media/tileInfo.PNG -------------------------------------------------------------------------------- /media/tilesInBucket.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ibesora/vt-optimizer/90fbc28e16975c1f3d6a24b3a383f9a08fbea72b/media/tilesInBucket.PNG -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vt-optimizer", 3 | "version": "0.0.2", 4 | "description": "A vector tile size inspector and optimizer", 5 | "main": "src/index.js", 6 | "bin": { 7 | "vt-optimizer": "./index.js" 8 | }, 9 | "scripts": { 10 | "lint": "eslint --fix --cache --ignore-path .gitignore src", 11 | "test": "run-s lint test-unit", 12 | "test-cov": "nyc --require=flow-remove-types/register --reporter=text-summary --reporter=lcov --cache npm run test-unit", 13 | "test-unit": "tap --reporter classic --no-coverage test/unit" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "git+https://github.com/ibesora/vt-optimizer.git" 18 | }, 19 | "keywords": [ 20 | "vector", 21 | "tiles", 22 | "mapbox" 23 | ], 24 | "author": "ibesora", 25 | "license": "MIT", 26 | "bugs": { 27 | "url": "https://github.com/ibesora/vt-optimizer/issues" 28 | }, 29 | "homepage": "https://github.com/ibesora/vt-optimizer#readme", 30 | "dependencies": { 31 | "@mapbox/vector-tile": "^1.3.1", 32 | "axios": ">=0.21.1", 33 | "command-line-args": "^5.0.2", 34 | "command-line-usage": "^5.0.5", 35 | "console.table": "^0.10.0", 36 | "geojson-vt": "^3.2.1", 37 | "inquirer": "^6.2.0", 38 | "listr": "^0.14.2", 39 | "pbf": "^3.1.0", 40 | "rxjs": "^6.3.3", 41 | "simplify-geojson": "^1.0.4", 42 | "sqlite3": "^4.1.0", 43 | "vt-pbf": "git+https://github.com/ibesora/vt-pbf.git" 44 | }, 45 | "devDependencies": { 46 | "@babel/eslint-parser": "^7.17.0", 47 | "chalk": "^2.4.1", 48 | "coveralls": "^3.0.2", 49 | "eslint": "^8.8.0", 50 | "npm-run-all": "^4.1.5", 51 | "tap": "^14.6.1" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/PaintPropertiesToCheck.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | module.exports = [ 5 | "fill-opacity", 6 | "fill-outline-color", 7 | "line-opacity", 8 | "line-width", 9 | "icon-size", 10 | "text-size", 11 | "text-max-width", 12 | "text-opacity", 13 | "raster-opacity", 14 | "circle-radius", 15 | "circle-opacity", 16 | "fill-extrusion-opacity", 17 | "heatmap-opacity" 18 | ]; 19 | -------------------------------------------------------------------------------- /src/UI.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | /*eslint camelcase: ["error", {allow: ["zoom_level", "tile_row", "tile_column"]}]*/ 3 | "use strict"; 4 | 5 | const Inquirer = require("inquirer"); 6 | const ColoredString = require("./core/ColoredString"); 7 | const Log = require("./core/Log"); 8 | const Utils = require("./core/Utils"); 9 | 10 | class UI { 11 | 12 | static printMetadata(minZoom, maxZoom, format, center, layers) { 13 | 14 | Log.log(""); 15 | Log.title("Vector Tile Info"); 16 | Log.log( 17 | ColoredString.format(ColoredString.green, "Zoom levels: "), 18 | ColoredString.format(ColoredString.white, minZoom, maxZoom) 19 | ); 20 | Log.log( 21 | ColoredString.format(ColoredString.green, "Format: "), 22 | ColoredString.format(ColoredString.white, format) 23 | ); 24 | Log.log( 25 | ColoredString.format(ColoredString.green, "Center: "), 26 | ColoredString.format(ColoredString.white, center) 27 | ); 28 | Log.log( 29 | ColoredString.format(ColoredString.green, "Layers: ") 30 | ); 31 | 32 | Log.list("", layers); 33 | 34 | } 35 | 36 | static printSummaryTable(vtSummary, tiles, avgTileSizeLimit, avgTileSizeWarning, tileSizeLimit) { 37 | const labels = ["Zoom level", "Tiles", "Total level size (KB)", "Average tile size (KB)", "Max tile size (KB)", ""]; 38 | var columnWidths = labels.map((label)=>label.length); 39 | const data = UI.createSummaryTableData(vtSummary, tiles, avgTileSizeLimit, avgTileSizeWarning, tileSizeLimit,columnWidths ); 40 | 41 | Log.log(""); 42 | Log.title("Vector Tile Summary"); 43 | Log.table(labels, data); 44 | 45 | } 46 | static formatDataPoint(value, columnSize, dontRound){ 47 | var rounded = dontRound ? value.toString() : value.toFixed(2); 48 | var padded = columnSize ? rounded.padStart(columnSize-1, " ") : rounded; 49 | return padded; 50 | }; 51 | static createSummaryTableData(vtSummary, tiles, avgTileSizeLimit, avgTileSizeWarning, tileSizeLimit, columnWidths) { 52 | 53 | const data = []; 54 | const bigTiles = tiles.reduce((obj, tile) => { 55 | 56 | obj[tile["zoom_level"]] = tile; 57 | return obj; 58 | 59 | }, 60 | {} 61 | ); 62 | 63 | for (const levelData of vtSummary) { 64 | 65 | const avgTileSizeInKB = levelData.avgTileSize / 1024.0; 66 | const currentLevel = levelData["zoom_level"]; 67 | const currentBigTile = bigTiles[currentLevel]; 68 | const avgSizeTooBig = (avgTileSizeInKB) > avgTileSizeLimit; // Mapbox recommends an average tile size of 50KB 69 | const avgSizeAlmostTooBig = (avgTileSizeInKB) > avgTileSizeWarning; 70 | let levelComment = ColoredString.green("✓"); 71 | let avgSizeMessage = UI.formatDataPoint(avgTileSizeInKB, columnWidths[3]); 72 | 73 | if (avgSizeTooBig) { 74 | 75 | levelComment = ColoredString.red("☓ Error: The average tile size of this level exceeds 50KB."); 76 | avgSizeMessage = ColoredString.red(avgSizeMessage); 77 | 78 | } else if (avgSizeAlmostTooBig) { 79 | 80 | levelComment = ColoredString.yellow("✓ Warning: The average tile size of this level almost exceeds 50KB."); 81 | avgSizeMessage = ColoredString.yellow(avgSizeMessage); 82 | 83 | } 84 | 85 | if (currentBigTile) { 86 | 87 | levelComment = `${levelComment} ${ColoredString.red(`Error: A total of ${currentBigTile.num} tiles are bigger than ${tileSizeLimit}KB`)}`; 88 | 89 | } 90 | 91 | 92 | data.push([ 93 | levelData["zoom_level"], 94 | levelData.tiles, 95 | UI.formatDataPoint(levelData.size / 1024.0, columnWidths[2]), 96 | avgSizeMessage, 97 | UI.formatDataPoint(levelData.maxSize / 1024.0, columnWidths[4]), 98 | levelComment 99 | ]); 100 | 101 | } 102 | 103 | return data; 104 | 105 | } 106 | 107 | static wantMoreInfoQuestion() { 108 | 109 | return new Promise((resolve) => { 110 | 111 | Inquirer.prompt([ 112 | { 113 | type: "confirm", 114 | name: "extraInfo", 115 | message: "Do you want to get more information about a given level?", 116 | default: false 117 | } 118 | ]).then(answers => { 119 | 120 | resolve(answers["extraInfo"]); 121 | 122 | }); 123 | 124 | }); 125 | 126 | } 127 | 128 | static selectLevelPrompt(vtSummary, avgTileSizeLimit, avgTileSizeWarning) { 129 | 130 | const levels = vtSummary.map((elem) => 131 | UI.formatLevelElement(elem, avgTileSizeLimit, avgTileSizeWarning) 132 | ); 133 | 134 | return new Promise(resolve => { 135 | 136 | Inquirer.prompt([{ 137 | type: "list", 138 | name: "zoomLevel", 139 | message: "Select a level", 140 | choices: levels 141 | }]).then(async (answers) => { 142 | 143 | const zoomLevel = parseInt(answers["zoomLevel"].split(" ")[0]); 144 | resolve(zoomLevel); 145 | 146 | }); 147 | 148 | }); 149 | 150 | } 151 | 152 | static formatLevelElement(elem, avgTileSizeLimit, avgTileSizeWarning) { 153 | 154 | const avgTileSizeInKB = elem.avgTileSize / 1024.0; 155 | const avgSizeTooBig = (avgTileSizeInKB) > avgTileSizeLimit; // Mapbox recommends an average tile size of 50KB 156 | const avgSizeAlmostTooBig = (avgTileSizeInKB) > avgTileSizeWarning; 157 | let avgSizeMessage = UI.formatDataPoint(avgTileSizeInKB) + " KB average size"; 158 | 159 | if (avgSizeTooBig) { 160 | 161 | avgSizeMessage = ColoredString.red(avgSizeMessage); 162 | 163 | } else if (avgSizeAlmostTooBig) { 164 | 165 | avgSizeMessage = ColoredString.yellow(avgSizeMessage); 166 | 167 | } 168 | 169 | return `${elem["zoom_level"]} (${elem.tiles} tiles - ${avgSizeMessage}) `; 170 | 171 | } 172 | 173 | static showTileDistributionData(data, avgTileSizeLimit, avgTileSizeWarning) { 174 | var labels = ["#", "Bucket min (KB)", "Bucket max (KB)", "Nº of tiles", "Running avg size (KB)", "% of tiles in this level", "% of level size", "Accum % of tiles", "Accum % size"]; 175 | var columnWidths = labels.map((label)=>label.length); 176 | const dataToPrint = data.map((elem, index) => 177 | UI.formatTileDistributionElement(elem, index, avgTileSizeLimit, avgTileSizeWarning, columnWidths) 178 | ); 179 | 180 | Log.title("Tile size distribution"); 181 | Log.table(labels, dataToPrint); 182 | 183 | } 184 | 185 | static formatTileDistributionElement(elem, index, avgTileSizeLimit, avgTileSizeWarning, columnWidths) { 186 | 187 | const avgTileSizeInKB = elem.runningAvgSize; 188 | const avgSizeTooBig = (avgTileSizeInKB) > avgTileSizeLimit; // Mapbox recommends an average tile size of 50KB 189 | const avgSizeAlmostTooBig = (avgTileSizeInKB) > avgTileSizeWarning; 190 | let avgSizeMessage = UI.formatDataPoint(avgTileSizeInKB, columnWidths[4]); 191 | 192 | if (avgSizeTooBig) { 193 | 194 | avgSizeMessage = ColoredString.red(avgSizeMessage); 195 | 196 | } else if (avgSizeAlmostTooBig) { 197 | 198 | avgSizeMessage = ColoredString.yellow(avgSizeMessage); 199 | 200 | } 201 | 202 | return [index + 1, 203 | UI.formatDataPoint(elem.minSize,columnWidths[1]), 204 | UI.formatDataPoint(elem.maxSize,columnWidths[2]), 205 | UI.formatDataPoint(elem.length, columnWidths[3],false), 206 | avgSizeMessage, 207 | UI.formatDataPoint(elem.currentPc,columnWidths[5]), 208 | UI.formatDataPoint(elem.currentBucketSizePc,columnWidths[6]), 209 | UI.formatDataPoint(elem.accumPc,columnWidths[7]), 210 | UI.formatDataPoint(elem.accumBucketSizePc,columnWidths[8]), 211 | ]; 212 | 213 | } 214 | 215 | static tilesInBucketQuestion() { 216 | 217 | return new Promise(resolve => { 218 | 219 | Inquirer.prompt([ 220 | { 221 | type: "confirm", 222 | name: "extraInfo", 223 | message: "Do you want to see which tiles are in a bucket?", 224 | default: false 225 | } 226 | ]).then(answers => { 227 | 228 | resolve(answers["extraInfo"]); 229 | 230 | }); 231 | 232 | }); 233 | 234 | } 235 | 236 | static selectBucketPrompt(bucketData) { 237 | 238 | const bucketNames = []; 239 | for (let index = 1; index <= bucketData.length; ++index) { 240 | const currBucketData = bucketData[index - 1]; 241 | 242 | var min = UI.formatDataPoint(currBucketData.minSize) +" KB "; 243 | var max = UI.formatDataPoint(currBucketData.maxSize) +" KB "; 244 | 245 | bucketNames.push(`${index} ${min} < Size <= ${max} (${currBucketData.length} tiles)`); 246 | 247 | } 248 | 249 | return new Promise(resolve => { 250 | 251 | Inquirer.prompt([{ 252 | type: "list", 253 | name: "bucket", 254 | message: "Select a bucket", 255 | choices: bucketNames 256 | }]).then((answers) => { 257 | 258 | const bucketIndex = parseInt(answers["bucket"].split(" ")[0]) - 1; 259 | resolve(bucketIndex); 260 | 261 | }); 262 | 263 | }); 264 | 265 | } 266 | 267 | static showBucketInfo(bucket, tileSizeLimit) { 268 | 269 | const info = bucket.sort((a, b) => b.size - a.size).map((tile) => 270 | UI.formatBucketInfo(tile, tileSizeLimit) 271 | ); 272 | 273 | Log.list("Tiles in this bucket", info); 274 | 275 | } 276 | 277 | static formatBucketInfo(tile, tileSizeLimit) { 278 | 279 | const size = UI.formatDataPoint(tile.size)+" KB"; 280 | const tileSizeMessage = (tile.size > tileSizeLimit ? ColoredString.red(size) : size); 281 | const {lon, lat} = Utils.tile2LonLat(tile.zoom_level, tile.tile_row, tile.tile_column); 282 | return `${tile.zoom_level}/${tile.tile_column}/${tile.tile_row} - ${tileSizeMessage} - LonLat: [${lon}, ${lat}]`; 283 | 284 | } 285 | 286 | static tileInfoQuestion() { 287 | 288 | return new Promise(resolve => { 289 | 290 | Inquirer.prompt([ 291 | { 292 | type: "confirm", 293 | name: "extraTileInfo", 294 | message: "Do you want to get more info about a tile?", 295 | default: false 296 | } 297 | ]).then(answers => { 298 | 299 | resolve(answers["extraTileInfo"]); 300 | 301 | }); 302 | 303 | }); 304 | 305 | } 306 | 307 | static selectTilePrompt(bucket, tileSizeLimit) { 308 | 309 | const tiles = bucket.map((tile) => 310 | UI.formatBucketInfo(tile, tileSizeLimit) 311 | ); 312 | 313 | return new Promise(resolve => { 314 | 315 | Inquirer.prompt([{ 316 | type: "list", 317 | name: "tile", 318 | message: "Select a tile", 319 | choices: tiles 320 | }]).then((answers) => { 321 | 322 | const tileIndex = answers["tile"].split(" ")[0].split("/"); 323 | const tile = {zoom_level: tileIndex[0], tile_column: tileIndex[1], tile_row: tileIndex[2]}; 324 | resolve(tile); 325 | 326 | }); 327 | 328 | }); 329 | 330 | } 331 | 332 | static showTileInfo(tileData, vectorTileLayers) { 333 | 334 | let totalFeatures = 0; 335 | let totalVertices = 0; 336 | let totalKeys = 0; 337 | let totalValues = 0; 338 | 339 | const layers = tileData.layers.map((layer) => { 340 | 341 | const vtLayer = vectorTileLayers[layer.name]; 342 | layer.layerVertices = vtLayer.features.reduce((accum, feature) => accum + feature.geometry.coordinates.reduce((accum, ring) => (ring.length ? accum + ring.length : feature.geometry.coordinates.length / 2), 0), 0); 343 | return layer; 344 | 345 | }); 346 | const info = layers.sort((a, b) => b.layerVertices - a.layerVertices).map((layer) => { 347 | 348 | totalFeatures += layer.features.length; 349 | totalVertices += layer.layerVertices; 350 | totalKeys += layer.keys.length; 351 | totalValues += layer.values.length; 352 | return [layer.name, layer.layerVertices, layer.features.length, layer.keys.length, layer.values.length]; 353 | 354 | }); 355 | 356 | Log.title("Tile information"); 357 | Log.log( 358 | ColoredString.format(ColoredString.green, "Layers in this tile: "), 359 | ColoredString.format(ColoredString.white, info.length) 360 | ); 361 | Log.log( 362 | ColoredString.format(ColoredString.green, "Features in this tile: "), 363 | ColoredString.format(ColoredString.white, totalFeatures) 364 | ); 365 | Log.log( 366 | ColoredString.format(ColoredString.green, "Vertices in this tile: "), 367 | ColoredString.format(ColoredString.white, totalVertices) 368 | ); 369 | Log.log( 370 | ColoredString.format(ColoredString.green, "Keys in this tile: "), 371 | ColoredString.format(ColoredString.white, totalKeys) 372 | ); 373 | Log.log( 374 | ColoredString.format(ColoredString.green, "Values in this tile: "), 375 | ColoredString.format(ColoredString.white, totalValues) 376 | ); 377 | Log.log( 378 | ColoredString.format(ColoredString.green, "Layers: ") 379 | ); 380 | Log.table(["Layer name", "# of vertices", "# of features", "# of keys", "# of values"], info); 381 | 382 | } 383 | 384 | static printSlimProcessResults(removedLayers) { 385 | 386 | const messages = []; 387 | const levels = Object.keys(removedLayers.perLevel); 388 | for (const level of levels) { 389 | 390 | const numFeatures = removedLayers.perLevel[level]; 391 | messages.push(`Removed ${numFeatures} features in level ${level}`); 392 | 393 | } 394 | 395 | const names = Object.keys(removedLayers.perLayerName).sort(); 396 | for (const name of names) { 397 | 398 | const layerData = removedLayers.perLayerName[name]; 399 | messages.push(`Removed layer ${name} from zoom levels ${Array.from(layerData).sort((a, b) => a - b).join(", ")}`); 400 | 401 | } 402 | 403 | Log.list("Process results", messages); 404 | if (messages.length === 0) { 405 | 406 | Log.log("Nothing removed"); 407 | 408 | } 409 | 410 | } 411 | 412 | } 413 | 414 | module.exports = UI; 415 | -------------------------------------------------------------------------------- /src/command-line-options.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | module.exports = [ 5 | {name: "help", alias: "h", type: Boolean, description: "Display this usage guide"}, 6 | {name: "mbtiles", alias: "m", type: String, description: "The input .mbtile to process"}, 7 | {name: "style", alias: "s", type: String, description: "The input style to process"}, 8 | {name: "out", alias: "o", type: String, description: "The output file" }, 9 | {name: "row", alias: "x", type: String, description: "The X coordinate of a tile" }, 10 | {name: "column", alias: "y", type: String, description: "The Y coordinate of a tile" }, 11 | {name: "zoom", alias: "z", type: String, description: "The Z coordinate of a tile" }, 12 | {name: "tolerance", alias: "t", type: String, description: "The simplification tolerance" }, 13 | {name: "layer", alias: "l", type: String, description: "The name of the layer we want to simplify" }, 14 | {name: "verbose", alias: "b", type: Boolean, description: "Create verbose logs"}, 15 | {name: "PBFUrl", alias: "u", type: String, description: "The URL of a PBF buffer we want to examine"}, 16 | ]; 17 | -------------------------------------------------------------------------------- /src/core/ColoredString.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const chalk = require("chalk"); 5 | 6 | class ColoredString { 7 | 8 | static format(color, ...args) { 9 | 10 | return color(args); 11 | 12 | } 13 | 14 | } 15 | 16 | ColoredString.blue = chalk.blue; 17 | ColoredString.yellow = chalk.yellow; 18 | ColoredString.red = chalk.red; 19 | ColoredString.green = chalk.green; 20 | ColoredString.white = chalk.white; 21 | ColoredString.bold = chalk.bold; 22 | 23 | module.exports = ColoredString; 24 | -------------------------------------------------------------------------------- /src/core/DataConverter.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const geojsonvt = require("geojson-vt"); 5 | const Pbf = require("pbf"); 6 | const { VectorTile } = require("@mapbox/vector-tile"); 7 | const vtpbf = require("vt-pbf"); 8 | const Utils = require("./Utils"); 9 | 10 | class DataConverter { 11 | 12 | static async mVTLayers2GeoJSON(tilePBF, zoomLevel, column, row) { 13 | 14 | const layers = {}; 15 | const tile = new VectorTile(new Pbf(tilePBF)); 16 | const layerNames = Object.keys(tile.layers); 17 | await Utils.asyncForEach(layerNames, async (layerName) => { 18 | 19 | const geojson = await DataConverter.mVTLayer2GeoJSON(tile, layerName, zoomLevel, column, row); 20 | layers[layerName] = geojson; 21 | 22 | }); 23 | 24 | return layers; 25 | 26 | } 27 | 28 | static mVTLayer2GeoJSON(tile, layerName, zoomLevel, column, row) { 29 | 30 | return new Promise((resolve) => { 31 | 32 | const features = []; 33 | 34 | const layerObject = tile.layers[layerName]; 35 | for (let i = 0; i < layerObject.length; ++i) { 36 | 37 | const feature = layerObject.feature(i); 38 | const geoJSON = feature.toGeoJSON(row, column, zoomLevel); 39 | features.push(geoJSON); 40 | 41 | } 42 | 43 | resolve({ type: "FeatureCollection", features }); 44 | 45 | }); 46 | 47 | } 48 | 49 | static async geoJSONs2VTPBF(geojsons, zoomLevel, column, row, extent) { 50 | 51 | const tiles = {}; 52 | const layerNames = Object.keys(geojsons); 53 | await Utils.asyncForEach(layerNames, async (layerName) => { 54 | 55 | const tile = await DataConverter.geoJSON2MVTLayer(geojsons[layerName]); 56 | DataConverter.convertTileCoords(tile, zoomLevel, column, row, extent); 57 | tiles[layerName] = tile; 58 | 59 | }); 60 | 61 | const buffer = vtpbf.fromGeojsonVt(tiles, {version: 2}); 62 | const binBuffer = Buffer.from(buffer); 63 | 64 | return binBuffer; 65 | 66 | } 67 | 68 | static geoJSON2MVTLayer(geojson) { 69 | 70 | return new Promise((resolve) => { 71 | 72 | const tileset = geojsonvt(geojson, { 73 | tolerance: 0, 74 | maxZoom: 0, 75 | indexMaxZoom: 0, 76 | indexMaxPoints: 0 77 | }); 78 | 79 | resolve(tileset.tiles[0]); 80 | 81 | }); 82 | 83 | } 84 | 85 | static convertTileCoords(tile, zoomLevel, column, row, extent) { 86 | 87 | tile.features.forEach(feature => { 88 | 89 | if (feature.type === 1) { 90 | 91 | feature.geometry = [feature.geometry]; 92 | 93 | } 94 | 95 | feature.geometry.forEach(ring => { 96 | 97 | for (let i = 0; i < ring.length; i += 2) { 98 | 99 | const inTileCoordinateX = ring[i]; 100 | const inTileCoordinateY = ring[i + 1]; 101 | const worldCoordinateX = Utils.normalized2WorldX(inTileCoordinateX); 102 | const worldCoordinateY = Utils.normalized2WorldY(inTileCoordinateY); 103 | const vTCoordinateX = Utils.worldX2VT(zoomLevel, row, extent, worldCoordinateX); 104 | const vTCoordinateY = Utils.worldY2VT(zoomLevel, column, extent, worldCoordinateY); 105 | 106 | ring[i] = vTCoordinateX; 107 | ring[i + 1] = vTCoordinateY; 108 | 109 | } 110 | 111 | }); 112 | 113 | }); 114 | 115 | } 116 | 117 | } 118 | 119 | module.exports = DataConverter; 120 | -------------------------------------------------------------------------------- /src/core/IO.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const fs = require("fs"); 5 | 6 | class IO { 7 | 8 | static readSync(filename) { 9 | 10 | return fs.readFileSync(filename); 11 | 12 | } 13 | 14 | static copyFileSync(srcFile, destFile) { 15 | 16 | return fs.copyFileSync(srcFile, destFile); 17 | 18 | } 19 | 20 | static deleteFileSync(fileName) { 21 | 22 | return fs.unlinkSync(fileName); 23 | 24 | } 25 | 26 | } 27 | 28 | module.exports = IO; 29 | -------------------------------------------------------------------------------- /src/core/Log.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const ColoredString = require("./ColoredString"); 5 | require("console.table"); 6 | 7 | class Log { 8 | 9 | static log(...args) { 10 | 11 | Log.logMessage(Log.Verbose, args.join(" ")); 12 | 13 | } 14 | 15 | static info(...args) { 16 | 17 | Log.logMessage(Log.Info, ColoredString.format(ColoredString.blue, args.join(""))); 18 | 19 | } 20 | 21 | static warning(...args) { 22 | 23 | Log.logMessage(Log.Warning, ColoredString.format(ColoredString.yellow, args.join(""))); 24 | 25 | } 26 | 27 | static error(...args) { 28 | 29 | Log.logMessage(Log.Error, ColoredString.format(ColoredString.red, args.join(""))); 30 | 31 | } 32 | 33 | static title(...args) { 34 | 35 | Log.logMessage(Log.Title, ColoredString.format(ColoredString.blue, args.join(""))); 36 | 37 | } 38 | 39 | static table(columns, data) { 40 | 41 | console.table(columns, data); 42 | 43 | } 44 | 45 | static list(title, data) { 46 | 47 | Log.title(title); 48 | data.forEach(element => Log.log(`• ${element}`)); 49 | 50 | } 51 | 52 | static logMessage(errorLevel, text) { 53 | 54 | if (Log.errorLevel >= errorLevel) { 55 | 56 | console.log(text); 57 | 58 | } 59 | 60 | } 61 | 62 | } 63 | 64 | Log.Verbose = Number.POSITIVE_INFINITY; 65 | Log.Info = 2; 66 | Log.Warning = 1; 67 | Log.Error = 0; 68 | Log.Title = -1; 69 | Log.errorLevel = Log.Verbose; 70 | 71 | module.exports = Log; 72 | -------------------------------------------------------------------------------- /src/core/MapboxStyle.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const IO = require("./IO"); 5 | const MapboxStyleLayer = require("./MapboxStyleLayer"); 6 | 7 | /* 8 | * Loads a Mapbox Style following the specification found in 9 | * https://www.mapbox.com/mapbox-gl-js/style-spec/ 10 | */ 11 | class MapboxStyle { 12 | 13 | constructor(fileName) { 14 | 15 | this.style = {}; 16 | this.layerNamesBySource = {}; 17 | this.layerDataById = {}; 18 | this.fileName = fileName; 19 | 20 | } 21 | 22 | open() { 23 | 24 | const self = this; 25 | 26 | return new Promise((resolve, reject) => { 27 | 28 | try { 29 | 30 | self.style = JSON.parse(IO.readSync(self.fileName)); 31 | self.layerNamesBySource = self.groupLayerBySource(self.style.layers); 32 | self.layerDataById = self.mapLayersToObject(self.style.layers); 33 | delete self.style.layers; 34 | 35 | resolve(); 36 | 37 | } catch (err) { 38 | 39 | reject(err); 40 | 41 | } 42 | 43 | }); 44 | 45 | } 46 | 47 | groupLayerBySource(layers) { 48 | 49 | return layers.reduce( 50 | (obj, val) => { 51 | 52 | // According to the spec the background layers 53 | // don't have a source thus we omit them 54 | if (val.source) { 55 | 56 | if (!obj[val.source]) { 57 | 58 | obj[val.source] = []; 59 | 60 | } 61 | 62 | obj[val.source].push(val.id); 63 | 64 | } 65 | 66 | return obj; 67 | 68 | }, 69 | {} 70 | ); 71 | 72 | } 73 | 74 | mapLayersToObject(layers) { 75 | 76 | return layers.reduce((obj, val) => { 77 | 78 | const layerName = val["source-layer"]; 79 | const layerData = val; 80 | if (!obj.hasOwnProperty(layerName)) { 81 | 82 | // Since a source layer can be used in multiple layers, 83 | // we need to store an array of layer definitions and 84 | // check everyone of them 85 | obj[layerName] = []; 86 | 87 | } 88 | obj[layerName].push(new MapboxStyleLayer(layerData)); 89 | 90 | return obj; 91 | 92 | }, {}); 93 | 94 | } 95 | 96 | getLayerNamesFromSource(source) { 97 | 98 | const self = this; 99 | 100 | return self.layerNamesBySource[source]; 101 | 102 | } 103 | 104 | isLayerVisibleOnLevel(layerName, level) { 105 | 106 | const self = this; 107 | const layers = self.layerDataById[layerName]; 108 | return layers && layers.some((l) => { 109 | 110 | return ( 111 | l && 112 | l.isVisibleOnZoomLevel(level) && 113 | l.isRendered(level) && 114 | !l.areAllPropertiesFilteredOut() 115 | ); 116 | 117 | }); 118 | 119 | } 120 | 121 | } 122 | 123 | module.exports = MapboxStyle; 124 | -------------------------------------------------------------------------------- /src/core/MapboxStyleLayer.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const PaintPropertiesToCheck = require("../PaintPropertiesToCheck"); 5 | 6 | /* 7 | * Represents a layer according to the specification found in 8 | * https://www.mapbox.com/mapbox-gl-js/style-spec/ 9 | */ 10 | class MapboxStyleLayer { 11 | 12 | constructor(data) { 13 | 14 | this.data = JSON.parse(JSON.stringify(data)); // Deep clone of the object 15 | 16 | } 17 | 18 | isVisibleOnZoomLevel(level) { 19 | 20 | const self = this; 21 | 22 | return self.checkLayoutVisibility() && 23 | self.checkZoomUnderflow(level) && 24 | self.checkZoomOverflow(level); 25 | 26 | } 27 | 28 | checkLayoutVisibility() { 29 | 30 | const self = this; 31 | 32 | return !self.data.layout || !self.data.layout.hasOwnProperty("visibility") || self.data.layout.visibility === "visible"; 33 | 34 | } 35 | 36 | checkZoomUnderflow(level) { 37 | 38 | const self = this; 39 | 40 | return !self.data.minzoom || level >= self.data.minzoom; 41 | 42 | } 43 | 44 | checkZoomOverflow(level) { 45 | 46 | const self = this; 47 | 48 | // Mapbox style spec states that "At zoom levels equal to or greater than the maxzoom, the layer will be hidden" 49 | return !self.data.maxzoom || self.data.maxzoom > level; 50 | 51 | } 52 | 53 | isRendered(level) { 54 | 55 | const self = this; 56 | let isRendered = true; 57 | 58 | for (let i = 0; isRendered && i < PaintPropertiesToCheck.length; ++i) { 59 | 60 | isRendered = self.checkPaintPropertyNotZero(PaintPropertiesToCheck[i], level); 61 | 62 | } 63 | 64 | return isRendered; 65 | 66 | } 67 | 68 | checkPaintPropertyNotZero(propertyName, level) { 69 | 70 | const self = this; 71 | const isObject = self.data.paint && self.data.paint.hasOwnProperty(propertyName) && 72 | typeof self.data.paint[propertyName] === "object"; 73 | 74 | let isPropertyNotZero = false; 75 | 76 | if (isObject) { 77 | 78 | isPropertyNotZero = self.checkStopNotZeroInLevel(self.data.paint[propertyName], level); 79 | 80 | } else { 81 | 82 | isPropertyNotZero = !self.data.paint || !self.data.paint.hasOwnProperty(propertyName) || self.data.paint[propertyName] !== 0; 83 | 84 | } 85 | 86 | return isPropertyNotZero; 87 | 88 | } 89 | 90 | checkStopNotZeroInLevel(stops, level) { 91 | 92 | let isNotZero = true; 93 | 94 | if (stops.base && stops.stops) { 95 | 96 | const stop = stops.stops.find((elem) => elem[0] === level); 97 | 98 | if (stop) { 99 | 100 | isNotZero = stop && stop[1] !== 0; 101 | 102 | } 103 | 104 | } 105 | 106 | return isNotZero; 107 | 108 | } 109 | 110 | areAllPropertiesFilteredOut() { 111 | 112 | return false; 113 | 114 | } 115 | 116 | } 117 | 118 | module.exports = MapboxStyleLayer; 119 | -------------------------------------------------------------------------------- /src/core/SQLite.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const sqlite3 = require("sqlite3"); 5 | 6 | class SQLite { 7 | 8 | constructor() { 9 | 10 | this.db = null; 11 | 12 | } 13 | 14 | open(fileName) { 15 | 16 | return new Promise((resolve, reject) => { 17 | 18 | this.db = new sqlite3.Database(fileName, sqlite3.OPEN_READWRITE, (error) => { 19 | 20 | if (error === null) { 21 | 22 | resolve(); 23 | 24 | } else { 25 | 26 | reject(`SQLite::open ${error.message} while trying to open the following file: ${fileName}`); 27 | 28 | } 29 | 30 | }); 31 | 32 | }); 33 | 34 | } 35 | 36 | get(sql, params) { 37 | 38 | return new Promise((resolve, reject) => { 39 | 40 | this.db.get(sql, params, (error, row) => { 41 | 42 | if (error === null) { 43 | 44 | resolve(row); 45 | 46 | } else { 47 | 48 | reject(`SQLite::get ${error.message} while running the following query: ${sql}`); 49 | 50 | } 51 | 52 | }); 53 | 54 | }); 55 | 56 | } 57 | 58 | all(sql, params) { 59 | 60 | return new Promise((resolve, reject) => { 61 | 62 | this.db.all(sql, params, (error, rows) => { 63 | 64 | if (error === null) { 65 | 66 | resolve(rows); 67 | 68 | } else { 69 | 70 | reject(`SQLite::all ${error.message} while running the following query: ${sql}`); 71 | 72 | } 73 | 74 | }); 75 | 76 | }); 77 | 78 | } 79 | 80 | run(sql, params) { 81 | 82 | return new Promise((resolve, reject) => { 83 | 84 | this.db.run(sql, params, (error) => { 85 | 86 | if (error === null) { 87 | 88 | resolve(); 89 | 90 | } else { 91 | 92 | reject(`SQLite::run ${error.message} while running the following query: ${sql}`); 93 | 94 | } 95 | 96 | }); 97 | 98 | }); 99 | 100 | } 101 | 102 | beginTransaction() { 103 | 104 | const self = this; 105 | return self.run("BEGIN TRANSACTION"); 106 | 107 | } 108 | 109 | endTransaction() { 110 | 111 | const self = this; 112 | return self.run("END TRANSACTION"); 113 | 114 | } 115 | 116 | } 117 | 118 | module.exports = SQLite; 119 | -------------------------------------------------------------------------------- /src/core/Simplifier.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const simplify = require("simplify-geojson"); 5 | 6 | class Simplifier { 7 | 8 | static simplifyGeoJSON(geojson, tolerance) { 9 | 10 | return new Promise((resolve, reject) => { 11 | 12 | try { 13 | 14 | const simplified = simplify(geojson, tolerance); 15 | resolve(simplified); 16 | 17 | } catch (err) { 18 | 19 | reject(err); 20 | 21 | } 22 | 23 | }); 24 | 25 | } 26 | 27 | } 28 | 29 | module.exports = Simplifier; 30 | -------------------------------------------------------------------------------- /src/core/Utils.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const axios = require("axios"); 5 | 6 | class Utils { 7 | 8 | static async asyncForEach(array, callback) { 9 | 10 | // The usual forEach doesn't wait for the function to finish so if we 11 | // use an async function it won't work 12 | 13 | for (let index = 0; index < array.length; ++index) { 14 | 15 | await callback(array[index], index, array); 16 | 17 | } 18 | 19 | } 20 | 21 | static toRadians(degrees) { 22 | 23 | return degrees * Math.PI / 180.0; 24 | 25 | } 26 | 27 | static toDegrees(radians) { 28 | 29 | return radians * 180.0 / Math.PI; 30 | 31 | } 32 | 33 | // https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Lon..2Flat._to_tile_numbers_2 34 | static tileX2Lon(x, zoomLevel) { 35 | 36 | return (x / Math.pow(2, zoomLevel) * 360.0 - 180.0); 37 | 38 | } 39 | 40 | static tileY2Lat(y, zoomLevel) { 41 | 42 | const n = Math.PI - 2 * Math.PI * y / Math.pow(2, zoomLevel); 43 | return Utils.toDegrees(Math.atan(0.5 * (Math.exp(n) - Math.exp(-n)))); 44 | 45 | } 46 | 47 | static lon2TileX(lon, zoomLevel) { 48 | 49 | return (Math.floor((lon + 180) / 360 * Math.pow(2, zoomLevel))); 50 | 51 | } 52 | 53 | static lat2TileY(lat, zoomLevel) { 54 | 55 | return (Math.floor((1 - Math.log(Math.tan(Utils.toRadians(lat)) + 1 / Math.cos(Utils.toRadians(lat))) / Math.PI) / 2 * Math.pow(2, zoomLevel))); 56 | 57 | } 58 | 59 | static tile2LonLat(zoomLevel, column, row) { 60 | 61 | const lon = Utils.tileX2Lon(column, zoomLevel); 62 | const lat = Utils.tileY2Lat(row, zoomLevel); 63 | 64 | return {lon, lat}; 65 | 66 | } 67 | 68 | static lonLat2Tile(zoomLevel, lon, lat) { 69 | 70 | const row = Utils.lat2TileY(lat, zoomLevel); 71 | const column = Utils.lon2TileX(lon, zoomLevel); 72 | 73 | return {row, column}; 74 | 75 | } 76 | 77 | // From geojson-vt 78 | static world2NormalizedX(lon) { 79 | 80 | return lon / 360 + 0.5; 81 | 82 | } 83 | 84 | // From geojson-vt 85 | static world2NormalizedY(lat) { 86 | 87 | const sin = Math.sin(lat * Math.PI / 180); 88 | const y2 = 0.5 - 0.25 * Math.log((1 + sin) / (1 - sin)) / Math.PI; 89 | return y2 < 0 ? 0 : y2 > 1 ? 1 : y2; 90 | 91 | } 92 | 93 | static world2Normalized(lon, lat) { 94 | 95 | return { 96 | x: Utils.world2NormalizedX(lon), 97 | y: Utils.world2NormalizedY(lat) 98 | }; 99 | 100 | } 101 | 102 | static normalized2WorldX(x) { 103 | 104 | return (x - 0.5) * 360; 105 | 106 | } 107 | 108 | static normalized2WorldY(y) { 109 | 110 | const y2 = Math.exp(Math.PI * (y - 0.5) / -0.25); 111 | return Math.asin((y2 - 1) / (1 + y2)) * 180 / Math.PI; 112 | 113 | } 114 | 115 | static normalized2World(x, y) { 116 | 117 | return { 118 | lon: Utils.normalized2WorldX(x), 119 | lat: Utils.normalized2WorldY(y) 120 | }; 121 | 122 | } 123 | 124 | // From @mapbox/vector-tile 125 | static vT2WorldX(zoomLevel, row, extent, x) { 126 | 127 | const size = extent * Math.pow(2, zoomLevel); 128 | const x0 = extent * row; 129 | 130 | return (x + x0) * 360 / size - 180; 131 | 132 | } 133 | 134 | // From @mapbox/vector-tile 135 | static vT2WorldY(zoomLevel, column, extent, y) { 136 | 137 | const size = extent * Math.pow(2, zoomLevel); 138 | const y0 = extent * column; 139 | const y2 = 180 - (y + y0) * 360 / size; 140 | 141 | return 360 / Math.PI * Math.atan(Math.exp(Utils.toRadians(y2))) - 90; 142 | 143 | } 144 | 145 | static vt2World(zoomLevel, column, row, extent, x, y) { 146 | 147 | return { 148 | lon: Utils.vT2WorldX(zoomLevel, row, extent, x), 149 | lat: Utils.vT2WorldY(zoomLevel, column, extent, y) 150 | }; 151 | 152 | } 153 | 154 | static worldX2VT(zoomLevel, row, extent, x) { 155 | 156 | const size = extent * Math.pow(2, zoomLevel); 157 | const x0 = extent * row; 158 | 159 | return (x + 180) * size / 360 - x0; 160 | 161 | } 162 | 163 | static worldY2VT(zoomLevel, column, extent, y) { 164 | 165 | const size = extent * Math.pow(2, zoomLevel); 166 | const y0 = extent * column; 167 | const tan = Math.tan(((y + 90) * Math.PI / 360)); 168 | const a = Utils.toDegrees(Math.log(tan)); 169 | 170 | return (180 - a) * size / 360 - y0; 171 | 172 | } 173 | 174 | static world2VT(zoomLevel, column, row, extent, lon, lat) { 175 | 176 | return { 177 | x: Utils.worldX2VT(zoomLevel, row, extent, lon), 178 | y: Utils.worldY2VT(zoomLevel, column, extent, lat) 179 | }; 180 | 181 | } 182 | 183 | static async loadFromURL(url) { 184 | 185 | return new Promise((resolve, reject) => { 186 | 187 | axios.get(url, {responseType: "arraybuffer"}) 188 | .then((data) => { 189 | 190 | resolve(data.data); 191 | 192 | }, 193 | (err) => reject(err) 194 | ); 195 | 196 | }); 197 | 198 | } 199 | 200 | } 201 | 202 | module.exports = Utils; 203 | -------------------------------------------------------------------------------- /src/core/VTProcessor.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | /*eslint camelcase: ["error", {allow: ["zoom_level", "tile_row", "tile_column"]}]*/ 3 | "use strict"; 4 | 5 | const Listr = require("listr"); 6 | const { Observable } = require("rxjs"); 7 | const DataConverter = require("./DataConverter"); 8 | const IO = require("./IO"); 9 | const Log = require("./Log"); 10 | const MapboxStyle = require("./MapboxStyle"); 11 | const Simplifier = require("./Simplifier"); 12 | const UI = require("../UI"); 13 | const Utils = require("./Utils"); 14 | const VTReader = require("./VTReader"); 15 | const VTWriter = require("./VTWriter"); 16 | 17 | class VTProcessor { 18 | 19 | static showInfo(filename) { 20 | 21 | const reader = new VTReader(filename); 22 | 23 | const tasks = [ 24 | {title: "Parsing VT file contents", task: () => reader.open().catch((err) => { 25 | 26 | throw new Error(err); 27 | 28 | })} 29 | ]; 30 | 31 | const taskRunner = new Listr(tasks); 32 | taskRunner.run().then( 33 | async () => { 34 | 35 | const {vtSummary, tiles} = await VTProcessor.logInfo(reader); 36 | UI.printMetadata(reader.metadata.minzoom, reader.metadata.mazoom, reader.metadata.format, 37 | reader.metadata.center, reader.layers); 38 | VTProcessor.infoLoop(reader, vtSummary, tiles); 39 | 40 | }, 41 | err => { 42 | 43 | Log.error(err); 44 | 45 | } 46 | ); 47 | 48 | } 49 | 50 | static async logInfo(reader) { 51 | 52 | if (!reader.isOpen) { 53 | 54 | Log.error("VTProcessor::showInfo() ", "VTReader not open"); 55 | 56 | } 57 | 58 | try { 59 | 60 | const vtSummary = await reader.getVTSummary(); 61 | const tiles = await reader.getTooBigTilesNumber(); 62 | return {vtSummary, tiles}; 63 | 64 | } catch (err) { 65 | 66 | Log.error(err); 67 | 68 | } 69 | 70 | } 71 | 72 | static async infoLoop(reader, vtSummary, tiles) { 73 | 74 | UI.printSummaryTable(vtSummary, tiles, VTProcessor.avgTileSizeLimit, VTProcessor.avgTileSizeWarning, reader.tileSizeLimit); 75 | 76 | while (await UI.wantMoreInfoQuestion()) { 77 | 78 | const selectedLevel = await UI.selectLevelPrompt(vtSummary, VTProcessor.avgTileSizeLimit, VTProcessor.avgTileSizeWarning); 79 | const {data, buckets} = await VTProcessor.computeLevelInfo(reader, selectedLevel); 80 | UI.showTileDistributionData(data, VTProcessor.avgTileSizeLimit, VTProcessor.avgTileSizeWarning); 81 | 82 | while (await UI.tilesInBucketQuestion()) { 83 | 84 | const selectedBucket = await UI.selectBucketPrompt(data); 85 | UI.showBucketInfo(buckets[selectedBucket], reader.tileSizeLimit); 86 | 87 | while (await UI.tileInfoQuestion()) { 88 | 89 | const tileIndex = await UI.selectTilePrompt(buckets[selectedBucket], reader.tileSizeLimit); 90 | const tileData = await VTProcessor.computeTileData(reader, tileIndex.zoom_level, tileIndex.tile_column, tileIndex.tile_row); 91 | const vt = await DataConverter.mVTLayers2GeoJSON(tileData.rawPBF, tileIndex.zoom_level, tileIndex.tile_column, tileIndex.tile_row); 92 | UI.showTileInfo(tileData, vt); 93 | UI.showBucketInfo(buckets[selectedBucket], reader.tileSizeLimit); 94 | 95 | } 96 | 97 | UI.showTileDistributionData(data, VTProcessor.avgTileSizeLimit, VTProcessor.avgTileSizeWarning); 98 | 99 | } 100 | 101 | UI.printSummaryTable(vtSummary, tiles, VTProcessor.avgTileSizeLimit, VTProcessor.avgTileSizeWarning); 102 | 103 | } 104 | 105 | } 106 | 107 | static async computeLevelInfo(reader, zoomLevel) { 108 | 109 | const levelTiles = await reader.getLevelTiles(zoomLevel); 110 | levelTiles.sort((a, b) => a.size - b.size); 111 | 112 | const buckets = []; 113 | const data = []; 114 | let tiles = []; 115 | const numBuckets = 10; 116 | let minSize = levelTiles[0].size; 117 | const maxSize = levelTiles[levelTiles.length - 1].size; 118 | const totalSize = levelTiles.reduce((accum, elem) => accum + elem.size, 0); 119 | const totalNumTiles = levelTiles.length; 120 | const bucketSize = (maxSize - minSize) / numBuckets; 121 | let currentBucketMaxSize = minSize + bucketSize; 122 | let processedTilesSize = 0; 123 | 124 | for (let i = 0; i < totalNumTiles; ++i) { 125 | 126 | if (levelTiles[i].size <= currentBucketMaxSize) { 127 | 128 | tiles.push(levelTiles[i]); 129 | 130 | } else { 131 | 132 | VTProcessor.addTilesToBucket(minSize, currentBucketMaxSize, totalNumTiles, 133 | totalSize, tiles, i, processedTilesSize, buckets, data); 134 | 135 | tiles = [levelTiles[i]]; 136 | minSize = currentBucketMaxSize; 137 | currentBucketMaxSize += bucketSize; 138 | 139 | } 140 | 141 | processedTilesSize += levelTiles[i].size; 142 | 143 | } 144 | 145 | VTProcessor.addTilesToBucket(minSize, currentBucketMaxSize, totalNumTiles, 146 | totalSize, tiles, totalNumTiles, processedTilesSize, buckets, data); 147 | 148 | return {data, buckets}; 149 | 150 | } 151 | 152 | static addTilesToBucket(minSize, maxSize, totalNumTiles, totalSize, tiles, processedTiles, processedTilesSize, buckets, data) { 153 | 154 | const currentBucketSize = tiles.reduce((accum, elem) => accum + elem.size, 0); 155 | const currentBucketSizePc = (currentBucketSize / totalSize) * 100.0; 156 | const currentPc = (tiles.length / totalNumTiles) * 100.0; 157 | const runningAvgSize = (processedTilesSize / processedTiles); 158 | let accumPc = 0; 159 | let accumBucketSizePc = 0; 160 | 161 | if (data.length !== 0) { 162 | 163 | accumPc = data[data.length - 1].accumPc; // Restore previous accumulated % 164 | accumBucketSizePc = data[data.length - 1].accumBucketSizePc; // Restore previous accumulated bucket size % 165 | 166 | } 167 | 168 | accumPc += currentPc; 169 | accumBucketSizePc += currentBucketSizePc; 170 | 171 | data.push({ 172 | minSize, 173 | maxSize, 174 | length: tiles.length, 175 | runningAvgSize, 176 | currentPc, 177 | currentBucketSizePc, 178 | accumPc, 179 | accumBucketSizePc 180 | }); 181 | buckets.push(tiles); 182 | 183 | } 184 | 185 | static async computeTileData(reader, zoomLevel, column, row) { 186 | 187 | const tileData = await reader.getTileData(zoomLevel, column, row); 188 | return tileData; 189 | 190 | } 191 | 192 | static slim(inputFile, styleFile, outputFile) { 193 | 194 | const outputFileName = outputFile || `${inputFile.slice(0, -8)}_out.mbtiles`; 195 | const reader = new VTReader(inputFile); 196 | const style = new MapboxStyle(styleFile); 197 | const writer = new VTWriter(outputFileName); 198 | 199 | const tasks = [ 200 | { 201 | title: "Parsing VT file contents", 202 | task: () => reader.open(true).catch(err => { 203 | 204 | throw new Error(err); 205 | 206 | }) 207 | }, 208 | { 209 | title: "Parsing the style file", 210 | task: () => { 211 | 212 | style.open(); 213 | 214 | } 215 | }, 216 | { 217 | title: "Processing tiles", 218 | task: (ctx) => { 219 | 220 | return new Observable(observer => { 221 | 222 | VTProcessor.slimVT(reader, style, observer).then((data) => { 223 | 224 | ctx.isUsingImagesTable = reader.hasImagesTable; 225 | ctx.newVTData = data.newVTData; 226 | ctx.removedLayers = data.removedLayers; 227 | observer.complete(); 228 | 229 | }) 230 | .catch((errMsg) => { 231 | 232 | observer.error(errMsg); 233 | 234 | }); 235 | 236 | }); 237 | 238 | } 239 | }, 240 | { 241 | title: `Writing output file to ${outputFileName}`, 242 | task: (ctx) => { 243 | 244 | return new Promise(async (resolve, reject) => { 245 | 246 | IO.copyFileSync(inputFile, outputFileName); 247 | 248 | try { 249 | 250 | await writer.open(); 251 | await writer.write(ctx.newVTData, ctx.isUsingImagesTable); 252 | resolve(); 253 | 254 | } catch (error) { 255 | 256 | IO.deleteFileSync(outputFileName); 257 | reject(error); 258 | 259 | } 260 | 261 | }); 262 | 263 | } 264 | } 265 | ]; 266 | 267 | const taskRunner = new Listr(tasks); 268 | taskRunner.run() 269 | .then( 270 | ctx => UI.printSlimProcessResults(ctx.removedLayers) 271 | , 272 | err => Log.error(err) 273 | ); 274 | 275 | } 276 | 277 | static slimVT(reader, styleParser, observer) { 278 | 279 | const newVTData = []; 280 | const removedLayers = { perLevel: {}, perLayerName: {}}; 281 | 282 | return new Promise((resolve, reject) => { 283 | 284 | try { 285 | 286 | reader.getTiles().then(async (indexes) => { 287 | 288 | let lastLevelProcessed = Infinity; 289 | 290 | await Utils.asyncForEach(indexes, async (tileIndex, loopIndex) => { 291 | 292 | try { 293 | 294 | await reader.getTileData(tileIndex.zoom_level, tileIndex.tile_column, tileIndex.tile_row).then(data => { 295 | 296 | delete data.rawPBF; 297 | VTProcessor.addTileLayersIfVisible(styleParser, data, tileIndex, newVTData, removedLayers); 298 | 299 | if (tileIndex.zoom_level !== lastLevelProcessed || (loopIndex % 100 === 0)) { 300 | 301 | observer.next(`Processing level ${tileIndex.zoom_level} tiles. Current progress: ${((loopIndex / indexes.length) * 100.0).toFixed(4)}%`); 302 | lastLevelProcessed = tileIndex.zoom_level; 303 | 304 | } 305 | 306 | }); 307 | 308 | } catch (err) { 309 | 310 | reject(err); 311 | 312 | } 313 | 314 | }); 315 | 316 | observer.complete(); 317 | resolve({newVTData, removedLayers}); 318 | 319 | }); 320 | 321 | } catch (err) { 322 | 323 | reject(err); 324 | 325 | } 326 | 327 | }); 328 | 329 | } 330 | 331 | static addTileLayersIfVisible(styleParser, tileData, tileIndex, newVTData, removedLayers) { 332 | 333 | const newVTLayers = []; 334 | const layers = Object.keys(tileData.layers); 335 | 336 | for (const index of layers) { 337 | 338 | const layer = tileData.layers[index]; 339 | if (styleParser.isLayerVisibleOnLevel(layer.name, tileIndex.zoom_level)) { 340 | 341 | newVTLayers.push(layer); 342 | 343 | } else { 344 | 345 | VTProcessor.addLayerToRemovedLayers(tileIndex.zoom_level, layer, removedLayers); 346 | tileData.layers[index] = null; // Free the memory allocated to this layer as we won't need it anymore 347 | 348 | } 349 | 350 | } 351 | 352 | newVTData.push({ 353 | zoom_level : tileIndex.zoom_level, 354 | tile_column : tileIndex.tile_column, 355 | tile_row : tileIndex.tile_row, 356 | layers : newVTLayers 357 | }); 358 | 359 | } 360 | 361 | static addLayerToRemovedLayers(zoomLevel, layer, layerSet) { 362 | 363 | if (!layerSet.perLevel.hasOwnProperty(zoomLevel)) { 364 | 365 | layerSet.perLevel[zoomLevel] = 0; 366 | 367 | } 368 | 369 | layerSet.perLevel[zoomLevel] += layer.features.length; 370 | 371 | if (!layerSet.perLayerName.hasOwnProperty(layer.name)) { 372 | 373 | layerSet.perLayerName[layer.name] = new Set(); 374 | 375 | } 376 | 377 | layerSet.perLayerName[layer.name].add(zoomLevel); 378 | 379 | } 380 | 381 | static simplifyTileLayer(inputFile, zoomLevel, column, row, layerName, tolerance) { 382 | 383 | const reader = new VTReader(inputFile); 384 | const writer = new VTWriter(inputFile); 385 | 386 | const tasks = [ 387 | { 388 | title: "Opening VT", 389 | task: () => reader.open().catch(err => { 390 | 391 | throw new Error(err); 392 | 393 | }) 394 | }, 395 | { 396 | title: "Reading tile", 397 | task: (ctx) => { 398 | 399 | return new Promise(async (resolve, reject) => { 400 | 401 | await reader.getTileData(zoomLevel, column, row) 402 | .then(data => { 403 | 404 | ctx.tileData = data; 405 | resolve(); 406 | 407 | }, 408 | (err) => reject(err) 409 | ); 410 | 411 | }); 412 | 413 | } 414 | }, 415 | { 416 | title: "Converting to GeoJSON", 417 | task: (ctx) => { 418 | 419 | return new Promise((resolve, reject) => { 420 | 421 | DataConverter.mVTLayers2GeoJSON(ctx.tileData.rawPBF, zoomLevel, column, row) 422 | .then((data) => { 423 | 424 | ctx.geojsons = data; 425 | resolve(); 426 | 427 | }, 428 | (err) => reject(err) 429 | ); 430 | 431 | }); 432 | 433 | } 434 | }, 435 | { 436 | title: `Simplifying layer ${layerName}`, 437 | task: (ctx) => { 438 | 439 | return new Promise((resolve, reject) => { 440 | 441 | const layerToSimplify = ctx.geojsons[layerName]; 442 | 443 | if (!layerToSimplify) { 444 | 445 | reject(`There is not a layer with name ${layerName} in the specified tile`); 446 | 447 | } 448 | 449 | ctx.startingCoordinatesNum = layerToSimplify.features.reduce((accum, feature) => accum + feature.geometry.coordinates.reduce((accum, ring) => (ring.length ? accum + ring.length : feature.geometry.coordinates.length / 2), 0), 0); 450 | Simplifier.simplifyGeoJSON(layerToSimplify, tolerance) 451 | .then(data => { 452 | 453 | ctx.simplifiedCoordinatesNum = data.features.reduce((accum, feature) => accum + feature.geometry.coordinates.reduce((accum, ring) => (ring.length ? accum + ring.length : feature.geometry.coordinates.length / 2), 0), 0); 454 | ctx.geojsons[layerName] = data; 455 | resolve(); 456 | 457 | }, 458 | (err) => reject(err) 459 | ); 460 | 461 | }); 462 | 463 | } 464 | }, 465 | { 466 | title: "Converting back to MVT", 467 | task: (ctx) => { 468 | 469 | return new Promise((resolve, reject) => { 470 | 471 | DataConverter.geoJSONs2VTPBF(ctx.geojsons, zoomLevel, column, row, ctx.tileData.layers[0].extent) 472 | .then((data) => { 473 | 474 | ctx.mvt = data; 475 | resolve(); 476 | 477 | }, 478 | (err) => reject(err) 479 | ); 480 | 481 | }); 482 | 483 | } 484 | }, 485 | { 486 | title: `Updating file ${inputFile}`, 487 | task: (ctx) => { 488 | 489 | return new Promise(async (resolve, reject) => { 490 | 491 | try { 492 | 493 | await writer.open(); 494 | await writer.writeTile(ctx.mvt, zoomLevel, column, row); 495 | resolve(); 496 | 497 | } catch (error) { 498 | 499 | reject(error); 500 | 501 | } 502 | 503 | }); 504 | 505 | } 506 | } 507 | ]; 508 | 509 | const taskRunner = new Listr(tasks); 510 | taskRunner.run() 511 | .then((ctx) => { 512 | 513 | Log.log(`Layer reduction ${((1.0 - ctx.simplifiedCoordinatesNum / ctx.startingCoordinatesNum) * 100.0).toFixed(2)}% (from ${ctx.startingCoordinatesNum} to ${ctx.simplifiedCoordinatesNum} vertices)`); 514 | 515 | }) 516 | .catch(err => Log.error(err)); 517 | 518 | } 519 | 520 | static showTileInfo(filename, zoomLevel, column, row) { 521 | 522 | const reader = new VTReader(filename); 523 | 524 | const tasks = [ 525 | { 526 | title: "Opening VT", 527 | task: () => reader.open().catch(err => { 528 | 529 | throw new Error(err); 530 | 531 | }) 532 | }, 533 | { 534 | title: "Reading tile", 535 | task: (ctx) => { 536 | 537 | return new Promise(async (resolve, reject) => { 538 | 539 | await reader.getTileData(zoomLevel, column, row) 540 | .then(data => { 541 | 542 | ctx.tileData = data; 543 | resolve(); 544 | 545 | }, 546 | (err) => reject(err) 547 | ); 548 | 549 | }); 550 | 551 | } 552 | }, 553 | { 554 | title: "Converting to GeoJSON", 555 | task: (ctx) => { 556 | 557 | return new Promise((resolve, reject) => { 558 | 559 | DataConverter.mVTLayers2GeoJSON(ctx.tileData.rawPBF, zoomLevel, column, row) 560 | .then((data) => { 561 | 562 | ctx.geojsons = data; 563 | resolve(); 564 | 565 | }, 566 | (err) => reject(err) 567 | ); 568 | 569 | }); 570 | 571 | } 572 | } 573 | ]; 574 | 575 | const taskRunner = new Listr(tasks); 576 | taskRunner.run().then( 577 | async (ctx) => { 578 | 579 | Log.log(JSON.stringify(ctx.geojsons)); 580 | 581 | }, 582 | err => Log.error(err) 583 | ); 584 | 585 | } 586 | 587 | static showURLTileInfo(url, zoomLevel, column, row) { 588 | 589 | const tasks = [ 590 | { 591 | title: "Downloading PBF", 592 | task: (ctx) => { 593 | 594 | return new Promise((resolve, reject) => { 595 | 596 | Utils.loadFromURL(url) 597 | .then((data) => { 598 | 599 | ctx.pbf = data; 600 | resolve(); 601 | 602 | }) 603 | .catch(err => { 604 | 605 | reject(err); 606 | 607 | }); 608 | 609 | }); 610 | 611 | } 612 | }, 613 | { 614 | title: "Converting to GeoJSON", 615 | task: (ctx) => { 616 | 617 | return new Promise((resolve, reject) => { 618 | 619 | DataConverter.mVTLayers2GeoJSON(ctx.pbf, zoomLevel, column, row) 620 | .then((data) => { 621 | 622 | ctx.geojsons = data; 623 | resolve(); 624 | 625 | }, 626 | (err) => reject(err) 627 | ); 628 | 629 | }); 630 | 631 | } 632 | } 633 | ]; 634 | 635 | const taskRunner = new Listr(tasks); 636 | taskRunner.run().then( 637 | async (ctx) => { 638 | 639 | Log.log(JSON.stringify(ctx.geojsons)); 640 | 641 | }, 642 | err => Log.error(err) 643 | ); 644 | 645 | } 646 | 647 | } 648 | 649 | VTProcessor.avgTileSizeWarning = 45; 650 | VTProcessor.avgTileSizeLimit = 50; 651 | 652 | module.exports = VTProcessor; 653 | -------------------------------------------------------------------------------- /src/core/VTReader.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | /*eslint camelcase: ["error", {allow: ["zoom_level", "tile_row", "tile_column"]}]*/ 3 | "use strict"; 4 | 5 | const Pbf = require("pbf"); 6 | const zlib = require("zlib"); 7 | const Log = require("./Log"); 8 | const SQLite = require("./SQLite"); 9 | const { Tile } = require("./vector-tile"); 10 | 11 | class VTReader { 12 | 13 | constructor(fileName) { 14 | 15 | this.db = new SQLite(); 16 | this.layers = []; 17 | this.metadata = {}; 18 | this.fileName = fileName; 19 | this.isOpen = false; 20 | this.hasData = false; 21 | this.data = null; 22 | this.hasImagesTable = false; 23 | 24 | } 25 | 26 | open(loadInMemory) { 27 | 28 | const self = this; 29 | 30 | return new Promise((resolve, reject) => { 31 | 32 | self.db.open(self.fileName) 33 | .then( 34 | () => { 35 | 36 | self.isOpen = true; 37 | self.checkImagesTable().then((hasImagesTable) => { 38 | 39 | self.hasImagesTable = hasImagesTable; 40 | 41 | self.parseMetadata().then(() => { 42 | 43 | if (loadInMemory) { 44 | 45 | self.loadTiles().then(() => resolve()); 46 | 47 | } else { 48 | 49 | resolve(); 50 | 51 | } 52 | 53 | }, 54 | (err) => reject(err) 55 | ); 56 | 57 | }, 58 | (err) => reject(err) 59 | ); 60 | 61 | }, 62 | (err) => reject(err) 63 | ); 64 | 65 | }); 66 | 67 | } 68 | 69 | checkImagesTable() { 70 | 71 | const self = this; 72 | return new Promise((resolve, reject) => { 73 | 74 | if (!self.isOpen) { 75 | 76 | reject("VT not open"); 77 | 78 | } 79 | self.db.all("SELECT 1 FROM sqlite_master WHERE type='table' and name='images'") 80 | .then((rows) => { 81 | 82 | resolve(rows.length !== 0); 83 | 84 | }); 85 | 86 | }); 87 | 88 | } 89 | 90 | parseMetadata() { 91 | 92 | const self = this; 93 | 94 | return new Promise((resolve, reject) => { 95 | 96 | if (!self.isOpen) { 97 | 98 | reject("VT not open"); 99 | 100 | } 101 | 102 | self.db.all("SELECT * FROM metadata") 103 | .then( 104 | (rows) => { 105 | 106 | self.metadata = rows.reduce((obj, val) => { 107 | 108 | obj[val.name] = val.value; 109 | return obj; 110 | 111 | }, {}); 112 | self.parseLayers(); 113 | resolve(); 114 | 115 | }, 116 | (err) => reject(err) 117 | ); 118 | 119 | }); 120 | 121 | } 122 | 123 | parseLayers() { 124 | 125 | const self = this; 126 | 127 | const data = self.metadata["json"] ? JSON.parse(self.metadata["json"]) : null; 128 | 129 | if (data) { 130 | 131 | self.layers = data["vector_layers"].map((layer) => layer.id).sort(); 132 | 133 | } else { 134 | 135 | Log.warning("No vector layers found"); 136 | 137 | } 138 | 139 | } 140 | 141 | getTiles() { 142 | 143 | const self = this; 144 | 145 | return new Promise((resolve, reject) => { 146 | 147 | if (!self.isOpen) { 148 | 149 | reject("VT not open"); 150 | 151 | } 152 | 153 | self.db.all("SELECT zoom_level, tile_column, tile_row FROM tiles ORDER BY zoom_level ASC") 154 | .then( 155 | (rows) => { 156 | 157 | resolve(rows); 158 | 159 | }, 160 | (err) => reject(err) 161 | ); 162 | 163 | }); 164 | 165 | } 166 | 167 | async loadTiles() { 168 | 169 | const self = this; 170 | 171 | return new Promise((resolve, reject) => { 172 | 173 | if (!self.isOpen) { 174 | 175 | reject("VT not open"); 176 | 177 | } 178 | 179 | if (!self.hasData) { 180 | 181 | self.db.all("SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles ORDER BY zoom_level ASC") 182 | .then( 183 | (rows) => { 184 | 185 | self.hasData = true; 186 | self.data = rows.reduce((obj, val) => { 187 | 188 | obj[`${val["zoom_level"]}_${val["tile_column"]}_${val["tile_row"]}`] = val["tile_data"]; 189 | return obj; 190 | 191 | }, {}); 192 | resolve(rows); 193 | 194 | }, 195 | (err) => reject(err) 196 | ); 197 | 198 | } 199 | 200 | }); 201 | 202 | } 203 | 204 | async getTileData(zoomLevel, column, row) { 205 | 206 | const self = this; 207 | 208 | return new Promise(async (resolve, reject) => { 209 | 210 | if (!self.isOpen) { 211 | 212 | reject("VT not open"); 213 | 214 | } 215 | 216 | if (self.hasData) { 217 | 218 | self.loadCachedData(resolve, reject, zoomLevel, column, row); 219 | 220 | } else { 221 | 222 | self.loadFromDatabase(resolve, reject, zoomLevel, column, row); 223 | 224 | } 225 | 226 | }); 227 | 228 | } 229 | 230 | loadCachedData(resolve, reject, zoomLevel, column, row) { 231 | 232 | const self = this; 233 | 234 | self.unzipTileData(self.data[`${zoomLevel}_${column}_${row}`], resolve, reject); 235 | 236 | } 237 | 238 | unzipTileData(data, resolve, reject) { 239 | 240 | zlib.gunzip(data, (err, buffer) => { 241 | 242 | if (err) { 243 | 244 | reject(new Error(`zlib : ${err.message}`)); 245 | 246 | } 247 | 248 | data = Tile.read(new Pbf(buffer)); 249 | data.rawPBF = buffer; 250 | resolve(data); 251 | 252 | }); 253 | 254 | } 255 | 256 | async loadFromDatabase(resolve, reject, zoomLevel, column, row) { 257 | 258 | const self = this; 259 | 260 | try { 261 | 262 | const rowData = await self.db.get(`SELECT tile_data, length(tile_data) as size FROM tiles WHERE zoom_level=${zoomLevel} AND tile_column=${column} AND tile_row=${row}`); 263 | const data = await new Promise((resolve) => { 264 | 265 | self.unzipTileData(rowData["tile_data"], resolve, reject); 266 | 267 | }); 268 | 269 | resolve(data); 270 | 271 | } catch (error) { 272 | 273 | reject(error); 274 | 275 | } 276 | 277 | } 278 | 279 | getVTSummary() { 280 | 281 | const self = this; 282 | 283 | return new Promise((resolve, reject) => { 284 | 285 | if (!self.isOpen) { 286 | 287 | reject("VT not open"); 288 | 289 | } 290 | 291 | self.db.all("SELECT zoom_level, COUNT(tile_column) as tiles, SUM(length(tile_data)) as size, AVG(length(tile_data)) as avgTileSize, MAX(length(tile_data)) as maxSize FROM tiles GROUP BY zoom_level ORDER BY zoom_level ASC") 292 | .then( 293 | (rows) => { 294 | 295 | resolve(rows); 296 | 297 | }, 298 | (err) => reject(err) 299 | ); 300 | 301 | }); 302 | 303 | } 304 | 305 | getLevelTiles(level) { 306 | 307 | const self = this; 308 | 309 | return new Promise((resolve, reject) => { 310 | 311 | if (!self.isOpen) { 312 | 313 | reject("VT not open"); 314 | 315 | } 316 | 317 | self.db.all(`SELECT zoom_level, tile_column, tile_row, length(tile_data) as size FROM tiles WHERE zoom_level=${level}`) 318 | .then( 319 | (rows) => { 320 | 321 | resolve(rows.map(row => { 322 | 323 | return { zoom_level: row.zoom_level, tile_column: row.tile_column, tile_row: row.tile_row, size: row.size / 1024.0}; 324 | 325 | })); 326 | 327 | }, 328 | (err) => reject(err) 329 | ); 330 | 331 | }); 332 | 333 | } 334 | 335 | getTooBigTilesNumber() { 336 | 337 | // The mapbox studio classic recommendations state that individual tiles must be less than 500Kb 338 | // https://www.mapbox.com/help/studio-classic-sources/#tiles-and-file-sizes 339 | 340 | const self = this; 341 | 342 | return new Promise((resolve, reject) => { 343 | 344 | if (!self.isOpen) { 345 | 346 | reject("VT not open"); 347 | 348 | } 349 | 350 | self.db.all(`SELECT zoom_level, COUNT(*) as num FROM tiles WHERE length(tile_data) > ${VTReader.tileSizeLimit * 1024} GROUP BY zoom_level ORDER BY zoom_level ASC`) 351 | .then( 352 | (rows) => { 353 | 354 | resolve(rows); 355 | 356 | }, 357 | (err) => reject(err) 358 | ); 359 | 360 | }); 361 | 362 | } 363 | 364 | } 365 | 366 | VTReader.tileSizeLimit = 500; 367 | 368 | module.exports = VTReader; 369 | -------------------------------------------------------------------------------- /src/core/VTWriter.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | /*eslint camelcase: ["error", {allow: ["zoom_level", "tile_row", "tile_column"]}]*/ 3 | "use strict"; 4 | 5 | const Pbf = require("pbf"); 6 | const zlib = require("zlib"); 7 | const SQLite = require("./SQLite"); 8 | const Utils = require("./Utils"); 9 | const { Tile } = require("./vector-tile"); 10 | 11 | class VTWriter { 12 | 13 | constructor(fileName) { 14 | 15 | this.db = new SQLite(); 16 | this.fileName = fileName; 17 | 18 | } 19 | 20 | open() { 21 | 22 | const self = this; 23 | 24 | return new Promise((resolve, reject) => { 25 | 26 | self.db.open(self.fileName).then( 27 | () => { 28 | 29 | self.isOpen = true; 30 | resolve(); 31 | 32 | }, 33 | (err) => { 34 | 35 | reject(err); 36 | 37 | } 38 | ); 39 | 40 | }); 41 | 42 | } 43 | 44 | write(data, isUsingImagesTable) { 45 | 46 | const self = this; 47 | 48 | return new Promise(async (resolve, reject) => { 49 | 50 | if (!self.isOpen) { 51 | 52 | reject("VT not open"); 53 | 54 | } 55 | 56 | try { 57 | 58 | await self.updateOrRemoveTiles(data, isUsingImagesTable); 59 | await self.updateMetadata(); 60 | resolve(); 61 | 62 | } catch (error) { 63 | 64 | reject(error); 65 | 66 | } 67 | 68 | }); 69 | 70 | } 71 | 72 | async updateOrRemoveTiles(data, isUsingImagesTable) { 73 | 74 | const self = this; 75 | 76 | return new Promise(async (resolve, reject) => { 77 | 78 | try { 79 | 80 | await self.db.beginTransaction(); 81 | 82 | await Utils.asyncForEach(data, async element => { 83 | 84 | if (element.layers.length !== 0) { 85 | 86 | const pbf = new Pbf(); 87 | Tile.write(element, pbf); 88 | const buffer = pbf.finish(); 89 | const binBuffer = Buffer.from(buffer); 90 | const compressedBuffer = zlib.gzipSync(binBuffer, {level: zlib.constants.Z_BEST_COMPRESSION}); 91 | 92 | if (isUsingImagesTable) { 93 | 94 | await self.updateImage(element.zoom_level, element.tile_row, element.tile_column, compressedBuffer); 95 | 96 | } else { 97 | 98 | await self.updateTile(element.zoom_level, element.tile_row, element.tile_column, compressedBuffer); 99 | 100 | } 101 | 102 | } else if (isUsingImagesTable) { 103 | 104 | await self.deleteImage(element.zoom_level, element.tile_row, element.tile_column); 105 | 106 | } else { 107 | 108 | await self.deleteTile(element.zoom_level, element.tile_row, element.tile_column); 109 | 110 | } 111 | 112 | }); 113 | 114 | await self.db.endTransaction(); 115 | 116 | // Updates the file size on disk by freeing the empty space left behind by the deletes 117 | await self.vacuum(); 118 | resolve(); 119 | 120 | } catch (error) { 121 | 122 | reject(error); 123 | 124 | } 125 | 126 | }); 127 | 128 | } 129 | 130 | updateTile(zoom_level, tile_row, tile_column, data) { 131 | 132 | const self = this; 133 | 134 | return self.db.run(`UPDATE tiles SET tile_data=(?) WHERE zoom_level=${zoom_level} AND tile_row=${tile_row} AND tile_column=${tile_column}`, data); 135 | 136 | } 137 | 138 | deleteTile(zoom_level, tile_row, tile_column) { 139 | 140 | const self = this; 141 | 142 | return self.db.run(`DELETE FROM tiles WHERE zoom_level=${zoom_level} AND tile_row=${tile_row} AND tile_column=${tile_column}`); 143 | 144 | } 145 | 146 | updateImage(zoom_level, tile_row, tile_column, data) { 147 | 148 | const self = this; 149 | 150 | return self.db.run(`UPDATE images SET tile_data=(?) WHERE tile_id=(SELECT tile_id FROM map WHERE zoom_level=${zoom_level} AND tile_row=${tile_row} AND tile_column=${tile_column} LIMIT 1)`, data); 151 | 152 | } 153 | 154 | deleteImage(zoom_level, tile_row, tile_column) { 155 | 156 | const self = this; 157 | 158 | return self.db.run(`DELETE FROM images WHERE tile_id=(SELECT tile_id FROM map WHERE zoom_level=${zoom_level} AND tile_row=${tile_row} AND tile_column=${tile_column} LIMIT 1)`); 159 | 160 | } 161 | 162 | async vacuum() { 163 | 164 | const self = this; 165 | 166 | return self.db.run("VACUUM"); 167 | 168 | } 169 | 170 | async updateMetadata() { 171 | 172 | const self = this; 173 | const bounds = "-180.0,-85.0511,180.0,85.0511"; 174 | 175 | const rows = await self.db.get("SELECT * FROM metadata WHERE name='bounds'"); 176 | 177 | if (rows) { 178 | 179 | return self.db.run(`UPDATE metadata SET value="${bounds}" WHERE name='bounds'`); 180 | 181 | } else { 182 | 183 | return self.db.run(`INSERT INTO metadata(name, value) VALUES("bounds", "${bounds}")`); 184 | 185 | } 186 | 187 | } 188 | 189 | async writeTile(binaryBuffer, z, y, x) { 190 | 191 | const self = this; 192 | 193 | const compressedBuffer = zlib.gzipSync(binaryBuffer, {level: zlib.constants.Z_BEST_COMPRESSION}); 194 | await self.updateImage(z, x, y, compressedBuffer); 195 | 196 | } 197 | 198 | } 199 | 200 | module.exports = VTWriter; 201 | -------------------------------------------------------------------------------- /src/core/vector-tile.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | /* eslint-disable */ 3 | "use strict"; // code generated by pbf v3.1.0 4 | 5 | // Tile ======================================== 6 | 7 | const Tile = exports.Tile = {}; 8 | 9 | Tile.read = function (pbf, end) { 10 | 11 | return pbf.readFields(Tile._readField, {layers: []}, end); 12 | 13 | }; 14 | Tile._readField = function (tag, obj, pbf) { 15 | 16 | if (tag === 3) { 17 | 18 | obj.layers.push(Tile.Layer.read(pbf, pbf.readVarint() + pbf.pos)); 19 | 20 | } 21 | 22 | }; 23 | Tile.write = function (obj, pbf) { 24 | 25 | if (obj.layers) { 26 | 27 | for (let i = 0; i < obj.layers.length; i++) { 28 | 29 | pbf.writeMessage(3, Tile.Layer.write, obj.layers[i]); 30 | 31 | } 32 | 33 | } 34 | 35 | }; 36 | 37 | Tile.GeomType = { 38 | "UNKNOWN": { 39 | "value": 0, 40 | "options": {} 41 | }, 42 | "POINT": { 43 | "value": 1, 44 | "options": {} 45 | }, 46 | "LINESTRING": { 47 | "value": 2, 48 | "options": {} 49 | }, 50 | "POLYGON": { 51 | "value": 3, 52 | "options": {} 53 | } 54 | }; 55 | 56 | // Tile.Value ======================================== 57 | 58 | Tile.Value = {}; 59 | 60 | Tile.Value.read = function (pbf, end) { 61 | 62 | return pbf.readFields(Tile.Value._readField, {string_value: "", float_value: 0, double_value: 0, int_value: 0, uint_value: 0, sint_value: 0, bool_value: false}, end); 63 | 64 | }; 65 | Tile.Value._readField = function (tag, obj, pbf) { 66 | 67 | if (tag === 1) { 68 | 69 | obj.string_value = pbf.readString(); 70 | 71 | } else if (tag === 2) { 72 | 73 | obj.float_value = pbf.readFloat(); 74 | 75 | } else if (tag === 3) { 76 | 77 | obj.double_value = pbf.readDouble(); 78 | 79 | } else if (tag === 4) { 80 | 81 | obj.int_value = pbf.readVarint(true); 82 | 83 | } else if (tag === 5) { 84 | 85 | obj.uint_value = pbf.readVarint(); 86 | 87 | } else if (tag === 6) { 88 | 89 | obj.sint_value = pbf.readSVarint(); 90 | 91 | } else if (tag === 7) { 92 | 93 | obj.bool_value = pbf.readBoolean(); 94 | 95 | } 96 | 97 | }; 98 | Tile.Value.write = function (obj, pbf) { 99 | 100 | if (obj.string_value) { 101 | 102 | pbf.writeStringField(1, obj.string_value); 103 | 104 | } 105 | if (obj.float_value) { 106 | 107 | pbf.writeFloatField(2, obj.float_value); 108 | 109 | } 110 | if (obj.double_value) { 111 | 112 | pbf.writeDoubleField(3, obj.double_value); 113 | 114 | } 115 | if (obj.int_value) { 116 | 117 | pbf.writeVarintField(4, obj.int_value); 118 | 119 | } 120 | if (obj.uint_value) { 121 | 122 | pbf.writeVarintField(5, obj.uint_value); 123 | 124 | } 125 | if (obj.sint_value) { 126 | 127 | pbf.writeSVarintField(6, obj.sint_value); 128 | 129 | } 130 | if (obj.bool_value) { 131 | 132 | pbf.writeBooleanField(7, obj.bool_value); 133 | 134 | } 135 | 136 | }; 137 | 138 | // Tile.Feature ======================================== 139 | 140 | Tile.Feature = {}; 141 | 142 | Tile.Feature.read = function (pbf, end) { 143 | 144 | return pbf.readFields(Tile.Feature._readField, {id: 0, tags: [], type: 0, geometry: []}, end); 145 | 146 | }; 147 | Tile.Feature._readField = function (tag, obj, pbf) { 148 | 149 | if (tag === 1) { 150 | 151 | obj.id = pbf.readVarint(); 152 | 153 | } else if (tag === 2) { 154 | 155 | pbf.readPackedVarint(obj.tags); 156 | 157 | } else if (tag === 3) { 158 | 159 | obj.type = pbf.readVarint(); 160 | 161 | } else if (tag === 4) { 162 | 163 | pbf.readPackedVarint(obj.geometry); 164 | 165 | } 166 | 167 | }; 168 | Tile.Feature.write = function (obj, pbf) { 169 | 170 | if (obj.id) { 171 | 172 | pbf.writeVarintField(1, obj.id); 173 | 174 | } 175 | if (obj.tags) { 176 | 177 | pbf.writePackedVarint(2, obj.tags); 178 | 179 | } 180 | if (obj.type) { 181 | 182 | pbf.writeVarintField(3, obj.type); 183 | 184 | } 185 | if (obj.geometry) { 186 | 187 | pbf.writePackedVarint(4, obj.geometry); 188 | 189 | } 190 | 191 | }; 192 | 193 | // Tile.Layer ======================================== 194 | 195 | Tile.Layer = {}; 196 | 197 | Tile.Layer.read = function (pbf, end) { 198 | 199 | return pbf.readFields(Tile.Layer._readField, {version: 0, name: "", features: [], keys: [], values: [], extent: 0}, end); 200 | 201 | }; 202 | Tile.Layer._readField = function (tag, obj, pbf) { 203 | 204 | if (tag === 15) { 205 | 206 | obj.version = pbf.readVarint(); 207 | 208 | } else if (tag === 1) { 209 | 210 | obj.name = pbf.readString(); 211 | 212 | } else if (tag === 2) { 213 | 214 | obj.features.push(Tile.Feature.read(pbf, pbf.readVarint() + pbf.pos)); 215 | 216 | } else if (tag === 3) { 217 | 218 | obj.keys.push(pbf.readString()); 219 | 220 | } else if (tag === 4) { 221 | 222 | obj.values.push(Tile.Value.read(pbf, pbf.readVarint() + pbf.pos)); 223 | 224 | } else if (tag === 5) { 225 | 226 | obj.extent = pbf.readVarint(); 227 | 228 | } 229 | 230 | }; 231 | Tile.Layer.write = function (obj, pbf) { 232 | 233 | if (obj.version) { 234 | 235 | pbf.writeVarintField(15, obj.version); 236 | 237 | } 238 | if (obj.name) { 239 | 240 | pbf.writeStringField(1, obj.name); 241 | 242 | } 243 | if (obj.features) { 244 | 245 | for (var i = 0; i < obj.features.length; i++) { 246 | 247 | pbf.writeMessage(2, Tile.Feature.write, obj.features[i]); 248 | 249 | } 250 | 251 | } 252 | if (obj.keys) { 253 | 254 | for (i = 0; i < obj.keys.length; i++) { 255 | 256 | pbf.writeStringField(3, obj.keys[i]); 257 | 258 | } 259 | 260 | } 261 | if (obj.values) { 262 | 263 | for (i = 0; i < obj.values.length; i++) { 264 | 265 | pbf.writeMessage(4, Tile.Value.write, obj.values[i]); 266 | 267 | } 268 | 269 | } 270 | if (obj.extent) { 271 | 272 | pbf.writeVarintField(5, obj.extent); 273 | 274 | } 275 | 276 | }; 277 | -------------------------------------------------------------------------------- /src/usage-sections.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const commandLineOptions = require("./command-line-options"); 5 | 6 | module.exports = [ 7 | { 8 | header: "Vector Tile Weight Loser", 9 | content: "Reduces a Mapbox VT file removing all the layers not used in a style" 10 | }, 11 | { 12 | header: "Options", 13 | optionList: commandLineOptions 14 | 15 | }, 16 | { 17 | header: "Synopsis", 18 | content: [ 19 | "$ node index.js -m vtFile", 20 | "$ node index.js -m vtFile -s styleFile", 21 | "$ node index.js -m vtFile -s styleFile -o outputFile", 22 | "$ node index.js -m vtFile -x Row -y Column -z ZoomLevel -l layerName -t tolerance", 23 | "$ node index.js -m vtFile -x Row -y Column -z ZoomLevel", 24 | "$ node index.js -u url -x Row -y Column -z ZoomLevel", 25 | "$ node index.js --help", 26 | ] 27 | }, 28 | { 29 | content: [ 30 | "Project home: https://github.com/ibesora/vt-optimizer", 31 | "Made with love by @ibesora" 32 | ] 33 | } 34 | ]; 35 | -------------------------------------------------------------------------------- /test/unit/ColoredString.test.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const test = require("tap").test; 5 | const ColoredString = require("../../src/core/ColoredString"); 6 | 7 | test("ColoredString", (t) => { 8 | 9 | t.test("#format does change text", (t) => { 10 | 11 | t.notEquals(ColoredString.format(ColoredString.blue, "test"), "test"); 12 | t.end(); 13 | 14 | }); 15 | 16 | t.end(); 17 | 18 | }); 19 | -------------------------------------------------------------------------------- /test/unit/IO.test.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const test = require("tap").test; 5 | const IO = require("../../src/core/IO"); 6 | 7 | test("IO", (t) => { 8 | 9 | t.test("#readSync", (t) => { 10 | 11 | t.doesNotThrow(() => IO.readSync("./files/trails.mbtiles"), "Reads an existing file"); 12 | t.throws(() => IO.readSync("./files/trails2.mbtiles"), "Unexisting file"); 13 | t.end(); 14 | 15 | }); 16 | 17 | t.test("#copyFileSync", (t) => { 18 | 19 | t.doesNotThrow(() => IO.copyFileSync("./files/trails.mbtiles", "./files/trails2.mbtiles"), "Copies an existing file"); 20 | t.doesNotThrow(() => IO.readSync("./files/trails2.mbtiles"), "File exists"); 21 | t.throws(() => IO.copyFileSync("./files/trails3.mbtiles", "./files/aux.mbtiles"), "File does not exists"); 22 | t.end(); 23 | 24 | }); 25 | 26 | t.test("#deleteFileSync", (t) => { 27 | 28 | t.doesNotThrow(() => IO.deleteFileSync("./files/trails2.mbtiles"), "Deletes an existing file"); 29 | t.throws(() => IO.deleteFileSync("./files/trails2.mbtiles"), "File does not exists"); 30 | t.end(); 31 | 32 | }); 33 | 34 | t.end(); 35 | 36 | }); 37 | -------------------------------------------------------------------------------- /test/unit/MapboxStyle.test.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const test = require("tap").test; 5 | const MapboxStyle = require("../../src/core/MapboxStyle"); 6 | 7 | test("MapboxStyle", (t) => { 8 | 9 | t.test("#open", (t) => { 10 | 11 | const style = new MapboxStyle("./files/style.json"); 12 | const style2 = new MapboxStyle("./files/style2.json"); 13 | 14 | t.resolves(() => style.open(), "Style open"); 15 | t.rejects(() => style2.open(), "Style does not exist"); 16 | t.end(); 17 | 18 | }); 19 | 20 | t.test("#groupLayerBySource", (t) => { 21 | 22 | const style = new MapboxStyle(); 23 | const layers = [{id: 1, source: "a"}, { id: 2, source: "b"}, {id:3}, {id:4, source:"b"}, {id:5, source: "a"}]; 24 | 25 | t.deepEquals(style.groupLayerBySource(layers), {"a": [1, 5], "b": [2, 4]}, "Layers are grouped by source"); 26 | t.end(); 27 | 28 | }); 29 | 30 | t.test("#mapLayersToObject", (t) => { 31 | 32 | const style = new MapboxStyle(); 33 | const layers = [{id: 1, "source-layer": "a"}, { id: 2, "source-layer": "b"}]; 34 | 35 | t.deepEquals(style.mapLayersToObject(layers), {"a": [{data : { id: 1, "source-layer": "a"}}], "b": [{ data: {id: 2, "source-layer": "b"}}]}, "Layers are mapped to objects"); 36 | t.end(); 37 | 38 | }); 39 | 40 | t.test("#getLayerNamesFromSource", async (t) => { 41 | 42 | const style = new MapboxStyle("./files/style.json"); 43 | await style.open(); 44 | 45 | t.deepEquals(style.getLayerNamesFromSource("mapbox-streets"), ["water"], "Got source's layers"); 46 | t.end(); 47 | 48 | }); 49 | 50 | t.test("#isLayerVisibleOnLevel", async (t) => { 51 | 52 | const style = new MapboxStyle("./files/style.json"); 53 | await style.open(); 54 | 55 | t.equals(style.isLayerVisibleOnLevel("water", 5), true, "Layer visible"); 56 | t.equals(style.isLayerVisibleOnLevel("water", 30), false, "Layer not visible"); 57 | t.end(); 58 | 59 | }); 60 | 61 | t.test("#isLayerVisibleOnLevel", async (t) => { 62 | 63 | const style = new MapboxStyle("./files/styleMultipleLayers.json"); 64 | await style.open(); 65 | 66 | t.equals(style.isLayerVisibleOnLevel("munich", 0), true, "Layer visible"); 67 | t.equals(style.isLayerVisibleOnLevel("munich", 1), true, "Layer visible"); 68 | t.end(); 69 | 70 | }); 71 | 72 | t.end(); 73 | 74 | }); 75 | -------------------------------------------------------------------------------- /test/unit/MapboxStyleLayer.test.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const test = require("tap").test; 5 | const MapboxStyleLayer = require("../../src/core/MapboxStyleLayer"); 6 | const PaintPropertiesToCheck = require("../../src/PaintPropertiesToCheck"); 7 | 8 | test("MapboxStyleLayer", (t) => { 9 | 10 | t.test("#isVisibleOnZoomLevel", (t) => { 11 | 12 | const layer = new MapboxStyleLayer({ 13 | "id": "water", 14 | "source": "mapbox-streets", 15 | "source-layer": "water", 16 | "type": "fill", 17 | "layout": { 18 | "visibility": "visible" 19 | }, 20 | "paint": { 21 | "fill-color": "#00ffff" 22 | }, 23 | "minzoom": 10, 24 | "maxzoom": 13 25 | }); 26 | 27 | t.equals(layer.isVisibleOnZoomLevel(9), false, "Layer not visible on level 9"); 28 | t.equals(layer.isVisibleOnZoomLevel(11), true, "Layer visible on level 11"); 29 | t.equals(layer.isVisibleOnZoomLevel(14), false, "Layer not visible on level 14"); 30 | t.end(); 31 | 32 | }); 33 | 34 | t.test("#checkLayoutVisibility", (t) => { 35 | 36 | const visibleLayer = new MapboxStyleLayer({}); 37 | const explicitlyVisibleLayer = new MapboxStyleLayer({layout: { visibility: "visible" }}); 38 | const explicitlyNonVisibleLayer = new MapboxStyleLayer({layout: { visibility: "none" }}); 39 | 40 | t.equals(visibleLayer.checkLayoutVisibility(), true, "Layer does not have explicit visibility"); 41 | t.equals(explicitlyVisibleLayer.checkLayoutVisibility(), true, "Layer does have explicit visibility"); 42 | t.equals(explicitlyNonVisibleLayer.checkLayoutVisibility(), false, "Layer does have explicit invisibility"); 43 | t.end(); 44 | 45 | }); 46 | 47 | t.test("#checkZoomUnderflow", (t) => { 48 | 49 | const visibleLayer = new MapboxStyleLayer({}); 50 | const minZoomLayer = new MapboxStyleLayer({minzoom: 4}); 51 | 52 | t.equals(visibleLayer.checkZoomUnderflow(5), true, "Min zoom default value"); 53 | t.equals(minZoomLayer.checkZoomUnderflow(5), true, "Min zoom explicitly set and lower than the one we are testing"); 54 | t.equals(minZoomLayer.checkZoomUnderflow(3), false, "Min zoom explicitly set and higher than the one we are testing"); 55 | t.end(); 56 | 57 | }); 58 | 59 | t.test("#checkZoomOverflow", (t) => { 60 | 61 | const visibleLayer = new MapboxStyleLayer({}); 62 | const maxZoomLayer = new MapboxStyleLayer({maxzoom: 14}); 63 | 64 | t.equals(visibleLayer.checkZoomOverflow(5), true, "Max zoom default value"); 65 | t.equals(maxZoomLayer.checkZoomOverflow(5), true, "Max zoom explicitly set and higher than the one we are testing"); 66 | t.equals(maxZoomLayer.checkZoomOverflow(16), false, "Max zoom explicitly set and lower than the one we are testing"); 67 | t.equals(maxZoomLayer.checkZoomOverflow(14), false, "Max zoom explicitly set and equal to the one we are testing"); 68 | t.end(); 69 | 70 | }); 71 | 72 | t.test("#isRendered", (t) => { 73 | 74 | for(let index in PaintPropertiesToCheck) { 75 | 76 | const layerDef = { paint: {}}; 77 | const property = PaintPropertiesToCheck[index] 78 | layerDef.paint[property] = 1; 79 | const visibleLayer = new MapboxStyleLayer(layerDef); 80 | layerDef.paint[property] = 0; 81 | const nonVisibleLayer = new MapboxStyleLayer(layerDef); 82 | t.equals(visibleLayer.isRendered(), true, `Layer with paint property ${property} different than 0. ${JSON.stringify(visibleLayer)}`); 83 | t.equals(nonVisibleLayer.isRendered(), false, `Layer with paint property ${property} equal to 0. ${JSON.stringify(nonVisibleLayer)}`); 84 | 85 | } 86 | 87 | t.end(); 88 | 89 | }); 90 | 91 | t.test("#checkPaintPropertyNotZero", (t) => { 92 | 93 | const invisibleLayer = new MapboxStyleLayer({ paint: { "fill-opacity": 0 }}); 94 | const invisibleLayerFloatValue = new MapboxStyleLayer({ paint: { "fill-opacity": 0.0 }}); 95 | const visibleLayer = new MapboxStyleLayer({ paint: { "fill-opacity": 0.5 }}); 96 | t.equals(invisibleLayer.checkPaintPropertyNotZero("fill-opacity"), false, `Layer with fill-opacity equal to 0. ${JSON.stringify(visibleLayer)}`); 97 | t.equals(invisibleLayerFloatValue.checkPaintPropertyNotZero("fill-opacity"), false, `Layer with fill-opacity equal to 0.0. ${JSON.stringify(visibleLayer)}`); 98 | t.equals(visibleLayer.checkPaintPropertyNotZero("heatmap-opacity"), true, `Layer with no heatmap-opacity. ${JSON.stringify(visibleLayer)}`); 99 | t.equals(visibleLayer.checkPaintPropertyNotZero("fill-opacity"), true, `Layer with fill-opacity non 0. ${JSON.stringify(visibleLayer)}`); 100 | 101 | t.end(); 102 | 103 | }); 104 | 105 | t.test("#areAllPropertiesFilteredOut", (t) => { 106 | 107 | const invisibleLayer = new MapboxStyleLayer({ paint: { "fill-opacity": 0 }}); 108 | t.equals(invisibleLayer.areAllPropertiesFilteredOut(), false); 109 | t.end(); 110 | 111 | }); 112 | 113 | t.end(); 114 | 115 | }); 116 | -------------------------------------------------------------------------------- /test/unit/Utils.test.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | "use strict"; 3 | 4 | const test = require("tap").test; 5 | const Utils = require("../../src/core/Utils"); 6 | 7 | test("Utils", (t) => { 8 | 9 | function almostEqual(a, b) { 10 | 11 | return Math.abs(a - b) < 0.00001; 12 | 13 | } 14 | 15 | t.test("#toRadians - toDegrees", (t) => { 16 | 17 | const coordX = 90; 18 | const coordY = 40.25; 19 | t.ok(almostEqual(Utils.toRadians(Utils.toDegrees(coordX)), coordX)); 20 | t.ok(almostEqual(Utils.toRadians(Utils.toDegrees(coordY)), coordY)); 21 | t.end(); 22 | 23 | }); 24 | 25 | t.test("#tile2Lon - lon2Tile", (t) => { 26 | 27 | const zoomLevel = 7; 28 | const tileX = 79; 29 | t.ok(almostEqual(Utils.lon2TileX(Utils.tileX2Lon(tileX, zoomLevel), zoomLevel), tileX)); 30 | t.end(); 31 | 32 | }); 33 | 34 | t.test("#tile2Lat - lat2Tile", (t) => { 35 | 36 | const zoomLevel = 7; 37 | const tileY = 64; 38 | t.ok(almostEqual(Utils.lat2TileY(Utils.tileY2Lat(tileY, zoomLevel), zoomLevel), tileY)); 39 | t.end(); 40 | 41 | }); 42 | 43 | t.test("#lonLat2Tile - tile2LonLat", (t) => { 44 | 45 | const lon = 0; 46 | const lat = 43.068887; 47 | const zoomLevel = 7; 48 | const {row, column} = Utils.lonLat2Tile(zoomLevel, lon, lat); 49 | const lonlat = Utils.tile2LonLat(zoomLevel, column, row); 50 | t.ok(almostEqual(lon, lonlat.lon)); 51 | t.ok(almostEqual(lat, lonlat.lat)); 52 | t.end(); 53 | 54 | }); 55 | 56 | t.test("#world2NormalizedX - normalized2WorldX", (t) => { 57 | 58 | const coordX = 42.123456; 59 | t.ok(almostEqual(Utils.normalized2WorldX(Utils.world2NormalizedX(coordX)), coordX)); 60 | t.end(); 61 | 62 | }); 63 | 64 | t.test("#world2NormalizedY - normalized2WorldY", (t) => { 65 | 66 | const coordY = 1.123456; 67 | t.ok(almostEqual(Utils.normalized2WorldY(Utils.world2NormalizedY(coordY)), coordY)); 68 | t.end(); 69 | 70 | }); 71 | 72 | t.test("#world2Normalized - normalized2World", (t) => { 73 | 74 | const lon = 1.123456; 75 | const lat = 42.123456; 76 | const {x, y} = Utils.world2Normalized(lon, lat); 77 | const lonlat = Utils.normalized2World(x, y); 78 | t.ok(almostEqual(lon, lonlat.lon)); 79 | t.ok(almostEqual(lat, lonlat.lat)); 80 | t.end(); 81 | 82 | }); 83 | 84 | t.test("#world2VTX - vT2WorldX", (t) => { 85 | 86 | const zoom = 7; 87 | const row = 79; 88 | const extent = 4096; 89 | const coordX = 42.123456; 90 | t.ok(almostEqual(Utils.vT2WorldX(zoom, row, extent, Utils.worldX2VT(zoom, row, extent, coordX)), coordX)); 91 | t.end(); 92 | 93 | }); 94 | 95 | t.test("#world2VTY - vT2WorldY", (t) => { 96 | 97 | const zoom = 7; 98 | const extent = 4096; 99 | const lon = 1.123456; 100 | const column = Utils.lon2TileX(lon, zoom); 101 | t.ok(almostEqual(Utils.vT2WorldY(zoom, column, extent, Utils.worldY2VT(zoom, column, extent, lon)), lon)); 102 | t.end(); 103 | 104 | }); 105 | 106 | t.test("#loadFromURL", (t) => { 107 | 108 | t.resolves(Utils.loadFromURL("https://geoserveis.icgc.cat/data/planet/1/1/1.pbf")); 109 | t.end(); 110 | 111 | }); 112 | 113 | t.end(); 114 | 115 | }); 116 | --------------------------------------------------------------------------------