├── .gitignore ├── .vscode └── tasks.json ├── LICENSE ├── README.md ├── geotiff ├── decode.js ├── decoder.js ├── geotiff-js-lib.js ├── index.js ├── mmap.js ├── shared.js └── yield-execution-every.js ├── package-lock.json ├── package.json └── webpack.config.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Auto-generated 2 | src/lzw-wasm.js 3 | 4 | # compiled output 5 | /dist 6 | 7 | 8 | # IDEs and editors 9 | .idea 10 | .project 11 | .classpath 12 | .c9/ 13 | *.launch 14 | .settings/ 15 | *.sublime-workspace 16 | 17 | # IDE - VSCode 18 | .vscode/* 19 | !.vscode/settings.json 20 | !.vscode/tasks.json 21 | !.vscode/launch.json 22 | !.vscode/extensions.json 23 | 24 | # misc 25 | .sass-cache 26 | connect.lock 27 | typings 28 | 29 | # Logs 30 | logs 31 | *.log 32 | npm-debug.log* 33 | yarn-debug.log* 34 | yarn-error.log* 35 | 36 | 37 | # Dependency directories 38 | node_modules/ 39 | jspm_packages/ 40 | 41 | # Optional npm cache directory 42 | .npm 43 | 44 | # Optional eslint cache 45 | .eslintcache 46 | 47 | # Optional REPL history 48 | .node_repl_history 49 | 50 | # Output of 'npm pack' 51 | *.tgz 52 | 53 | # Yarn Integrity file 54 | .yarn-integrity 55 | 56 | # dotenv environment variables file 57 | .env 58 | 59 | # next.js build output 60 | .next 61 | 62 | # Lerna 63 | lerna-debug.log 64 | 65 | # System Files 66 | .DS_Store 67 | Thumbs.db -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "type": "npm", 8 | "script": "build", 9 | "problemMatcher": [ 10 | "$eslint-stylish" 11 | ] 12 | }, 13 | { 14 | "type": "npm", 15 | "script": "watch", 16 | "problemMatcher": [ 17 | "$eslint-stylish" 18 | ] 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Ceres Imaging 4 | Copyright (c) 2015 EOX IT Services GmbH 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Description 2 | FastGeoTIFF is a layer over GeoTIFF.js (https://geotiffjs.github.io/) 3 | optimized for high performance raster reads of commonly formatted GeoTIFFs. 4 | For many common non-tiled, planar, RAW or LZW compressed GeoTIFFs, FastGeoTIFF 5 | can decode an ImageData of the rasters 5-10x faster than GeoTIFF.js. 6 | 7 | E.g. a 150MB LZW compressed GeoTIFF can be decoded to ImageData in ~1s. 8 | 9 | If FastGeoTIFF can't read the file, it will fall back to the (bundled) copy 10 | of GeoTIFF.js. 11 | 12 | Uncompressed GeoTIFFs are 'read' using a direct mmap. LZW compressed GeoTIFFs 13 | are decoded using 'fast-lzw' (https://github.com/ceresimaging/fast-lzw), which uses web assembly for performance. 14 | 15 | ```javascript 16 | import { readRasterFromURL } from 'fast-geotiff' 17 | 18 | // Load the ImageData from a URL (an arraybuffer version also exists) 19 | const imageData = await readRasterFromURL('http://server.com/some-image.tiff') 20 | 21 | // Draw it to an 22 | const canvas = new OffscreenCanvas(imageData.width, imageData.height) 23 | canvas.getContext("2d").putImageData(imageData, 0, 0) 24 | const dataURL = canvas.getDataURL() 25 | const img = document.createElement("img") 26 | img.setAttribute("src", canvas.getDataURL()) 27 | document.appendChild(img) 28 | ``` 29 | -------------------------------------------------------------------------------- /geotiff/decode.js: -------------------------------------------------------------------------------- 1 | import { arrayTypeFor, getDecoder, uint16ToUint8, range } from './shared' 2 | import zip from 'pop-zip/zip' 3 | import flatten from 'array-flatten' 4 | 5 | const partition = (arr, numPartitions) => { 6 | const len = arr.length 7 | numPartitions = Math.max(1, Math.min(arr.length, numPartitions)) 8 | const partitionSize = Math.floor(len / numPartitions) 9 | return range(numPartitions).map((n) => { 10 | const start = n * partitionSize 11 | const end = start + partitionSize 12 | return arr.slice(start, end) 13 | }) 14 | } 15 | 16 | export default async function readGeoTIFF (img, arrayBuffer) { 17 | const format = img.fileDirectory.SampleFormat ? Math.max(...img.fileDirectory.SampleFormat) : 1 18 | const maxBitsPerSample = Math.max(...img.fileDirectory.BitsPerSample) 19 | 20 | if (img.isTiled || !img.planarConfiguration || img.getTileHeight() != 1 || format != 1) { 21 | throw Error("Can't handle this type of GeoTIFF, try readGeoTiff, or geotiff.js instead") 22 | } 23 | 24 | const width = img.getWidth() 25 | const height = img.getHeight() 26 | const numSamples = img.fileDirectory.SamplesPerPixel 27 | 28 | const ArrayType = arrayTypeFor(format, maxBitsPerSample) 29 | 30 | const decoder = getDecoder(img.fileDirectory) 31 | 32 | let rawArray = new Uint8Array(arrayBuffer) 33 | if (decoder.usingWorkers) { 34 | if (window.SharedArrayBuffer && !(arrayBuffer instanceof window.SharedArrayBuffer)) { 35 | arrayBuffer = new window.SharedArrayBuffer(rawArray.byteLength) 36 | const copyArray = new Uint8Array(arrayBuffer, 0, rawArray.byteLength) 37 | copyArray.set(rawArray) 38 | rawArray = copyArray 39 | } 40 | } 41 | 42 | // OK, this is messed up, but if we don't do this, we don't get 43 | // complete strips, and LZW decode fails (???) 44 | // You can see the image corruption decrease as we read more and 45 | // more data. This is especially pronounced in the 'transparent' 46 | // parts of images, which should compress more, which makes me 47 | // wonder if we're misunderstanding how to read the strips? 48 | const MAGIC_FACTOR_READ_N_EXTRA_BYTES_FOR_EACH_STRIP = 16384 49 | 50 | const strips = zip( 51 | img.fileDirectory.StripOffsets, 52 | img.fileDirectory.StripByteCounts 53 | ).map(([ byteOffset, numBytes ]) => 54 | rawArray.subarray( 55 | byteOffset, 56 | byteOffset + MAGIC_FACTOR_READ_N_EXTRA_BYTES_FOR_EACH_STRIP + numBytes 57 | ) 58 | ) 59 | 60 | const numPartitions = decoder.numWorkers ? decoder.numWorkers() : 1 61 | let rgba = new ArrayType(width * height * numSamples) 62 | 63 | const decoderResults = flatten( 64 | await Promise.all( 65 | partition(strips, numPartitions) 66 | .map(rawStripArrays => decoder.decodeAll 67 | ? decoder.decodeAll(img.fileDirectory, rawStripArrays) 68 | : Promise.all( 69 | rawStripArrays.map((rawStripArray) => decoder.decode(img.fileDirectory, rawStripArray)) 70 | ) 71 | ) 72 | ) 73 | ) 74 | 75 | decoderResults 76 | .map(stripBytes => new ArrayType(stripBytes.buffer, stripBytes.byteOffset, stripBytes.length / ArrayType.BYTES_PER_ELEMENT)) 77 | .reduce((rgbaIndex, strip) => { 78 | rgba.set(strip, rgbaIndex) 79 | return rgbaIndex + strip.length 80 | }, 0) 81 | 82 | if (ArrayType == Uint16Array) { 83 | rgba = uint16ToUint8(rgba) 84 | } 85 | 86 | return new ImageData(rgba, width, height) 87 | } -------------------------------------------------------------------------------- /geotiff/decoder.js: -------------------------------------------------------------------------------- 1 | import { LZW } from 'fast-lzw' 2 | import { getDecoder as getDecoderGeotiffJS } from 'geotiff/dist/compression' 3 | import BaseDecoder from 'geotiff/dist/compression/basedecoder' 4 | import YieldExecutionEvery from './yield-execution-every' 5 | 6 | const WORKER_POOL_SIZE = 4 7 | const YIELD_EXECUTION_EVERY_MS = 100 8 | 9 | let lzw = null 10 | 11 | class FastLLZWDecoder extends BaseDecoder { 12 | constructor (...args) { 13 | super(args) 14 | lzw = lzw || new LZW(WORKER_POOL_SIZE) 15 | this.usingWorkers = lzw.usingWorkers 16 | } 17 | async decodeAll(fileDirectory, buffers) { 18 | return await lzw.decompress(buffers) 19 | } 20 | async decodeBlock(buffer) { 21 | return await lzw.decompress([buffer]) 22 | } 23 | numWorkers() { 24 | return WORKER_POOL_SIZE 25 | } 26 | } 27 | 28 | class YieldingDecoder extends BaseDecoder { 29 | constructor(baseDecoder, ...args) { 30 | super(args) 31 | this.decoder = baseDecoder 32 | this.yielder = new YieldExecutionEvery(YIELD_EXECUTION_EVERY_MS) 33 | } 34 | async decodeBlock(buffer) { 35 | await this.yielder.maybeYield() 36 | return this.decoder.decodeBlock(buffer) 37 | } 38 | } 39 | 40 | const isLZW = fileDirectory => fileDirectory.Compression == 5 41 | const getDecoder = (fileDirectory) => isLZW(fileDirectory) 42 | ? new FastLLZWDecoder() 43 | : new YieldingDecoder( 44 | getDecoderGeotiffJS(fileDirectory) 45 | ) 46 | 47 | export { 48 | getDecoder, 49 | } 50 | -------------------------------------------------------------------------------- /geotiff/geotiff-js-lib.js: -------------------------------------------------------------------------------- 1 | import { uint16ToUint8 } from './shared' 2 | import { fromArrayBuffer } from 'geotiff/dist/geotiff.bundle' 3 | 4 | async function arrayBufferToGeoTiffJSImage (arrayBuffer) { 5 | const tiff = await fromArrayBuffer(arrayBuffer) 6 | return await tiff.getImage() 7 | } 8 | 9 | async function readGeoTIFF(img, arrayBuffer) { 10 | const rasters = await img.readRasters({ interleave: true }) 11 | const rgba = uint16ToUint8(rasters) 12 | return new ImageData(rgba, img.getWidth(), img.getHeight()) 13 | } 14 | 15 | export { arrayBufferToGeoTiffJSImage, readGeoTIFF } 16 | -------------------------------------------------------------------------------- /geotiff/index.js: -------------------------------------------------------------------------------- 1 | import readGeoTIFFDecode from './decode' 2 | import readGeoTiffMMAP from './mmap' 3 | import { readGeoTIFF as readGeoTIFFJsLib, arrayBufferToGeoTiffJSImage } from './geotiff-js-lib' 4 | 5 | const PROFILE=true 6 | 7 | function decode(img, arrayBuffer) { 8 | const format = img.fileDirectory.SampleFormat ? Math.max(...img.fileDirectory.SampleFormat) : 1 9 | 10 | if (img.planarConfiguration && !img.isTiled && img.getTileHeight() == 1 && format == 1) { 11 | if (img.fileDirectory.Compression == 1) { 12 | // RAW, not compressed, use MMAP for insane speed 13 | return readGeoTiffMMAP(img, arrayBuffer) 14 | } else { 15 | // Its all in a direct read order, but compressed 16 | return readGeoTIFFDecode(img, arrayBuffer) 17 | } 18 | } else { 19 | // Its something else, let GeoTiff.js deal with it 20 | return readGeoTIFFJsLib(img) 21 | } 22 | } 23 | 24 | async function readRasterFromURL(url) { 25 | time(`readRaster:${url}`) 26 | const response = await fetch(url) 27 | const arrayBuffer = await response.arrayBuffer() 28 | const imageData = readRaster(arrayBuffer) 29 | timeEnd(`readRaster:${url}`) 30 | return imageData 31 | } 32 | 33 | function time(s) { 34 | if (PROFILE) 35 | console.time(s) 36 | } 37 | 38 | function timeEnd(s) { 39 | if (PROFILE) 40 | console.timeEnd(s) 41 | } 42 | 43 | async function readRaster(arrayBuffer) { 44 | const img = await arrayBufferToGeoTiffJSImage(arrayBuffer) 45 | const imageData = await decode(img, arrayBuffer) 46 | return imageData 47 | } 48 | 49 | export { readRaster, readRasterFromURL } -------------------------------------------------------------------------------- /geotiff/mmap.js: -------------------------------------------------------------------------------- 1 | import {uint16ToUint8} from './shared' 2 | 3 | export default async function readGeoTiff (img, arrayBuffer) { 4 | const format = img.fileDirectory.SampleFormat ? Math.max(...img.fileDirectory.SampleFormat) : 1 5 | const maxBitsPerSample = Math.max(...img.fileDirectory.BitsPerSample) 6 | 7 | if (img.isTiled || !img.planarConfiguration || img.fileDirectory.Compression != 1 || img.getTileHeight() != 1 || format != 1 || maxBitsPerSample != 16) { 8 | // CANNOT DECODE, only for uncompressed, planar GeoTIFFS with strips 9 | throw Error("Can't handle this type of GeoTIFF, try readGeoTiff, or geotiff.js instead") 10 | } 11 | 12 | const stripOffsets = img.fileDirectory.StripOffsets 13 | const uint16Array = new Uint16Array(arrayBuffer, 14 | stripOffsets[0], 15 | // FIXME: weird, the file is actually LONGER than the byte counts here suggest 16 | // and if you don't read the whole thing... it doesn't work.... huh 17 | // stripOffsets[stripOffsets.length - 1] + img.fileDirectory.StripByteCounts[stripOffsets.length - 1] 18 | ) 19 | 20 | const rgba = uint16ToUint8(uint16Array) 21 | 22 | return new ImageData(rgba, img.getWidth(), img.getHeight()) 23 | } -------------------------------------------------------------------------------- /geotiff/shared.js: -------------------------------------------------------------------------------- 1 | import {getDecoder} from './decoder' 2 | 3 | const formats = { 4 | // unsigned integer data 5 | 1: { 6 | 8: Uint8Array, 7 | 16: Uint16Array, 8 | 32: Uint32Array 9 | }, 10 | // twos complement signed integer data 11 | 2: { 12 | 8: Int8Array, 13 | 16: Int16Array, 14 | 32: Int32Array 15 | }, 16 | // floating point data 17 | 3: { 18 | 32: Float32Array, 19 | 64: Float64Array 20 | } 21 | } 22 | 23 | const arrayTypeFor = (format, bitsPerSample) => formats[format][bitsPerSample] 24 | const arrayForType = (format, bitsPerSample, size) => new arrayTypeFor(format, bitsPerSample)(size) 25 | 26 | const range = n => Array(n).fill().map((_, i) => i) 27 | 28 | function uint16ToUint8(uint16) { 29 | const rgba = new Uint8ClampedArray(uint16.length) 30 | 31 | for (let i=0; i < rgba.length; i++) { 32 | const val = uint16[i] >> 8 33 | const isAlphaChannel = (i+1) % 4 == 0 34 | const notBlack = val != 0 35 | 36 | rgba[i] = isAlphaChannel && notBlack ? 255 : val 37 | } 38 | 39 | return rgba 40 | } 41 | 42 | export { arrayForType, arrayTypeFor, range, getDecoder, uint16ToUint8 } -------------------------------------------------------------------------------- /geotiff/yield-execution-every.js: -------------------------------------------------------------------------------- 1 | 2 | export default class YieldExecutionEvery { 3 | constructor(yieldEveryMS) { 4 | this.yieldEveryMS = yieldEveryMS 5 | this.lastYieldTime = performance.now() 6 | } 7 | async maybeYield() { 8 | const timeSinceYield = performance.now() - this.lastYieldTime 9 | if (timeSinceYield > this.yieldEveryMS) { 10 | await new Promise(resolve => setTimeout(resolve, 0)) 11 | this.lastYieldTime = performance.now() 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fast-geotiff", 3 | "version": "1.0.0-beta.4", 4 | "description": "Optimized raster reads for GeoTIFF.js, 5-10x faster on many common GeoTIFFs", 5 | "main": "dist/index.js", 6 | "scripts": { 7 | "watch": "webpack --watch", 8 | "build": "webpack" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git+https://github.com/ceresimaging/fast-geotiff.git" 13 | }, 14 | "keywords": [ 15 | "fast", 16 | "geotiff", 17 | "geotiff.js", 18 | "performance", 19 | "WASM" 20 | ], 21 | "author": "Seth Nickell ", 22 | "license": "MIT", 23 | "bugs": { 24 | "url": "https://github.com/ceresimaging/fast-geotiff/issues" 25 | }, 26 | "homepage": "https://github.com/ceresimaging/fast-geotiff#readme", 27 | "dependencies": { 28 | "array-flatten": "^2.1.2", 29 | "fast-lzw": "^1.0.0-beta.3", 30 | "pop-zip": "^1.0.0" 31 | }, 32 | "peerDependencies": { 33 | "geotiff": "1.0.0-beta.6" 34 | }, 35 | "devDependencies": { 36 | "@babel/core": "^7.5.5", 37 | "@babel/preset-env": "^7.5.5", 38 | "@babel/runtime": "^7.5.5", 39 | "babel-loader": "^8.0.6", 40 | "geotiff": "^1.0.0-beta.6", 41 | "webpack": "^4.39.0", 42 | "webpack-cli": "^3.3.6" 43 | }, 44 | "files": [ 45 | "dist/" 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | 3 | module.exports = { 4 | entry: './geotiff/index.js', 5 | output: { 6 | path: path.resolve(__dirname, 'dist'), 7 | filename: 'index.js', 8 | libraryTarget: "umd", 9 | umdNamedDefine: true, 10 | library: 'fast-geotiff' 11 | }, 12 | mode: "production", 13 | module: { 14 | rules: [ 15 | { 16 | test: /\.js$/, 17 | include: path.resolve(__dirname, 'src'), 18 | exclude: /(node_modules|dist)/, 19 | use: { 20 | loader: 'babel-loader', 21 | options: { 22 | presets: ['@babel/preset-env'], 23 | } 24 | } 25 | } 26 | ] 27 | }, 28 | externals: [ 29 | /^geotiff\//i, 30 | ], 31 | 32 | } --------------------------------------------------------------------------------