├── .jshintrc ├── LICENSE ├── README.md ├── dist ├── lz4-block-codec-any.js ├── lz4-block-codec-js.js ├── lz4-block-codec-wasm.js └── lz4-block-codec.wasm ├── src ├── README.md └── lz4-block-codec.wat └── test ├── index.html └── index.js /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "browser": true, 3 | "devel": true, 4 | "eqeqeq": true, 5 | "esnext": true, 6 | "globals": { 7 | "self": false 8 | }, 9 | "newcap": false, 10 | "nonew": false, 11 | "strict": "global", 12 | "sub": true, 13 | "undef": true, 14 | "unused": true, 15 | "validthis": true 16 | } 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2018, Raymond Hill 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lz4-wasm 2 | 3 | LZ4 block codec: A WebAssembly implementation. 4 | 5 | The current implementation encode/decode LZ4 block format as per [official documentation](https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md). 6 | 7 | LZ4 frame format is not implemented at this time. 8 | 9 | # Files 10 | 11 | `./src/lz4-block-codec.wat`: the WebAssembly source code 12 | 13 | `./dist/lz4-block-codec.wasm`: the compiled WebAssembly source code, generated using: 14 | 15 | wat2wasm ./lz4-block-codec.wat -o ./dist/lz4-block-codec.wasm 16 | wasm-opt ./lz4-block-codec.wasm -O4 -o ./dist/lz4-block-codec.wasm 17 | 18 | You can get `wat2wasm` at , and `wasm-opt` at . 19 | 20 | # Test 21 | 22 | Note: the test/benchmark page uses javascript implementation of other compression libraries solely for the sake of comparison with javascript-based solutions. 23 | 24 | [Test & benchmark page](https://gorhill.github.io/lz4-wasm/test/index.html). 25 | -------------------------------------------------------------------------------- /dist/lz4-block-codec-any.js: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | 3 | lz4-block-codec-any.js 4 | A wrapper to instanciate a wasm- and/or js-based LZ4 block 5 | encoder/decoder. 6 | Copyright (C) 2018 Raymond Hill 7 | 8 | BSD-2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 9 | 10 | Redistribution and use in source and binary forms, with or without 11 | modification, are permitted provided that the following conditions are 12 | met: 13 | 14 | 1. Redistributions of source code must retain the above copyright 15 | notice, this list of conditions and the following disclaimer. 16 | 17 | 2. Redistributions in binary form must reproduce the above 18 | copyright notice, this list of conditions and the following disclaimer 19 | in the documentation and/or other materials provided with the 20 | distribution. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | 34 | Home: https://github.com/gorhill/lz4-wasm 35 | 36 | I used the same license as the one picked by creator of LZ4 out of respect 37 | for his creation, see https://lz4.github.io/lz4/ 38 | 39 | */ 40 | 41 | 'use strict'; 42 | 43 | /******************************************************************************/ 44 | 45 | (function(context) { // >>>> Start of private namespace 46 | 47 | /******************************************************************************/ 48 | 49 | let wd = (function() { 50 | let url = document.currentScript.src; 51 | let match = /[^\/]+$/.exec(url); 52 | return match !== null ? 53 | url.slice(0, match.index) : 54 | ''; 55 | })(); 56 | 57 | let removeScript = function(script) { 58 | if ( !script ) { return; } 59 | if ( script.parentNode === null ) { return; } 60 | script.parentNode.removeChild(script); 61 | }; 62 | 63 | let createInstanceWASM = function() { 64 | if ( context.LZ4BlockWASM instanceof Function ) { 65 | let instance = new context.LZ4BlockWASM(); 66 | return instance.init().then(( ) => { return instance; }); 67 | } 68 | if ( context.LZ4BlockWASM === null ) { 69 | return Promise.resolve(null); 70 | } 71 | return new Promise((resolve, reject) => { 72 | let script = document.createElement('script'); 73 | script.src = wd + 'lz4-block-codec-wasm.js'; 74 | script.addEventListener('load', ( ) => { 75 | if ( context.LZ4BlockWASM instanceof Function === false ) { 76 | context.LZ4BlockWASM = null; 77 | context.LZ4BlockWASM = undefined; 78 | resolve(null); 79 | } else { 80 | let instance = new context.LZ4BlockWASM(); 81 | instance.init() 82 | .then(( ) => { 83 | resolve(instance); 84 | }) 85 | .catch(error => { 86 | reject(error); 87 | }); 88 | } 89 | }); 90 | script.addEventListener('error', ( ) => { 91 | context.LZ4BlockWASM = null; 92 | resolve(null); 93 | }); 94 | document.head.appendChild(script); 95 | removeScript(script); 96 | }); 97 | }; 98 | 99 | let createInstanceJS = function() { 100 | if ( context.LZ4BlockJS instanceof Function ) { 101 | let instance = new context.LZ4BlockJS(); 102 | return instance.init().then(( ) => { return instance; }); 103 | } 104 | if ( context.LZ4BlockJS === null ) { 105 | return Promise.resolve(null); 106 | } 107 | return new Promise((resolve, reject) => { 108 | let script = document.createElement('script'); 109 | script.src = wd + 'lz4-block-codec-js.js'; 110 | script.addEventListener('load', ( ) => { 111 | if ( context.LZ4BlockJS instanceof Function === false ) { 112 | context.LZ4BlockJS = null; 113 | resolve(null); 114 | } else { 115 | let instance = new context.LZ4BlockJS(); 116 | instance.init() 117 | .then(( ) => { 118 | resolve(instance); 119 | }) 120 | .catch(error => { 121 | reject(error); 122 | }); 123 | } 124 | }); 125 | script.addEventListener('error', ( ) => { 126 | context.LZ4BlockJS = null; 127 | resolve(null); 128 | }); 129 | document.head.appendChild(script); 130 | removeScript(script); 131 | }); 132 | }; 133 | 134 | /******************************************************************************/ 135 | 136 | context.lz4BlockCodec = { 137 | createInstance: function(flavor) { 138 | let instantiator; 139 | if ( flavor === 'wasm' ) { 140 | instantiator = createInstanceWASM; 141 | } else if ( flavor === 'js' ) { 142 | instantiator = createInstanceJS; 143 | } else { 144 | instantiator = createInstanceWASM || createInstanceJS; 145 | } 146 | return (instantiator)() 147 | .then(instance => { 148 | if ( instance ) { return instance; } 149 | if ( flavor === undefined ) { 150 | return createInstanceJS(); 151 | } 152 | return null; 153 | }) 154 | .catch(( ) => { 155 | if ( flavor === undefined ) { 156 | return createInstanceJS(); 157 | } 158 | return null; 159 | }); 160 | }, 161 | reset: function() { 162 | context.LZ4BlockWASM = undefined; 163 | context.LZ4BlockJS = undefined; 164 | } 165 | }; 166 | 167 | /******************************************************************************/ 168 | 169 | })(this || self); // <<<< End of private namespace 170 | 171 | /******************************************************************************/ 172 | -------------------------------------------------------------------------------- /dist/lz4-block-codec-js.js: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | 3 | lz4-block-codec-js.js 4 | A javascript wrapper around a pure javascript implementation of 5 | LZ4 block format codec. 6 | Copyright (C) 2018 Raymond Hill 7 | 8 | BSD-2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 9 | 10 | Redistribution and use in source and binary forms, with or without 11 | modification, are permitted provided that the following conditions are 12 | met: 13 | 14 | 1. Redistributions of source code must retain the above copyright 15 | notice, this list of conditions and the following disclaimer. 16 | 17 | 2. Redistributions in binary form must reproduce the above 18 | copyright notice, this list of conditions and the following disclaimer 19 | in the documentation and/or other materials provided with the 20 | distribution. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | 34 | Home: https://github.com/gorhill/lz4-wasm 35 | 36 | I used the same license as the one picked by creator of LZ4 out of respect 37 | for his creation, see https://lz4.github.io/lz4/ 38 | 39 | */ 40 | 41 | 'use strict'; 42 | 43 | /******************************************************************************/ 44 | 45 | (function(context) { // >>>> Start of private namespace 46 | 47 | /******************************************************************************/ 48 | 49 | let growOutputBuffer = function(instance, size) { 50 | if ( 51 | instance.outputBuffer === undefined || 52 | instance.outputBuffer.byteLength < size 53 | ) { 54 | instance.outputBuffer = new ArrayBuffer(size + 0xFFFF & 0x7FFF0000); 55 | } 56 | return instance.outputBuffer; 57 | }; 58 | 59 | let encodeBound = function(size) { 60 | return size > 0x7E000000 ? 61 | 0 : 62 | size + (size / 255 | 0) + 16; 63 | }; 64 | 65 | let encodeBlock = function(instance, iBuf, oOffset) { 66 | let iLen = iBuf.byteLength; 67 | if ( iLen >= 0x7E000000 ) { throw new RangeError(); } 68 | 69 | // "The last match must start at least 12 bytes before end of block" 70 | let lastMatchPos = iLen - 12; 71 | 72 | // "The last 5 bytes are always literals" 73 | let lastLiteralPos = iLen - 5; 74 | 75 | if ( instance.hashTable === undefined ) { 76 | instance.hashTable = new Int32Array(65536); 77 | } 78 | instance.hashTable.fill(-65536); 79 | 80 | if ( iBuf instanceof ArrayBuffer ) { 81 | iBuf = new Uint8Array(iBuf); 82 | } 83 | 84 | let oLen = oOffset + encodeBound(iLen); 85 | let oBuf = new Uint8Array(growOutputBuffer(instance, oLen), 0, oLen); 86 | let iPos = 0; 87 | let oPos = oOffset; 88 | let anchorPos = 0; 89 | 90 | // sequence-finding loop 91 | for (;;) { 92 | let refPos; 93 | let mOffset; 94 | let sequence = iBuf[iPos] << 8 | iBuf[iPos+1] << 16 | iBuf[iPos+2] << 24; 95 | 96 | // match-finding loop 97 | while ( iPos <= lastMatchPos ) { 98 | sequence = sequence >>> 8 | iBuf[iPos+3] << 24; 99 | let hash = (sequence * 0x9E37 & 0xFFFF) + (sequence * 0x79B1 >>> 16) & 0xFFFF; 100 | refPos = instance.hashTable[hash]; 101 | instance.hashTable[hash] = iPos; 102 | mOffset = iPos - refPos; 103 | if ( 104 | mOffset < 65536 && 105 | iBuf[refPos+0] === ((sequence ) & 0xFF) && 106 | iBuf[refPos+1] === ((sequence >>> 8) & 0xFF) && 107 | iBuf[refPos+2] === ((sequence >>> 16) & 0xFF) && 108 | iBuf[refPos+3] === ((sequence >>> 24) & 0xFF) 109 | ) { 110 | break; 111 | } 112 | iPos += 1; 113 | } 114 | 115 | // no match found 116 | if ( iPos > lastMatchPos ) { break; } 117 | 118 | // match found 119 | let lLen = iPos - anchorPos; 120 | let mLen = iPos; 121 | iPos += 4; refPos += 4; 122 | while ( iPos < lastLiteralPos && iBuf[iPos] === iBuf[refPos] ) { 123 | iPos += 1; refPos += 1; 124 | } 125 | mLen = iPos - mLen; 126 | let token = mLen < 19 ? mLen - 4 : 15; 127 | 128 | // write token, length of literals if needed 129 | if ( lLen >= 15 ) { 130 | oBuf[oPos++] = 0xF0 | token; 131 | let l = lLen - 15; 132 | while ( l >= 255 ) { 133 | oBuf[oPos++] = 255; 134 | l -= 255; 135 | } 136 | oBuf[oPos++] = l; 137 | } else { 138 | oBuf[oPos++] = (lLen << 4) | token; 139 | } 140 | 141 | // write literals 142 | while ( lLen-- ) { 143 | oBuf[oPos++] = iBuf[anchorPos++]; 144 | } 145 | 146 | if ( mLen === 0 ) { break; } 147 | 148 | // write offset of match 149 | oBuf[oPos+0] = mOffset; 150 | oBuf[oPos+1] = mOffset >>> 8; 151 | oPos += 2; 152 | 153 | // write length of match if needed 154 | if ( mLen >= 19 ) { 155 | let l = mLen - 19; 156 | while ( l >= 255 ) { 157 | oBuf[oPos++] = 255; 158 | l -= 255; 159 | } 160 | oBuf[oPos++] = l; 161 | } 162 | 163 | anchorPos = iPos; 164 | } 165 | 166 | // last sequence is literals only 167 | let lLen = iLen - anchorPos; 168 | if ( lLen >= 15 ) { 169 | oBuf[oPos++] = 0xF0; 170 | let l = lLen - 15; 171 | while ( l >= 255 ) { 172 | oBuf[oPos++] = 255; 173 | l -= 255; 174 | } 175 | oBuf[oPos++] = l; 176 | } else { 177 | oBuf[oPos++] = lLen << 4; 178 | } 179 | while ( lLen-- ) { 180 | oBuf[oPos++] = iBuf[anchorPos++]; 181 | } 182 | 183 | return new Uint8Array(oBuf.buffer, 0, oPos); 184 | }; 185 | 186 | let decodeBlock = function(instance, iBuf, iOffset, oLen) { 187 | let iLen = iBuf.byteLength; 188 | let oBuf = new Uint8Array(growOutputBuffer(instance, oLen), 0, oLen); 189 | let iPos = iOffset, oPos = 0; 190 | 191 | while ( iPos < iLen ) { 192 | let token = iBuf[iPos++]; 193 | 194 | // literals 195 | let clen = token >>> 4; 196 | 197 | // length of literals 198 | if ( clen !== 0 ) { 199 | if ( clen === 15 ) { 200 | let l; 201 | for (;;) { 202 | l = iBuf[iPos++]; 203 | if ( l !== 255 ) { break; } 204 | clen += 255; 205 | } 206 | clen += l; 207 | } 208 | 209 | // copy literals 210 | let end = iPos + clen; 211 | while ( iPos < end ) { 212 | oBuf[oPos++] = iBuf[iPos++]; 213 | } 214 | if ( iPos === iLen ) { break; } 215 | } 216 | 217 | // match 218 | let mOffset = iBuf[iPos+0] | (iBuf[iPos+1] << 8); 219 | if ( mOffset === 0 || mOffset > oPos ) { return; } 220 | iPos += 2; 221 | 222 | // length of match 223 | clen = (token & 0x0F) + 4; 224 | if ( clen === 19 ) { 225 | let l; 226 | for (;;) { 227 | l = iBuf[iPos++]; 228 | if ( l !== 255 ) { break; } 229 | clen += 255; 230 | } 231 | clen += l; 232 | } 233 | 234 | // copy match 235 | let mPos = oPos - mOffset; 236 | let end = oPos + clen; 237 | while ( oPos < end ) { 238 | oBuf[oPos++] = oBuf[mPos++]; 239 | } 240 | } 241 | 242 | return oBuf; 243 | }; 244 | 245 | /******************************************************************************/ 246 | 247 | context.LZ4BlockJS = function() { 248 | this.hashTable = undefined; 249 | this.outputBuffer = undefined; 250 | }; 251 | 252 | context.LZ4BlockJS.prototype = { 253 | flavor: 'js', 254 | init: function() { 255 | return Promise.resolve(); 256 | }, 257 | 258 | reset: function() { 259 | this.hashTable = undefined; 260 | this.outputBuffer = undefined; 261 | }, 262 | 263 | bytesInUse: function() { 264 | let bytesInUse = 0; 265 | if ( this.hashTable !== undefined ) { 266 | bytesInUse += this.hashTable.byteLength; 267 | } 268 | if ( this.outputBuffer !== undefined ) { 269 | bytesInUse += this.outputBuffer.byteLength; 270 | } 271 | return bytesInUse; 272 | }, 273 | 274 | encodeBlock: function(input, outputOffset) { 275 | if ( input instanceof ArrayBuffer ) { 276 | input = new Uint8Array(input); 277 | } else if ( input instanceof Uint8Array === false ) { 278 | throw new TypeError(); 279 | } 280 | return encodeBlock(this, input, outputOffset); 281 | }, 282 | 283 | decodeBlock: function(input, inputOffset, outputSize) { 284 | if ( input instanceof ArrayBuffer ) { 285 | input = new Uint8Array(input); 286 | } else if ( input instanceof Uint8Array === false ) { 287 | throw new TypeError(); 288 | } 289 | return decodeBlock(this, input, inputOffset, outputSize); 290 | } 291 | }; 292 | 293 | /******************************************************************************/ 294 | 295 | })(this || self); // <<<< End of private namespace 296 | 297 | /******************************************************************************/ 298 | -------------------------------------------------------------------------------- /dist/lz4-block-codec-wasm.js: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | 3 | lz4-block-codec-wasm.js 4 | A javascript wrapper around a WebAssembly implementation of 5 | LZ4 block format codec. 6 | Copyright (C) 2018 Raymond Hill 7 | 8 | BSD-2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 9 | 10 | Redistribution and use in source and binary forms, with or without 11 | modification, are permitted provided that the following conditions are 12 | met: 13 | 14 | 1. Redistributions of source code must retain the above copyright 15 | notice, this list of conditions and the following disclaimer. 16 | 17 | 2. Redistributions in binary form must reproduce the above 18 | copyright notice, this list of conditions and the following disclaimer 19 | in the documentation and/or other materials provided with the 20 | distribution. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | 34 | Home: https://github.com/gorhill/lz4-wasm 35 | 36 | I used the same license as the one picked by creator of LZ4 out of respect 37 | for his creation, see https://lz4.github.io/lz4/ 38 | 39 | */ 40 | 41 | /* global WebAssembly */ 42 | 43 | 'use strict'; 44 | 45 | /******************************************************************************/ 46 | 47 | (function(context) { // >>>> Start of private namespace 48 | 49 | /******************************************************************************/ 50 | 51 | let wd = (function() { 52 | let url = document.currentScript.src; 53 | let match = /[^\/]+$/.exec(url); 54 | return match !== null ? 55 | url.slice(0, match.index) : 56 | ''; 57 | })(); 58 | 59 | let growMemoryTo = function(wasmInstance, byteLength) { 60 | let lz4api = wasmInstance.exports; 61 | let neededByteLength = lz4api.getLinearMemoryOffset() + byteLength; 62 | let pageCountBefore = lz4api.memory.buffer.byteLength >>> 16; 63 | let pageCountAfter = (neededByteLength + 65535) >>> 16; 64 | if ( pageCountAfter > pageCountBefore ) { 65 | lz4api.memory.grow(pageCountAfter - pageCountBefore); 66 | } 67 | return lz4api.memory.buffer; 68 | }; 69 | 70 | let encodeBlock = function(wasmInstance, inputArray, outputOffset) { 71 | let lz4api = wasmInstance.exports; 72 | let mem0 = lz4api.getLinearMemoryOffset(); 73 | let hashTableSize = 65536 * 4; 74 | let inputSize = inputArray.byteLength; 75 | if ( inputSize >= 0x7E000000 ) { throw new RangeError(); } 76 | let memSize = 77 | hashTableSize + 78 | inputSize + 79 | outputOffset + lz4api.lz4BlockEncodeBound(inputSize); 80 | let memBuffer = growMemoryTo(wasmInstance, memSize); 81 | let hashTable = new Int32Array(memBuffer, mem0, 65536); 82 | hashTable.fill(-65536, 0, 65536); 83 | let inputMem = new Uint8Array(memBuffer, mem0 + hashTableSize, inputSize); 84 | inputMem.set(inputArray); 85 | let outputSize = lz4api.lz4BlockEncode( 86 | mem0 + hashTableSize, 87 | inputSize, 88 | mem0 + hashTableSize + inputSize + outputOffset 89 | ); 90 | if ( outputSize === 0 ) { return; } 91 | let outputArray = new Uint8Array( 92 | memBuffer, 93 | mem0 + hashTableSize + inputSize, 94 | outputOffset + outputSize 95 | ); 96 | return outputArray; 97 | }; 98 | 99 | let decodeBlock = function(wasmInstance, inputArray, inputOffset, outputSize) { 100 | let inputSize = inputArray.byteLength; 101 | let lz4api = wasmInstance.exports; 102 | let mem0 = lz4api.getLinearMemoryOffset(); 103 | let memSize = inputSize + outputSize; 104 | let memBuffer = growMemoryTo(wasmInstance, memSize); 105 | let inputArea = new Uint8Array(memBuffer, mem0, inputSize); 106 | inputArea.set(inputArray); 107 | outputSize = lz4api.lz4BlockDecode( 108 | mem0 + inputOffset, 109 | inputSize - inputOffset, 110 | mem0 + inputSize 111 | ); 112 | if ( outputSize === 0 ) { return; } 113 | return new Uint8Array(memBuffer, mem0 + inputSize, outputSize); 114 | }; 115 | 116 | /******************************************************************************/ 117 | 118 | context.LZ4BlockWASM = function() { 119 | this.lz4wasmInstance = undefined; 120 | }; 121 | 122 | context.LZ4BlockWASM.prototype = { 123 | flavor: 'wasm', 124 | 125 | init: function() { 126 | if ( 127 | typeof WebAssembly !== 'object' || 128 | typeof WebAssembly.instantiateStreaming !== 'function' 129 | ) { 130 | this.lz4wasmInstance = null; 131 | } 132 | if ( this.lz4wasmInstance === null ) { 133 | return Promise.reject(); 134 | } 135 | if ( this.lz4wasmInstance instanceof WebAssembly.Instance ) { 136 | return Promise.resolve(this.lz4wasmInstance); 137 | } 138 | if ( this.lz4wasmInstance === undefined ) { 139 | this.lz4wasmInstance = WebAssembly.instantiateStreaming( 140 | fetch(wd + 'lz4-block-codec.wasm', { mode: 'same-origin' }) 141 | ).then(result => { 142 | this.lz4wasmInstance = undefined; 143 | this.lz4wasmInstance = result && result.instance || null; 144 | if ( this.lz4wasmInstance !== null ) { return this; } 145 | return null; 146 | }); 147 | this.lz4wasmInstance.catch(( ) => { 148 | this.lz4wasmInstance = null; 149 | return null; 150 | }); 151 | } 152 | return this.lz4wasmInstance; 153 | }, 154 | 155 | reset: function() { 156 | this.lz4wasmInstance = undefined; 157 | }, 158 | 159 | bytesInUse: function() { 160 | return this.lz4wasmInstance instanceof WebAssembly.Instance ? 161 | this.lz4wasmInstance.exports.memory.buffer.byteLength : 162 | 0; 163 | }, 164 | 165 | encodeBlock: function(input, outputOffset) { 166 | if ( this.lz4wasmInstance instanceof WebAssembly.Instance === false ) { 167 | throw new Error('LZ4BlockWASM: not initialized'); 168 | } 169 | if ( input instanceof ArrayBuffer ) { 170 | input = new Uint8Array(input); 171 | } else if ( input instanceof Uint8Array === false ) { 172 | throw new TypeError(); 173 | } 174 | return encodeBlock(this.lz4wasmInstance, input, outputOffset); 175 | }, 176 | 177 | decodeBlock: function(input, inputOffset, outputSize) { 178 | if ( this.lz4wasmInstance instanceof WebAssembly.Instance === false ) { 179 | throw new Error('LZ4BlockWASM: not initialized'); 180 | } 181 | if ( input instanceof ArrayBuffer ) { 182 | input = new Uint8Array(input); 183 | } else if ( input instanceof Uint8Array === false ) { 184 | throw new TypeError(); 185 | } 186 | return decodeBlock(this.lz4wasmInstance, input, inputOffset, outputSize); 187 | } 188 | }; 189 | 190 | /******************************************************************************/ 191 | 192 | })(this || self); // <<<< End of private namespace 193 | 194 | /******************************************************************************/ 195 | -------------------------------------------------------------------------------- /dist/lz4-block-codec.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorhill/lz4-wasm/8995cdef7b3adc4a4d2fc4fc86b84a86789ff858/dist/lz4-block-codec.wasm -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- 1 | To create the binary file: 2 | 3 | wat2wasm -o dist/lz4-block-codec.wasm src/lz4-block-codec.wat 4 | wasm-opt dist/lz4-block-codec.wasm -O4 -o dist/lz4-block-codec.wasm 5 | 6 | 7 | `wat2wasm`: available at . 8 | 9 | `wasm-opt`: available at . 10 | -------------------------------------------------------------------------------- /src/lz4-block-codec.wat: -------------------------------------------------------------------------------- 1 | ;; 2 | ;; lz4-block-codec.wat: a WebAssembly implementation of LZ4 block format codec 3 | ;; Copyright (C) 2018 Raymond Hill 4 | ;; 5 | ;; BSD-2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | ;; 7 | ;; Redistribution and use in source and binary forms, with or without 8 | ;; modification, are permitted provided that the following conditions are 9 | ;; met: 10 | ;; 11 | ;; 1. Redistributions of source code must retain the above copyright 12 | ;; notice, this list of conditions and the following disclaimer. 13 | ;; 14 | ;; 2. Redistributions in binary form must reproduce the above 15 | ;; copyright notice, this list of conditions and the following disclaimer 16 | ;; in the documentation and/or other materials provided with the 17 | ;; distribution. 18 | ;; 19 | ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | ;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | ;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | ;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | ;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | ;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | ;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | ;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | ;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | ;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | ;; 31 | ;; Home: https://github.com/gorhill/lz4-wasm 32 | ;; 33 | ;; I used the same license as the one picked by creator of LZ4 out of respect 34 | ;; for his creation, see https://lz4.github.io/lz4/ 35 | ;; 36 | 37 | (module 38 | ;; 39 | ;; module start 40 | ;; 41 | 42 | ;; (func $log (import "imports" "log") (param i32 i32 i32)) 43 | 44 | (memory (export "memory") 1) 45 | 46 | ;; 47 | ;; Public functions 48 | ;; 49 | 50 | ;; 51 | ;; Return an offset to the first byte of usable linear memory. 52 | ;; Might be useful in the future to reserve memory space for whatever purpose, 53 | ;; like config variables, etc. 54 | ;; 55 | (func $getLinearMemoryOffset (export "getLinearMemoryOffset") 56 | (result i32) 57 | i32.const 0 58 | ) 59 | 60 | ;; 61 | ;; unsigned int lz4BlockEncodeBound() 62 | ;; 63 | ;; Return the maximum size of the output buffer holding the compressed data. 64 | ;; 65 | ;; Reference implementation: 66 | ;; https://github.com/lz4/lz4/blob/dev/lib/lz4.h#L156 67 | ;; 68 | (func (export "lz4BlockEncodeBound") 69 | (param $ilen i32) 70 | (result i32) 71 | get_local $ilen 72 | i32.const 0x7E000000 73 | i32.gt_u 74 | if 75 | i32.const 0 76 | return 77 | end 78 | get_local $ilen 79 | get_local $ilen 80 | i32.const 255 81 | i32.div_u 82 | i32.add 83 | i32.const 16 84 | i32.add 85 | ) 86 | 87 | ;; 88 | ;; unsigned int lz4BlockEncode( 89 | ;; unsigned int inPtr, 90 | ;; unsigned int ilen, 91 | ;; unsigned int outPtr 92 | ;; ) 93 | ;; 94 | ;; https://github.com/lz4/lz4/blob/dev/lib/lz4.c#L651 95 | ;; 96 | ;; The implementation below is modified from the reference one. 97 | ;; 98 | ;; - There is no skip adjustement for repeated failure to find a match. 99 | ;; 100 | ;; - All configurable values are hard-coded to match the generic version 101 | ;; of the compressor. 102 | ;; 103 | ;; Note the size of the input block is NOT encoded in the output buffer, it 104 | ;; is for the caller to figure how they will save that information on 105 | ;; their side. At this point it is probably a trivial amount of work to 106 | ;; implement the LZ4 frame format, which encode the content size, but this 107 | ;; is for another day. 108 | ;; 109 | (func $lz4BlockEncode (export "lz4BlockEncode") 110 | (param $inPtr i32) ;; pointer to start of input buffer 111 | (param $ilen i32) ;; size of input buffer 112 | (param $outPtr i32) ;; pointer to start of output buffer 113 | (result i32) 114 | (local $hashPtrBeg i32) ;; start of hash buffer 115 | (local $hashPtr i32) ;; current hash entry 116 | (local $anchorPtr i32) ;; anchor position in input 117 | (local $inPtrEnd1 i32) ;; point in input at which match-finding must cease 118 | (local $inPtrEnd2 i32) ;; point in input at which match-length finding must cease 119 | (local $inPtrEnd i32) ;; point to end of input 120 | (local $outPtrBeg i32) ;; start of output buffer 121 | (local $refPtr i32) ;; start of match in input 122 | (local $seq32 i32) ;; 4-byte value from current input position 123 | (local $llen i32) ;; length of found literals 124 | (local $moffset i32) ;; offset to found match from current input position 125 | (local $mlen i32) ;; length of found match 126 | get_local $ilen ;; empty input = empty output 127 | i32.const 0x7E000000 ;; max input size: 0x7E000000 128 | i32.gt_u 129 | if 130 | i32.const 0 131 | return 132 | end 133 | get_local $ilen ;; "blocks < 13 bytes cannot be compressed" 134 | i32.const 13 135 | i32.lt_u 136 | if 137 | i32.const 0 138 | return 139 | end 140 | call $getLinearMemoryOffset ;; hash table is at start of usable memory 141 | set_local $hashPtrBeg 142 | get_local $inPtr 143 | tee_local $anchorPtr 144 | get_local $ilen 145 | i32.add 146 | tee_local $inPtrEnd 147 | i32.const -5 ;; "The last 5 bytes are always literals." 148 | i32.add 149 | tee_local $inPtrEnd2 150 | i32.const -7 ;; "The last match must start at least 12 bytes before end of block" 151 | i32.add 152 | set_local $inPtrEnd1 153 | get_local $outPtr 154 | set_local $outPtrBeg 155 | ;; 156 | ;; sequence processing loop 157 | ;; 158 | block $noMoreSequence loop $nextSequence 159 | get_local $inPtr 160 | get_local $inPtrEnd1 161 | i32.ge_u ;; 5 or less bytes left? 162 | br_if $noMoreSequence 163 | get_local $inPtr ;; first sequence of 3 bytes before match-finding loop 164 | i32.load8_u 165 | i32.const 8 166 | i32.shl 167 | get_local $inPtr 168 | i32.load8_u offset=1 169 | i32.const 16 170 | i32.shl 171 | i32.or 172 | get_local $inPtr 173 | i32.load8_u offset=2 174 | i32.const 24 175 | i32.shl 176 | i32.or 177 | set_local $seq32 178 | ;; 179 | ;; match-finding loop 180 | ;; 181 | loop $findMatch block $noMatchFound 182 | get_local $inPtr 183 | get_local $inPtrEnd2 184 | i32.gt_u ;; less than 12 bytes left? 185 | br_if $noMoreSequence 186 | get_local $seq32 ;; update last byte of current sequence 187 | i32.const 8 188 | i32.shr_u 189 | get_local $inPtr 190 | i32.load8_u offset=3 191 | i32.const 24 192 | i32.shl 193 | i32.or 194 | tee_local $seq32 195 | i32.const 0x9E3779B1 ;; compute 16-bit hash 196 | i32.mul 197 | i32.const 16 198 | i32.shr_u ;; hash value is at top of stack 199 | i32.const 2 ;; lookup refPtr at hash entry 200 | i32.shl 201 | get_local $hashPtrBeg 202 | i32.add 203 | tee_local $hashPtr 204 | i32.load 205 | set_local $refPtr 206 | get_local $hashPtr ;; update hash entry with inPtr 207 | get_local $inPtr 208 | i32.store 209 | get_local $inPtr 210 | get_local $refPtr 211 | i32.sub 212 | tee_local $moffset ;; remember match offset, we will need it in case of match 213 | i32.const 0xFFFF 214 | i32.gt_s ;; match offset > 65535 = unusable match 215 | br_if $noMatchFound 216 | ;; 217 | ;; confirm match: different sequences can yield same hash 218 | ;; compare-branch each byte to potentially save memory read ops 219 | ;; 220 | get_local $seq32 ;; byte 0 221 | i32.const 0xFF 222 | i32.and 223 | get_local $refPtr 224 | i32.load8_u 225 | i32.ne ;; refPtr[0] !== inPtr[0] 226 | br_if $noMatchFound 227 | get_local $seq32 ;; byte 1 228 | i32.const 8 229 | i32.shr_u 230 | i32.const 0xFF 231 | i32.and 232 | get_local $refPtr 233 | i32.load8_u offset=1 234 | i32.ne 235 | br_if $noMatchFound ;; refPtr[1] !== inPtr[1] 236 | get_local $seq32 ;; byte 2 237 | i32.const 16 238 | i32.shr_u 239 | i32.const 0xFF 240 | i32.and 241 | get_local $refPtr 242 | i32.load8_u offset=2 243 | i32.ne ;; refPtr[2] !== inPtr[2] 244 | br_if $noMatchFound 245 | get_local $seq32 ;; byte 3 246 | i32.const 24 247 | i32.shr_u 248 | i32.const 0xFF 249 | i32.and 250 | get_local $refPtr 251 | i32.load8_u offset=3 252 | i32.ne ;; refPtr[3] !== inPtr[3] 253 | br_if $noMatchFound 254 | ;; 255 | ;; a valid match has been found at this point 256 | ;; 257 | get_local $inPtr ;; compute length of literals 258 | get_local $anchorPtr 259 | i32.sub 260 | set_local $llen 261 | get_local $inPtr ;; find match length 262 | i32.const 4 ;; skip over confirmed 4-byte match 263 | i32.add 264 | set_local $inPtr 265 | get_local $refPtr 266 | i32.const 4 267 | i32.add 268 | tee_local $mlen ;; remember refPtr to later compute match length 269 | set_local $refPtr 270 | block $endOfMatch loop ;; scan input buffer until match ends 271 | get_local $inPtr 272 | get_local $inPtrEnd2 273 | i32.ge_u 274 | br_if $endOfMatch 275 | get_local $inPtr 276 | i32.load8_u 277 | get_local $refPtr 278 | i32.load8_u 279 | i32.ne 280 | br_if $endOfMatch 281 | get_local $inPtr 282 | i32.const 1 283 | i32.add 284 | set_local $inPtr 285 | get_local $refPtr 286 | i32.const 1 287 | i32.add 288 | set_local $refPtr 289 | br 0 290 | end end $endOfMatch 291 | ;; encode token 292 | get_local $outPtr ;; output token 293 | get_local $llen 294 | get_local $refPtr 295 | get_local $mlen 296 | i32.sub 297 | tee_local $mlen 298 | call $writeToken 299 | get_local $outPtr 300 | i32.const 1 301 | i32.add 302 | set_local $outPtr 303 | get_local $llen ;; encode/write length of literals if needed 304 | i32.const 15 305 | i32.ge_s 306 | if 307 | get_local $outPtr 308 | get_local $llen 309 | call $writeLength 310 | set_local $outPtr 311 | end 312 | ;; copy literals 313 | get_local $outPtr 314 | get_local $anchorPtr 315 | get_local $llen 316 | call $copy 317 | get_local $outPtr 318 | get_local $llen 319 | i32.add 320 | set_local $outPtr 321 | ;; encode match offset 322 | get_local $outPtr 323 | get_local $moffset 324 | i32.store8 325 | get_local $outPtr 326 | get_local $moffset 327 | i32.const 8 328 | i32.shr_u 329 | i32.store8 offset=1 330 | get_local $outPtr 331 | i32.const 2 332 | i32.add 333 | set_local $outPtr 334 | get_local $mlen ;; encode/write length of match if needed 335 | i32.const 15 336 | i32.ge_s 337 | if 338 | get_local $outPtr 339 | get_local $mlen 340 | call $writeLength 341 | set_local $outPtr 342 | end 343 | get_local $inPtr ;; advance anchor to current position 344 | set_local $anchorPtr 345 | br $nextSequence 346 | end $noMatchFound 347 | get_local $inPtr ;; no match found: advance to next byte 348 | i32.const 1 349 | i32.add 350 | set_local $inPtr 351 | br $findMatch end ;; match offset > 65535 = unusable match 352 | end end $noMoreSequence 353 | ;; 354 | ;; generate last (match-less) sequence if compression succeeded 355 | ;; 356 | get_local $outPtr 357 | get_local $outPtrBeg 358 | i32.eq 359 | if 360 | i32.const 0 361 | return 362 | end 363 | get_local $outPtr 364 | get_local $inPtrEnd 365 | get_local $anchorPtr 366 | i32.sub 367 | tee_local $llen 368 | i32.const 0 369 | call $writeToken 370 | get_local $outPtr 371 | i32.const 1 372 | i32.add 373 | set_local $outPtr 374 | get_local $llen 375 | i32.const 15 376 | i32.ge_u 377 | if 378 | get_local $outPtr 379 | get_local $llen 380 | call $writeLength 381 | set_local $outPtr 382 | end 383 | get_local $outPtr 384 | get_local $anchorPtr 385 | get_local $llen 386 | call $copy 387 | get_local $outPtr ;; return number of written bytes 388 | get_local $llen 389 | i32.add 390 | get_local $outPtrBeg 391 | i32.sub 392 | ) 393 | 394 | ;; 395 | ;; unsigned int lz4BlockDecode( 396 | ;; unsigned int inPtr, 397 | ;; unsigned int ilen 398 | ;; unsigned int outPtr 399 | ;; ) 400 | ;; 401 | ;; Reference: 402 | ;; https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md 403 | ;; 404 | (func (export "lz4BlockDecode") 405 | (param $inPtr0 i32) ;; start of input buffer 406 | (param $ilen i32) ;; length of input buffer 407 | (param $outPtr0 i32) ;; start of output buffer 408 | (result i32) 409 | (local $inPtr i32) ;; current position in input buffer 410 | (local $inPtrEnd i32) ;; end of input buffer 411 | (local $outPtr i32) ;; current position in output buffer 412 | (local $matchPtr i32) ;; position of current match 413 | (local $token i32) ;; sequence token 414 | (local $clen i32) ;; number of bytes to copy 415 | (local $_ i32) ;; general purpose variable 416 | get_local $ilen ;; if ( ilen == 0 ) { return 0; } 417 | i32.eqz 418 | if 419 | i32.const 0 420 | return 421 | end 422 | get_local $inPtr0 423 | tee_local $inPtr ;; current position in input buffer 424 | get_local $ilen 425 | i32.add 426 | set_local $inPtrEnd 427 | get_local $outPtr0 ;; start of output buffer 428 | set_local $outPtr ;; current position in output buffer 429 | block $noMoreSequence loop ;; iterate through all sequences 430 | get_local $inPtr 431 | get_local $inPtrEnd 432 | i32.ge_u 433 | br_if $noMoreSequence ;; break when nothing left to read in input buffer 434 | get_local $inPtr ;; read token -- consume one byte 435 | i32.load8_u 436 | get_local $inPtr 437 | i32.const 1 438 | i32.add 439 | set_local $inPtr 440 | tee_local $token ;; extract length of literals from token 441 | i32.const 4 442 | i32.shr_u 443 | tee_local $clen ;; consume extra length bytes if present 444 | i32.eqz 445 | if else 446 | get_local $clen 447 | i32.const 15 448 | i32.eq 449 | if loop 450 | get_local $inPtr 451 | i32.load8_u 452 | get_local $inPtr 453 | i32.const 1 454 | i32.add 455 | set_local $inPtr 456 | tee_local $_ 457 | get_local $clen 458 | i32.add 459 | set_local $clen 460 | get_local $_ 461 | i32.const 255 462 | i32.eq 463 | br_if 0 464 | end end 465 | get_local $outPtr ;; copy literals to ouput buffer 466 | get_local $inPtr 467 | get_local $clen 468 | call $copy 469 | get_local $outPtr ;; advance output buffer pointer past copy 470 | get_local $clen 471 | i32.add 472 | set_local $outPtr 473 | get_local $clen ;; advance input buffer pointer past literals 474 | get_local $inPtr 475 | i32.add 476 | tee_local $inPtr 477 | get_local $inPtrEnd ;; exit if this is the last sequence 478 | i32.eq 479 | br_if $noMoreSequence 480 | end 481 | get_local $outPtr ;; read match offset 482 | get_local $inPtr 483 | i32.load8_u 484 | get_local $inPtr 485 | i32.load8_u offset=1 486 | i32.const 8 487 | i32.shl 488 | i32.or 489 | i32.sub 490 | tee_local $matchPtr 491 | get_local $outPtr ;; match position can't be outside input buffer bounds 492 | i32.eq 493 | br_if $noMoreSequence 494 | get_local $matchPtr 495 | get_local $inPtrEnd 496 | i32.lt_u 497 | br_if $noMoreSequence 498 | get_local $inPtr ;; advance input pointer past match offset bytes 499 | i32.const 2 500 | i32.add 501 | set_local $inPtr 502 | get_local $token ;; extract length of match from token 503 | i32.const 15 504 | i32.and 505 | i32.const 4 506 | i32.add 507 | tee_local $clen 508 | i32.const 19 ;; consume extra length bytes if present 509 | i32.eq 510 | if loop 511 | get_local $inPtr 512 | i32.load8_u 513 | get_local $inPtr 514 | i32.const 1 515 | i32.add 516 | set_local $inPtr 517 | tee_local $_ 518 | get_local $clen 519 | i32.add 520 | set_local $clen 521 | get_local $_ 522 | i32.const 255 523 | i32.eq 524 | br_if 0 525 | end end 526 | get_local $outPtr ;; copy match to ouput buffer 527 | get_local $matchPtr 528 | get_local $clen 529 | call $copy 530 | get_local $clen ;; advance output buffer pointer past copy 531 | get_local $outPtr 532 | i32.add 533 | set_local $outPtr 534 | br 0 535 | end end $noMoreSequence 536 | get_local $outPtr ;; return number of written bytes 537 | get_local $outPtr0 538 | i32.sub 539 | ) 540 | 541 | ;; 542 | ;; Private functions 543 | ;; 544 | 545 | ;; 546 | ;; Encode a sequence token 547 | ;; 548 | ;; Reference documentation: 549 | ;; https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md 550 | ;; 551 | (func $writeToken 552 | (param $outPtr i32) 553 | (param $llen i32) 554 | (param $mlen i32) 555 | get_local $outPtr 556 | get_local $llen 557 | i32.const 15 558 | get_local $llen 559 | i32.const 15 560 | i32.lt_u 561 | select 562 | i32.const 4 563 | i32.shl 564 | get_local $mlen 565 | i32.const 15 566 | get_local $mlen 567 | i32.const 15 568 | i32.lt_u 569 | select 570 | i32.or 571 | i32.store8 572 | ) 573 | 574 | ;; 575 | ;; Encode and output length bytes. The return value is the pointer following 576 | ;; the last byte written. 577 | ;; 578 | ;; Reference documentation: 579 | ;; https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md 580 | ;; 581 | (func $writeLength 582 | (param $outPtr i32) 583 | (param $len i32) 584 | (result i32) 585 | get_local $len 586 | i32.const 15 587 | i32.sub 588 | set_local $len 589 | loop 590 | get_local $outPtr 591 | get_local $len 592 | i32.const 255 593 | get_local $len 594 | i32.const 255 595 | i32.lt_u 596 | select 597 | i32.store8 598 | get_local $outPtr 599 | i32.const 1 600 | i32.add 601 | set_local $outPtr 602 | get_local $len 603 | i32.const 255 604 | i32.sub 605 | tee_local $len 606 | i32.const 0 607 | i32.ge_s 608 | br_if 0 609 | end 610 | get_local $outPtr 611 | ) 612 | 613 | ;; 614 | ;; Copy n bytes from source to destination. 615 | ;; 616 | ;; It is overlap-safe only from left-to-right -- which is only what is 617 | ;; required in the current module. 618 | ;; 619 | (func $copy 620 | (param $dst i32) 621 | (param $src i32) 622 | (param $len i32) 623 | block $lessThan8 loop 624 | get_local $len 625 | i32.const 8 626 | i32.lt_u 627 | br_if $lessThan8 628 | get_local $dst 629 | get_local $src 630 | i32.load8_u 631 | i32.store8 632 | get_local $dst 633 | get_local $src 634 | i32.load8_u offset=1 635 | i32.store8 offset=1 636 | get_local $dst 637 | get_local $src 638 | i32.load8_u offset=2 639 | i32.store8 offset=2 640 | get_local $dst 641 | get_local $src 642 | i32.load8_u offset=3 643 | i32.store8 offset=3 644 | get_local $dst 645 | get_local $src 646 | i32.load8_u offset=4 647 | i32.store8 offset=4 648 | get_local $dst 649 | get_local $src 650 | i32.load8_u offset=5 651 | i32.store8 offset=5 652 | get_local $dst 653 | get_local $src 654 | i32.load8_u offset=6 655 | i32.store8 offset=6 656 | get_local $dst 657 | get_local $src 658 | i32.load8_u offset=7 659 | i32.store8 offset=7 660 | get_local $dst 661 | i32.const 8 662 | i32.add 663 | set_local $dst 664 | get_local $src 665 | i32.const 8 666 | i32.add 667 | set_local $src 668 | get_local $len 669 | i32.const -8 670 | i32.add 671 | set_local $len 672 | br 0 673 | end end $lessThan8 674 | get_local $len 675 | i32.const 4 676 | i32.ge_u 677 | if 678 | get_local $dst 679 | get_local $src 680 | i32.load8_u 681 | i32.store8 682 | get_local $dst 683 | get_local $src 684 | i32.load8_u offset=1 685 | i32.store8 offset=1 686 | get_local $dst 687 | get_local $src 688 | i32.load8_u offset=2 689 | i32.store8 offset=2 690 | get_local $dst 691 | get_local $src 692 | i32.load8_u offset=3 693 | i32.store8 offset=3 694 | get_local $dst 695 | i32.const 4 696 | i32.add 697 | set_local $dst 698 | get_local $src 699 | i32.const 4 700 | i32.add 701 | set_local $src 702 | get_local $len 703 | i32.const -4 704 | i32.add 705 | set_local $len 706 | end 707 | get_local $len 708 | i32.const 2 709 | i32.ge_u 710 | if 711 | get_local $dst 712 | get_local $src 713 | i32.load8_u 714 | i32.store8 715 | get_local $dst 716 | get_local $src 717 | i32.load8_u offset=1 718 | i32.store8 offset=1 719 | get_local $dst 720 | i32.const 2 721 | i32.add 722 | set_local $dst 723 | get_local $src 724 | i32.const 2 725 | i32.add 726 | set_local $src 727 | get_local $len 728 | i32.const -2 729 | i32.add 730 | set_local $len 731 | end 732 | get_local $len 733 | i32.eqz 734 | if else 735 | get_local $dst 736 | get_local $src 737 | i32.load8_u 738 | i32.store8 739 | end 740 | ) 741 | 742 | ;; 743 | ;; module end 744 | ;; 745 | ) 746 | -------------------------------------------------------------------------------- /test/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Benchmark: node-lz4, snappyjs, lz4-wasm 7 | 8 | 9 |

Benchmark: 10 | node-lz4, 11 | snappyjs, 12 | lz4-wasm 13 |

14 |

15 |
16 |

17 |
18 |
19 |
20 |
21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | /* globals Benchmark, SnappyJS, lz4BlockCodec, require */ 2 | 3 | 'use strict'; 4 | 5 | /******************************************************************************/ 6 | 7 | // Helpers for pierrec/node-lz4 8 | 9 | let Buffer = require('buffer').Buffer; 10 | let LZ4 = require('lz4'); 11 | 12 | function lz4Encode(input) { 13 | let inputBuffer = new Buffer(input); 14 | let outputBuffer = new Buffer(LZ4.encodeBound(inputBuffer.length)); 15 | let lz4OutputSize = LZ4.encodeBlock(inputBuffer, outputBuffer); 16 | return outputBuffer.toArrayBuffer().slice(0, lz4OutputSize); 17 | } 18 | 19 | function lz4Decode(input) { 20 | let inputBuffer = new Buffer(input); 21 | let outputBuffer = new Buffer(gFileBuffer.byteLength); 22 | let lz4OutputSize = LZ4.decodeBlock(inputBuffer, outputBuffer); 23 | return outputBuffer.toArrayBuffer().slice(0, lz4OutputSize); 24 | } 25 | 26 | /******************************************************************************/ 27 | 28 | let lz4BlockJS; 29 | 30 | function lz4BlockJSEncode(input) { 31 | if ( !lz4BlockJS ) { return; } 32 | return lz4BlockJS.encodeBlock(input, 0); 33 | } 34 | 35 | function lz4BlockJSDecode(input) { 36 | if ( !lz4BlockJS ) { return; } 37 | return lz4BlockJS.decodeBlock(input, 0, gFileBuffer.byteLength); 38 | } 39 | 40 | /******************************************************************************/ 41 | 42 | let lz4BlockWASM; 43 | 44 | function lz4BlockWASMEncode(input) { 45 | if ( !lz4BlockWASM ) { return; } 46 | return lz4BlockWASM.encodeBlock(input, 0); 47 | } 48 | 49 | function lz4BlockWASMDecode(input) { 50 | if ( !lz4BlockWASM ) { return; } 51 | return lz4BlockWASM.decodeBlock(input, 0, gFileBuffer.byteLength); 52 | } 53 | 54 | /******************************************************************************/ 55 | 56 | function validateCodec(a) { 57 | let b = gFileBuffer; 58 | if ( a.byteLength !== b.byteLength ) { return false; } 59 | let ab = a; 60 | if ( ab instanceof Uint8Array === false ) { 61 | ab = new Uint8Array(a); 62 | } 63 | let bb = new Uint8Array(b); 64 | for ( let i = 0, n = ab.byteLength; i < n; i++ ) { 65 | if ( ab[i] !== bb[i] ) { return false; } 66 | } 67 | return true; 68 | } 69 | 70 | /******************************************************************************/ 71 | 72 | function openFile(ev) { 73 | let input = ev.target; 74 | if ( input.files.length === 0 ) { return; } 75 | let fr = new FileReader(); 76 | fr.onload = processFile; 77 | fr.readAsArrayBuffer(input.files[0]); 78 | } 79 | 80 | function processFile(ev) { 81 | stdout(0, ''); 82 | gFileBuffer = ev.target.result; 83 | // Ensure valid encoders: encode/decode than compare result with 84 | // original. 85 | let ulen = gFileBuffer.byteLength, clen; 86 | let compressed; 87 | 88 | // Verify codecs are working as expected 89 | compressed = SnappyJS.compress(gFileBuffer); 90 | if ( validateCodec(SnappyJS.uncompress(compressed)) ) { 91 | stdout(0, ' zhipeng-jia/snappyjs: '); 92 | clen = compressed.byteLength; 93 | stdout(0, clen.toLocaleString() + ' / ' + ulen.toLocaleString() + ' = '); 94 | stdout(0, (clen * 100 / ulen).toFixed(0) + '%'); 95 | stdout(0, '\n'); 96 | } else { 97 | stdout(0, ' zhipeng-jia/snappyjs: failed\n'); 98 | } 99 | 100 | compressed = lz4Encode(gFileBuffer); 101 | if ( validateCodec(lz4Decode(compressed)) ) { 102 | stdout(0, ' pierrec/node-lz4: '); 103 | clen = compressed.byteLength; 104 | stdout(0, clen.toLocaleString() + ' / ' + ulen.toLocaleString() + ' = '); 105 | stdout(0, (clen * 100 / ulen).toFixed(0) + '%'); 106 | stdout(0, '\n'); 107 | } else { 108 | stdout(0, ' pierrec/node-lz4: failed\n'); 109 | } 110 | 111 | // For lz4-block.js, we also verify that its output is really 112 | // lz4-compatible -- node-lz4 is used as the reference codec. 113 | if ( validateCodec(lz4Decode(lz4BlockJSEncode(gFileBuffer))) === false ) { 114 | stdout(0, ' gorhill/lz4-block.js: failed to encode\n'); 115 | } else if ( validateCodec(lz4BlockJSDecode(lz4Encode(gFileBuffer))) === false ) { 116 | stdout(0, ' gorhill/lz4-block.js: failed to decode\n'); 117 | } else { 118 | compressed = new Uint8Array(lz4BlockJSEncode(gFileBuffer)); 119 | if ( validateCodec(lz4BlockJSDecode(compressed)) ) { 120 | stdout(0, ' gorhill/lz4-block.js: '); 121 | clen = compressed.byteLength; 122 | stdout(0, clen.toLocaleString() + ' / ' + ulen.toLocaleString() + ' = '); 123 | stdout(0, (clen * 100 / ulen).toFixed(0) + '%'); 124 | stdout(0, '\n'); 125 | } else { 126 | stdout(0, ' gorhill/lz4-block.js: failed\n'); 127 | } 128 | } 129 | 130 | // For lz4-block.wasm, we also verify that its output is really 131 | // lz4-compatible -- node-lz4 is used as the reference codec. 132 | if ( validateCodec(lz4Decode(lz4BlockWASMEncode(gFileBuffer))) === false ) { 133 | stdout(0, 'gorhill/lz4-block.wasm: failed to encode\n'); 134 | } else if ( validateCodec(lz4BlockWASMDecode(lz4Encode(gFileBuffer))) === false ) { 135 | stdout(0, 'gorhill/lz4-block.wasm: failed to decode\n'); 136 | } else { 137 | compressed = new Uint8Array(lz4BlockWASMEncode(gFileBuffer)); 138 | if ( validateCodec(lz4BlockWASMDecode(compressed)) === false ) { 139 | stdout(0, 'gorhill/lz4-block.wasm: failed to self-decode\n'); 140 | } else { 141 | stdout(0, 'gorhill/lz4-block.wasm: '); 142 | clen = compressed.byteLength; 143 | stdout(0, clen.toLocaleString() + ' / ' + ulen.toLocaleString() + ' = '); 144 | stdout(0, (clen * 100 / ulen).toFixed(0) + '%'); 145 | stdout(0, '\n'); 146 | } 147 | } 148 | 149 | let runbtn = document.querySelector('#runBenchmark'); 150 | if ( gFileBuffer instanceof ArrayBuffer ) { 151 | runbtn.removeAttribute('disabled'); 152 | } else { 153 | runbtn.setAttribute('disabled', ''); 154 | } 155 | } 156 | 157 | /******************************************************************************/ 158 | 159 | let gFileBuffer; 160 | let gBenchmarks = []; 161 | let gWhich; 162 | 163 | function stdout(which, text) { 164 | var r = document.querySelector('#results-' + which); 165 | if ( text === '' ) { 166 | r.innerHTML = ''; 167 | } else { 168 | r.innerHTML += text; 169 | } 170 | } 171 | 172 | function prepareBenchmark() { 173 | gBenchmarks.push((function() { 174 | let bms = new Benchmark.Suite(); 175 | bms 176 | .add(' - zhipeng-jia/snappyjs', ( ) => { 177 | SnappyJS.compress(gFileBuffer); 178 | }) 179 | .add(' - pierrec/node-lz4', ( ) => { 180 | lz4Encode(gFileBuffer); 181 | }) 182 | .add(' - gorhill/lz4-block.js', ( ) => { 183 | lz4BlockJSEncode(gFileBuffer); 184 | }) 185 | .add(' - gorhill/lz4-block.wasm', ( ) => { 186 | lz4BlockWASMEncode(gFileBuffer); 187 | }) 188 | .on('start', function() { 189 | stdout(gWhich + 2, 'Compress:\n'); 190 | }) 191 | .on('cycle', event => { 192 | let mbps = Math.floor(event.target.hz * gFileBuffer.byteLength >>> 20); 193 | stdout(gWhich + 2, String(event.target) + ': ' + mbps + ' MB/s\n'); 194 | }) 195 | .on('complete', ( ) => { 196 | nextBenchmark(); 197 | }); 198 | return bms; 199 | })()); 200 | gBenchmarks.push((function() { 201 | let lz4Compressed; 202 | let lz4JSCompressed; 203 | let snappyCompressed; 204 | let lz4wasmCompressed; 205 | let bms = new Benchmark.Suite(); 206 | bms 207 | .add(' - zhipeng-jia/snappyjs', ( ) => { 208 | SnappyJS.uncompress(snappyCompressed); 209 | }) 210 | .add(' - pierrec/node-lz4', ( ) => { 211 | lz4Decode(lz4Compressed); 212 | }) 213 | .add(' - gorhill/lz4-block.js', ( ) => { 214 | lz4BlockJSDecode(lz4JSCompressed); 215 | }) 216 | .add(' - gorhill/lz4-block.wasm', ( ) => { 217 | lz4BlockWASMDecode(lz4wasmCompressed); 218 | }) 219 | .on('start', function() { 220 | snappyCompressed = SnappyJS.compress(gFileBuffer); 221 | lz4Compressed = lz4Encode(gFileBuffer); 222 | lz4JSCompressed = new Uint8Array(lz4BlockJSEncode(gFileBuffer)); 223 | lz4wasmCompressed = new Uint8Array(lz4BlockWASMEncode(gFileBuffer)); 224 | stdout(gWhich + 2, 'Uncompress:\n'); 225 | }) 226 | .on('cycle', event => { 227 | let mbps = Math.floor(event.target.hz * gFileBuffer.byteLength >>> 20); 228 | stdout(gWhich + 2, String(event.target) + ': ' + mbps + ' MB/s\n'); 229 | }) 230 | .on('complete', ( ) => { 231 | nextBenchmark(); 232 | }); 233 | return bms; 234 | })()); 235 | } 236 | 237 | /******************************************************************************/ 238 | 239 | function nextBenchmark() { 240 | stdout(gWhich + 2, ' Done.\n\n'); 241 | gWhich += 1; 242 | var bms = gBenchmarks[gWhich]; 243 | if ( bms ) { 244 | bms.run({ 'async': true }); 245 | } 246 | } 247 | 248 | function doBenchmark() { 249 | stdout(1, ''); 250 | stdout(1, 'Benchmarking, the higher ops/sec the better.\n'); 251 | stdout(1, Benchmark.platform.toString() + '.'); 252 | stdout(1, '\n\n'); 253 | for ( let i = 0; i < gBenchmarks.length; i++ ) { 254 | stdout(i + 2, ''); 255 | } 256 | gWhich = 0; 257 | gBenchmarks[gWhich].run({ 'async': true }); 258 | } 259 | 260 | Promise.all([ 261 | lz4BlockCodec.createInstance('js').then(instance => { 262 | lz4BlockJS = instance; 263 | }), 264 | lz4BlockCodec.createInstance('wasm').then(instance => { 265 | lz4BlockWASM = instance; 266 | }) 267 | ]).then(( ) => { 268 | document.querySelector('input[type="file"]') 269 | .addEventListener('change', openFile); 270 | prepareBenchmark(); 271 | document.getElementById('runBenchmark').onclick = function() { 272 | doBenchmark(); 273 | }; 274 | }); 275 | --------------------------------------------------------------------------------