├── .gitmodules ├── Test └── .empty ├── todo.txt ├── .gitignore ├── README.md ├── TestCases ├── expected_while.js ├── break_trailing_comma_after_label.js ├── trailing_while_parse_error.js └── break_eats_semicolons.js ├── parse-json.js ├── node-augh.js ├── configurations ├── uncompressed.json ├── preorder-varints-NTS.json ├── preorder-varints-NTS-brIf.json ├── default.json ├── reasonable-brIf.json ├── reasonable.json ├── dedupe-LAsort-VSPT-TTS-splitpreorder-varints-NTS.json └── dedupe-LAsort-VSPT-TTS-splitpreorder-varints-NTS-brIf.json ├── parse ├── treebuilder.js ├── deduplicating.js ├── asm-expressionchain.js ├── asmlike-json-treebuilder.js └── json-treebuilder.js ├── leb-test.js ├── size-comparison.sh ├── test-roundtrip.bat ├── third_party ├── encoding │ ├── MDN_LICENSE │ ├── JSIL_LICENSE │ ├── TIDY_LICENSE │ └── encoding.js └── cashew │ ├── LICENSE │ └── asm-tokenizer.js ├── test-roundtrip.sh ├── batch-test.sh ├── decode.js ├── encode.js ├── astutil.js ├── configuration.js ├── shapes-jsontree.json ├── LICENSE ├── ast-common.js ├── ast-decoder.js └── ast-encoder.js /.gitmodules: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Test/.empty: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /todo.txt: -------------------------------------------------------------------------------- 1 | post-order 2 | split string length headers into a stream -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /Test 2 | *.gz 3 | *.lzham 4 | *.brotli 5 | third_party/lzhamtest 6 | third_party/7z.exe 7 | third_party/bro -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Research prototype for investigating binary AST representation approaches. Experimenting to find the best balance of post-compression size, pre-compression size, complexity, and decode speed. 2 | -------------------------------------------------------------------------------- /TestCases/expected_while.js: -------------------------------------------------------------------------------- 1 | 2 | function gca() { 3 | a = k[184756] | 0; 4 | a: do 5 | if (a) 6 | do { 7 | k[184756] = a 8 | } while ((a | 0) != 0); 9 | while (0); 10 | return 11 | } 12 | -------------------------------------------------------------------------------- /parse-json.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/node 2 | 3 | var filename = process.argv[2]; 4 | var json = require('fs').readFileSync(filename, { encoding: 'utf8' }); 5 | console.time('JSON.parse'); 6 | var tree = JSON.parse(json); 7 | console.timeEnd('JSON.parse'); 8 | json = null; 9 | global.gc(); 10 | console.log('heapUsed ' + process.memoryUsage().heapUsed); 11 | -------------------------------------------------------------------------------- /node-augh.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | var typedArrayConstructors = [ 3 | Uint8Array, Uint16Array, Uint32Array, 4 | Int8Array, Int16Array, Int32Array, 5 | Float32Array, Float64Array 6 | ]; 7 | 8 | for (var i = 0, l = typedArrayConstructors.length; i < l; i++) { 9 | var ac = typedArrayConstructors[i]; 10 | 11 | var instance = new ac(1); 12 | 13 | if (!instance.slice) { 14 | Object.getPrototypeOf(instance).slice = function () { 15 | var subarray = this.subarray.apply(this, arguments); 16 | var copy = new ac(subarray); 17 | return copy; 18 | }; 19 | } 20 | } 21 | })(); -------------------------------------------------------------------------------- /configurations/uncompressed.json: -------------------------------------------------------------------------------- 1 | { 2 | "DeduplicateObjects": false, 3 | "DeduplicationUsageThreshold": 1, 4 | "EnableVarints": false, 5 | "SortTables": false, 6 | "LocalityAwareSorting": false, 7 | "LargeTableThreshold": 8192, 8 | "LocalityCutoffSmall": 128, 9 | "LocalityCutoffLarge": 8192, 10 | "LocalityMinimumThreshold": 3, 11 | "InlineUseCountThreshold": null, 12 | "InlineObjectSizeThreshold": null, 13 | "ConditionalInlining": false, 14 | "PartitionedInlining": false, 15 | "NoOverridingPrimitiveStream": false, 16 | "RelativeIndexes": false, 17 | "TypeTagStream": false, 18 | "ValueStreamPerType": false, 19 | "ThreeByteIndices": false, 20 | "NullTerminatedStrings": false 21 | } -------------------------------------------------------------------------------- /parse/treebuilder.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | (function (root, factory) { 4 | if (typeof define === 'function' && define.amd) { 5 | define(['exports'], factory); 6 | } else if (typeof exports !== 'undefined') { 7 | factory(exports); 8 | } else { 9 | factory((root.treeBuilder = {})); 10 | } 11 | }(this, function (exports) { 12 | var jsonTreeBuilder = require("./json-treebuilder.js"); 13 | var asmlikeJsonTreeBuilder = require("./asmlike-json-treebuilder.js"); 14 | var deduplicating = require("./deduplicating.js"); 15 | 16 | exports.JSON = jsonTreeBuilder.Builder; 17 | exports.AsmlikeJSON = asmlikeJsonTreeBuilder.Builder; 18 | exports.MakeDeduplicating = deduplicating.MakeBuilder; 19 | })); -------------------------------------------------------------------------------- /configurations/preorder-varints-NTS.json: -------------------------------------------------------------------------------- 1 | { 2 | "DeduplicateObjects": false, 3 | "DeduplicationUsageThreshold": 1, 4 | "EnableVarints": true, 5 | "SortTables": false, 6 | "LocalityAwareSorting": false, 7 | "LargeTableThreshold": 8192, 8 | "LocalityCutoffSmall": 128, 9 | "LocalityCutoffLarge": 8192, 10 | "LocalityMinimumThreshold": 3, 11 | "InlineUseCountThreshold": 10, 12 | "InlineObjectSizeThreshold": null, 13 | "ConditionalInlining": true, 14 | "PartitionedInlining": false, 15 | "NoOverridingPrimitiveStream": false, 16 | "RelativeIndexes": false, 17 | "TypeTagStream": false, 18 | "ValueStreamPerType": false, 19 | "ThreeByteIndices": false, 20 | "NullTerminatedStrings": true, 21 | "InternedSymbols": true, 22 | "LocalSymbolsBeforeGlobals": true, 23 | "BrIf": false 24 | } -------------------------------------------------------------------------------- /configurations/preorder-varints-NTS-brIf.json: -------------------------------------------------------------------------------- 1 | { 2 | "DeduplicateObjects": false, 3 | "DeduplicationUsageThreshold": 1, 4 | "EnableVarints": true, 5 | "SortTables": false, 6 | "LocalityAwareSorting": false, 7 | "LargeTableThreshold": 8192, 8 | "LocalityCutoffSmall": 128, 9 | "LocalityCutoffLarge": 8192, 10 | "LocalityMinimumThreshold": 3, 11 | "InlineUseCountThreshold": 10, 12 | "InlineObjectSizeThreshold": null, 13 | "ConditionalInlining": true, 14 | "PartitionedInlining": false, 15 | "NoOverridingPrimitiveStream": false, 16 | "RelativeIndexes": false, 17 | "TypeTagStream": false, 18 | "ValueStreamPerType": false, 19 | "ThreeByteIndices": false, 20 | "NullTerminatedStrings": true, 21 | "InternedSymbols": true, 22 | "LocalSymbolsBeforeGlobals": true, 23 | "BrIf": true 24 | } 25 | -------------------------------------------------------------------------------- /configurations/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "DeduplicateObjects": true, 3 | "DeduplicationUsageThreshold": 1, 4 | "EnableVarints": true, 5 | "SortTables": true, 6 | "LocalityAwareSorting": true, 7 | "LargeTableThreshold": 8192, 8 | "LocalityCutoffSmall": 128, 9 | "LocalityCutoffLarge": 8192, 10 | "LocalityMinimumThreshold": 3, 11 | "InlineUseCountThreshold": 10, 12 | "InlineObjectSizeThreshold": null, 13 | "ConditionalInlining": true, 14 | "PartitionedInlining": true, 15 | "NoOverridingPrimitiveStream": false, 16 | "RelativeIndexes": false, 17 | "TypeTagStream": true, 18 | "ValueStreamPerType": true, 19 | "ThreeByteIndices": false, 20 | "NullTerminatedStrings": true, 21 | "InternedSymbols": true, 22 | "LocalSymbolsBeforeGlobals": true, 23 | "PackedInliningFlags": true, 24 | "PackedIndexStream": false 25 | } -------------------------------------------------------------------------------- /configurations/reasonable-brIf.json: -------------------------------------------------------------------------------- 1 | { 2 | "DeduplicateObjects": true, 3 | "DeduplicationUsageThreshold": 1, 4 | "EnableVarints": true, 5 | "SortTables": true, 6 | "LocalityAwareSorting": false, 7 | "LargeTableThreshold": 8192, 8 | "LocalityCutoffSmall": 128, 9 | "LocalityCutoffLarge": 8192, 10 | "LocalityMinimumThreshold": 3, 11 | "InlineUseCountThreshold": 10, 12 | "InlineObjectSizeThreshold": null, 13 | "ConditionalInlining": true, 14 | "PartitionedInlining": false, 15 | "NoOverridingPrimitiveStream": false, 16 | "RelativeIndexes": false, 17 | "TypeTagStream": true, 18 | "ValueStreamPerType": true, 19 | "ThreeByteIndices": false, 20 | "NullTerminatedStrings": true, 21 | "InternedSymbols": true, 22 | "LocalSymbolsBeforeGlobals": true, 23 | "PackedInliningFlags": true, 24 | "PackedIndexStream": false, 25 | "BrIf": true 26 | } -------------------------------------------------------------------------------- /configurations/reasonable.json: -------------------------------------------------------------------------------- 1 | { 2 | "DeduplicateObjects": true, 3 | "DeduplicationUsageThreshold": 1, 4 | "EnableVarints": true, 5 | "SortTables": true, 6 | "LocalityAwareSorting": false, 7 | "LargeTableThreshold": 8192, 8 | "LocalityCutoffSmall": 128, 9 | "LocalityCutoffLarge": 8192, 10 | "LocalityMinimumThreshold": 3, 11 | "InlineUseCountThreshold": 10, 12 | "InlineObjectSizeThreshold": null, 13 | "ConditionalInlining": true, 14 | "PartitionedInlining": false, 15 | "NoOverridingPrimitiveStream": false, 16 | "RelativeIndexes": false, 17 | "TypeTagStream": true, 18 | "ValueStreamPerType": true, 19 | "ThreeByteIndices": false, 20 | "NullTerminatedStrings": true, 21 | "InternedSymbols": true, 22 | "LocalSymbolsBeforeGlobals": true, 23 | "PackedInliningFlags": true, 24 | "PackedIndexStream": false, 25 | "BrIf": false 26 | } -------------------------------------------------------------------------------- /configurations/dedupe-LAsort-VSPT-TTS-splitpreorder-varints-NTS.json: -------------------------------------------------------------------------------- 1 | { 2 | "DeduplicateObjects": true, 3 | "DeduplicationUsageThreshold": 1, 4 | "EnableVarints": true, 5 | "SortTables": true, 6 | "LocalityAwareSorting": true, 7 | "LargeTableThreshold": 8192, 8 | "LocalityCutoffSmall": 128, 9 | "LocalityCutoffLarge": 8192, 10 | "LocalityMinimumThreshold": 3, 11 | "InlineUseCountThreshold": 10, 12 | "InlineObjectSizeThreshold": null, 13 | "ConditionalInlining": true, 14 | "PartitionedInlining": true, 15 | "NoOverridingPrimitiveStream": false, 16 | "RelativeIndexes": false, 17 | "TypeTagStream": true, 18 | "ValueStreamPerType": true, 19 | "ThreeByteIndices": false, 20 | "NullTerminatedStrings": true, 21 | "InternedSymbols": true, 22 | "LocalSymbolsBeforeGlobals": true, 23 | "PackedInliningFlags": true, 24 | "PackedIndexStream": false, 25 | "BrIf": false 26 | } -------------------------------------------------------------------------------- /configurations/dedupe-LAsort-VSPT-TTS-splitpreorder-varints-NTS-brIf.json: -------------------------------------------------------------------------------- 1 | { 2 | "DeduplicateObjects": true, 3 | "DeduplicationUsageThreshold": 1, 4 | "EnableVarints": true, 5 | "SortTables": true, 6 | "LocalityAwareSorting": true, 7 | "LargeTableThreshold": 8192, 8 | "LocalityCutoffSmall": 128, 9 | "LocalityCutoffLarge": 8192, 10 | "LocalityMinimumThreshold": 3, 11 | "InlineUseCountThreshold": 10, 12 | "InlineObjectSizeThreshold": null, 13 | "ConditionalInlining": true, 14 | "PartitionedInlining": true, 15 | "NoOverridingPrimitiveStream": false, 16 | "RelativeIndexes": false, 17 | "TypeTagStream": true, 18 | "ValueStreamPerType": true, 19 | "ThreeByteIndices": false, 20 | "NullTerminatedStrings": true, 21 | "InternedSymbols": true, 22 | "LocalSymbolsBeforeGlobals": true, 23 | "PackedInliningFlags": true, 24 | "PackedIndexStream": false, 25 | "BrIf": true 26 | } -------------------------------------------------------------------------------- /leb-test.js: -------------------------------------------------------------------------------- 1 | require("./third_party/encoding/encoding.js"); 2 | var common = require("./ast-common.js"); 3 | 4 | function testValue (i) { 5 | var bw = encoding.makeByteWriter(); 6 | common.writeLEBInt32(bw, i); 7 | var bytes = bw.getResult(); 8 | var bytestr = ""; 9 | for (var j = 0; j < bytes.length; j++) { 10 | var byte = bytes[j]; 11 | var hex = byte.toString(16); 12 | if (hex.length < 2) 13 | hex = "0" + hex; 14 | 15 | bytestr += hex; 16 | } 17 | 18 | // console.log(i, bytestr); 19 | 20 | var br = encoding.makeByteReader(bytes); 21 | var expected = i | 0; 22 | var result = common.readLEBInt32(br); 23 | 24 | if (expected !== result) 25 | console.log("expected " + expected + " got " + result); 26 | }; 27 | 28 | testValue(0); 29 | testValue(-624485); 30 | testValue(0xFFF6789b); 31 | 32 | for (var i = -65539; i < 65539; i += 1) { 33 | testValue(i); 34 | } -------------------------------------------------------------------------------- /size-comparison.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | FILE=$1 3 | 4 | function sizeof { 5 | stat -c%s "$1" 6 | } 7 | 8 | function kb { 9 | echo "scale=1; $1 / 1024" | bc 10 | } 11 | 12 | function percentage { 13 | echo "scale=2; $1 * 100 / $2" | bc 14 | } 15 | 16 | rm -f $FILE.brotli $FILE.lzham $FILE.gz 17 | 18 | SIZE_RAW=$(sizeof $FILE) 19 | echo $FILE $(kb $SIZE_RAW)KiB 20 | 21 | #tooooooo slow 22 | #pigz -11 -f -k $FILE 23 | #SIZE_ZOPFLI=$(sizeof $FILE.gz) 24 | 25 | pigz -9 -f -k $FILE & 26 | #gonna die of old age before this finishes 27 | #third_party/bro --force --quality 11 --input $FILE --output $FILE.brotli & 28 | third_party/lzhamtest c $FILE $FILE.lzham > /dev/null & 29 | 30 | wait 31 | 32 | SIZE_GZIP=$(sizeof $FILE.gz) 33 | #SIZE_BROTLI=$(sizeof $FILE.brotli) 34 | SIZE_LZHAM=$(sizeof $FILE.lzham) 35 | 36 | echo " gzip " $(kb $SIZE_GZIP)KiB $(percentage $SIZE_GZIP $SIZE_RAW)% 37 | echo " lzham " $(kb $SIZE_LZHAM)KiB $(percentage $SIZE_LZHAM $SIZE_RAW)% 38 | #echo " brotli" $(kb $SIZE_BROTLI)KiB $(percentage $SIZE_BROTLI $SIZE_RAW)% 39 | -------------------------------------------------------------------------------- /test-roundtrip.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | set INFILE=%1\%2 4 | 5 | if "%INFILE%" == "\" ( 6 | set INFILE=.\ast-encoder.js 7 | set FILE_PREFIX=ast-encoder.js 8 | ) else ( 9 | set FILE_PREFIX=%2 10 | ) 11 | 12 | set CONFIGURATION_NAME=%3 13 | 14 | if "%CONFIGURATION_NAME%" == "" ( 15 | set CONFIGURATION_NAME=default 16 | ) 17 | 18 | set CONFIGURATION=configurations\%CONFIGURATION_NAME%.json 19 | set OUTDIR=Test\output\%CONFIGURATION_NAME% 20 | set OUTFILE=%OUTDIR%\%FILE_PREFIX%.binast 21 | 22 | mkdir %OUTDIR% 23 | 24 | del /Q /F %OUTFILE%.ast.json %OUTFILE%.ast.json 25 | 26 | echo // encoding 27 | node --expose-gc --nouse-idle-notification --max-old-space-size=8192 encode.js %INFILE% %OUTFILE% %OUTFILE%.ast.json %CONFIGURATION% 28 | echo // read ast json 29 | node --expose-gc parse-json.js "%OUTFILE%.ast.json" 30 | echo // decoding 31 | node --expose-gc --nouse-idle-notification --max-old-space-size=8192 decode.js %OUTFILE% %OUTFILE%.ast.decoded.json %CONFIGURATION% 32 | echo // size comparison 33 | call size-comparison.bat %OUTFILE% 34 | echo // diff 35 | fc /B %OUTFILE%.ast.json %OUTFILE%.ast.decoded.json 36 | 37 | :end 38 | -------------------------------------------------------------------------------- /third_party/encoding/MDN_LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /third_party/encoding/JSIL_LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2011-2015 Katelyn Gadd 4 | Some sponsored contributions (c) Mozilla Corporation 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /test-roundtrip.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | if [ -z $1 ]; then 5 | FILE_PREFIX=ast-encoder.js 6 | INFILE=./ast-encoder.js ; 7 | else 8 | FILE_PREFIX=$2 9 | INFILE=$1/$2 ; 10 | fi 11 | 12 | if [ -z $3 ]; then 13 | CONFIGURATION_NAME=default 14 | else 15 | CONFIGURATION_NAME=$3 16 | fi; 17 | 18 | CONFIGURATION=configurations/$CONFIGURATION_NAME.json 19 | OUTDIR=Test/output/$CONFIGURATION_NAME 20 | OUTFILE=$OUTDIR/$FILE_PREFIX.binast 21 | 22 | mkdir -p $OUTDIR 23 | 24 | rm -f $OUTFILE.ast.json $OUTFILE.ast.json 25 | 26 | # echo // input 27 | echo // encoding 28 | if [ '$4' == 'debug-encode' ]; then 29 | node-debug --expose-gc --nouse-idle-notification --max-old-space-size=8192 encode.js $INFILE $OUTFILE $OUTFILE.ast.json $CONFIGURATION 30 | else 31 | node --expose-gc --nouse-idle-notification --max-old-space-size=8192 encode.js $INFILE $OUTFILE $OUTFILE.ast.json $CONFIGURATION 32 | fi; 33 | echo // read ast json 34 | node --expose-gc parse-json.js "$OUTFILE.ast.json" 35 | echo // decoding 36 | if [ '$4' == 'debug-decode' ]; then 37 | node-debug --expose-gc --nouse-idle-notification --max-old-space-size=8192 decode.js $OUTFILE $OUTFILE.ast.decoded.json $CONFIGURATION 38 | else 39 | node --expose-gc --nouse-idle-notification --max-old-space-size=8192 decode.js $OUTFILE $OUTFILE.ast.decoded.json $CONFIGURATION 40 | fi; 41 | echo // size comparison 42 | ./size-comparison.sh $OUTFILE 43 | echo // diff 44 | diff $OUTFILE.ast.json $OUTFILE.ast.decoded.json -------------------------------------------------------------------------------- /batch-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | shopt -s nullglob 4 | 5 | FILE_PREFIX=$2 6 | NAME="${FILE_PREFIX%.*}" 7 | INFILE=$1/$2 8 | 9 | echo // Running encodings 10 | 11 | RUNNING_ENCODERS=0 12 | FORK_WIDTH=4 13 | 14 | for c in configurations/*.json 15 | do 16 | CONFIGURATION="$c" 17 | CONFIGURATION_FILENAME=$(basename "$CONFIGURATION") 18 | CONFIGURATION_NAME="${CONFIGURATION_FILENAME%.*}" 19 | 20 | OUTDIR=Test/output/$FILE_PREFIX/$CONFIGURATION_NAME 21 | OUTFILE=$OUTDIR/$NAME.binast 22 | 23 | mkdir -p $OUTDIR 24 | 25 | echo $OUTDIR 26 | node --expose-gc --nouse-idle-notification --max-old-space-size=8192 encode.js $INFILE $OUTFILE $OUTFILE.ast.json $CONFIGURATION > $OUTFILE.log & 27 | 28 | RUNNING_ENCODERS=$((RUNNING_ENCODERS + 1)) 29 | if ((RUNNING_ENCODERS >= FORK_WIDTH)) 30 | then 31 | RUNNING_ENCODERS=0 32 | echo ... waiting 33 | wait 34 | echo 35 | fi 36 | done 37 | 38 | wait 39 | echo 40 | 41 | echo // Doing compression tests 42 | 43 | ./size-comparison.sh $INFILE > $INFILE-sizes.log 44 | cat $INFILE-sizes.log 45 | 46 | RUNNING_COMPRESSORS=0 47 | 48 | for c in configurations/*.json 49 | do 50 | CONFIGURATION="$c" 51 | CONFIGURATION_FILENAME=$(basename "$CONFIGURATION") 52 | CONFIGURATION_NAME="${CONFIGURATION_FILENAME%.*}" 53 | 54 | OUTDIR=Test/output/$FILE_PREFIX/$CONFIGURATION_NAME 55 | OUTFILE=$OUTDIR/$NAME.binast 56 | 57 | echo $CONFIGURATION_NAME ... 58 | ./size-comparison.sh $OUTFILE > $OUTFILE-sizes.log & 59 | 60 | RUNNING_COMPRESSORS=$((RUNNING_COMPRESSORS + 1)) 61 | if ((RUNNING_COMPRESSORS >= FORK_WIDTH)) 62 | then 63 | RUNNING_COMPRESSORS=0 64 | echo ... waiting 65 | wait 66 | fi 67 | done 68 | 69 | wait 70 | 71 | echo // Done -------------------------------------------------------------------------------- /decode.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/node 2 | 3 | require('./astutil.js'); 4 | require('./node-augh.js'); 5 | require('./third_party/encoding/encoding.js'); 6 | 7 | var common = require("./ast-common.js"); 8 | var Configuration = require("./configuration.js"); 9 | var astDecoder = require('./ast-decoder.js'); 10 | var fs = require('fs'); 11 | 12 | if (process.argv.length < 4) { 13 | console.log("USAGE: decode input.bast [astOutput.json] [configuration.json]"); 14 | process.exit(1); 15 | } 16 | 17 | var inputFile = process.argv[2]; 18 | var outputAstFile = process.argv[3]; 19 | var configurationPath = process.argv[4]; 20 | 21 | if (!configurationPath) 22 | throw new Error("Configuration file required"); 23 | 24 | var configuration = Configuration.FromJson( 25 | fs.readFileSync(configurationPath, { encoding: "utf8" }) 26 | ); 27 | 28 | var inputBuffer = fs.readFileSync(inputFile), inputBytes; 29 | if (inputBuffer.toArrayBuffer) 30 | inputBytes = inputBuffer.toArrayBuffer; 31 | else 32 | inputBytes = new Uint8Array(inputBuffer); 33 | 34 | var shapes = astDecoder.ShapeTable.fromJson( 35 | fs.readFileSync("shapes-jsontree.json", { encoding: "utf8" }) 36 | ); 37 | 38 | console.time("bytesToModule"); 39 | var inputModule = astDecoder.bytesToModule(configuration, shapes, inputBytes); 40 | console.timeEnd("bytesToModule"); 41 | 42 | inputBytes = null; 43 | if ((typeof (global) !== "undefined") && global.gc) { 44 | global.gc(); 45 | } 46 | 47 | var outputAst = inputModule.root; 48 | 49 | console.log("heapUsed " + process.memoryUsage().heapUsed); 50 | 51 | if (common.DumpJson && outputAstFile) { 52 | var json; 53 | if (astDecoder.PrettyJson) 54 | json = JSON.stringify(outputAst, null, 2) 55 | else 56 | json = JSON.stringify(outputAst); 57 | 58 | fs.writeFileSync(outputAstFile, json); 59 | } -------------------------------------------------------------------------------- /third_party/encoding/TIDY_LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 1998-2008 World Wide Web Consortium 2 | (Massachusetts Institute of Technology, European Research 3 | Consortium for Informatics and Mathematics, Keio University). 4 | All Rights Reserved. 5 | 6 | CVS Info : 7 | 8 | $Author: arnaud02 $ 9 | $Date: 2008/04/22 11:00:42 $ 10 | $Revision: 1.22 $ 11 | 12 | Contributing Author(s): 13 | 14 | Dave Raggett 15 | 16 | The contributing author(s) would like to thank all those who 17 | helped with testing, bug fixes and suggestions for improvements. 18 | This wouldn't have been possible without your help. 19 | 20 | COPYRIGHT NOTICE: 21 | 22 | This software and documentation is provided "as is," and 23 | the copyright holders and contributing author(s) make no 24 | representations or warranties, express or implied, including 25 | but not limited to, warranties of merchantability or fitness 26 | for any particular purpose or that the use of the software or 27 | documentation will not infringe any third party patents, 28 | copyrights, trademarks or other rights. 29 | 30 | The copyright holders and contributing author(s) will not be held 31 | liable for any direct, indirect, special or consequential damages 32 | arising out of any use of the software or documentation, even if 33 | advised of the possibility of such damage. 34 | 35 | Permission is hereby granted to use, copy, modify, and distribute 36 | this source code, or portions hereof, documentation and executables, 37 | for any purpose, without fee, subject to the following restrictions: 38 | 39 | 1. The origin of this source code must not be misrepresented. 40 | 2. Altered versions must be plainly marked as such and must 41 | not be misrepresented as being the original source. 42 | 3. This Copyright notice may not be removed or altered from any 43 | source or altered source distribution. 44 | 45 | The copyright holders and contributing author(s) specifically 46 | permit, without fee, and encourage the use of this source code 47 | as a component for supporting the Hypertext Markup Language in 48 | commercial products. If you use this source code in a product, 49 | acknowledgment is not required but would be appreciated. 50 | 51 | 52 | Created 2001-05-20 by Charles Reitzel 53 | Updated 2002-07-01 by Charles Reitzel - 1st Implementation -------------------------------------------------------------------------------- /encode.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/node 2 | 3 | require('./astutil.js'); 4 | require('./node-augh.js'); 5 | require('./third_party/encoding/encoding.js'); 6 | 7 | var common = require("./ast-common.js"); 8 | var Configuration = require("./configuration.js"); 9 | var asmParse = require('./third_party/cashew/asm-parse.js'); 10 | var astEncoder = require('./ast-encoder.js'); 11 | var treeBuilder = require("./parse/treebuilder.js"); 12 | var fs = require('fs'); 13 | 14 | if (process.argv.length < 4) { 15 | console.log("USAGE: encode input.js output.bast [astOutput.json] [configuration.json]"); 16 | process.exit(1); 17 | } 18 | 19 | var inputFile = process.argv[2]; 20 | var outputFile = process.argv[3]; 21 | var outputAstFile = process.argv[4]; 22 | var configurationPath = process.argv[5]; 23 | 24 | if (!configurationPath) 25 | throw new Error("Configuration file required"); 26 | 27 | console.log("// Configuration " + configurationPath); 28 | var configuration = Configuration.FromJson( 29 | fs.readFileSync(configurationPath, { encoding: "utf8" }) 30 | ); 31 | 32 | var shapes = astEncoder.ShapeTable.fromJson( 33 | fs.readFileSync("shapes-jsontree.json", { encoding: "utf8" }) 34 | ); 35 | 36 | var inputJs = fs.readFileSync(inputFile, { encoding: "utf8" }); 37 | var fdOut = fs.openSync(outputFile, "w"); 38 | 39 | var inputReader = encoding.makeCharacterReader(inputJs); 40 | 41 | var astBuilderType = astEncoder.JsAstModuleBuilder; 42 | 43 | if (configuration.DeduplicateObjects) { 44 | astBuilderType = treeBuilder.MakeDeduplicating( 45 | astBuilderType, 46 | common.GetObjectId, 47 | configuration.DeduplicationUsageThreshold 48 | ); 49 | } 50 | 51 | var astBuilder = new (astBuilderType)(configuration, shapes); 52 | astBuilder.brIf = configuration.BrIf; 53 | 54 | console.time("asm-parse"); 55 | var inputAst = asmParse.parse(inputReader, astBuilder); 56 | console.timeEnd("asm-parse"); 57 | 58 | var outputModule = astBuilder.finish(inputAst); 59 | 60 | if (configuration.DeduplicateObjects) { 61 | console.log( 62 | astBuilder.nodesPruned + "/" + astBuilder.nodesFinalized + 63 | " parse nodes deduplicated (" + 64 | (astBuilder.nodesPruned / astBuilder.nodesFinalized * 100) 65 | .toFixed(1) + "%)" 66 | ); 67 | } 68 | 69 | // TODO: Deduplicate arrays? 70 | 71 | console.time("serializeModule"); 72 | var bytes = astEncoder.serializeModule(outputModule); 73 | console.timeEnd("serializeModule"); 74 | 75 | fs.writeSync(fdOut, new Buffer(bytes), 0, bytes.length); 76 | 77 | if (common.DumpJson && outputAstFile) { 78 | var converter = function (k, v) { 79 | if ((typeof (v) === "object") && v && (v.type === "symbol")) { 80 | return v.valueOf(); 81 | } 82 | 83 | return v; 84 | }; 85 | 86 | var json; 87 | if (astEncoder.PrettyJson) 88 | json = JSON.stringify(inputAst, converter, 2) 89 | else 90 | json = JSON.stringify(inputAst, converter); 91 | 92 | fs.writeFileSync(outputAstFile, json); 93 | } 94 | 95 | fs.closeSync(fdOut); -------------------------------------------------------------------------------- /parse/deduplicating.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | (function (root, factory) { 4 | if (typeof define === 'function' && define.amd) { 5 | define(['exports'], factory); 6 | } else if (typeof exports !== 'undefined') { 7 | factory(exports); 8 | } else { 9 | factory((root.deduplicatingTreeBuilder = {})); 10 | } 11 | }(this, function (exports) { 12 | function MakeBuilder (baseClass, getObjectId, deduplicationUsageThreshold) { 13 | var ctor = function (/* ... */) { 14 | this.internTable = Object.create(null); 15 | this.hitCounts = new WeakMap(); 16 | this.nodesFinalized = 0; 17 | this.nodesPruned = 0; 18 | 19 | return baseClass.apply(this, arguments); 20 | }; 21 | 22 | ctor.prototype = Object.create(baseClass.prototype); 23 | 24 | ctor.prototype.$getTable = function (type) { 25 | var result = this.internTable[type]; 26 | if (!result) 27 | result = this.internTable[type] = Object.create(null); 28 | 29 | return result; 30 | }; 31 | 32 | var appendValueHash = function (head, v) { 33 | var vt = typeof (v); 34 | if (v === null) 35 | head += "\x01" 36 | else if (Array.isArray(v)) { 37 | // It's important to examine the structure of the array, 38 | // since we want to deduplicate by contents instead of 39 | // by referential identity 40 | head += "\x04"; 41 | 42 | for (var i = 0, l = v.length; i < l; i++) { 43 | var element = v[i]; 44 | head = appendValueHash(head, element); 45 | } 46 | 47 | head += "\x05"; 48 | } else if (vt === "object") 49 | head += getObjectId(v); 50 | else if (vt === "string") 51 | head += v; 52 | else if (vt === "number") 53 | head += v.toString(); 54 | else if (vt === "boolean") 55 | head += v ? "\x03" : "\x02"; 56 | else 57 | throw new Error("Unexpected '" + vt + "' while interning"); 58 | 59 | head += "\x00"; 60 | 61 | return head; 62 | }; 63 | 64 | ctor.prototype.getHitCount = function (obj) { 65 | var result = this.hitCounts.get(obj) || 0; 66 | return result | 0; 67 | }; 68 | 69 | ctor.prototype.finalize = function (obj) { 70 | var table = this.$getTable(obj.type); 71 | this.nodesFinalized += 1; 72 | 73 | // TODO: Generate some sort of cheap integer hash instead 74 | // TODO: Use shape table instead of walking properties 75 | var bruteForceHash = ""; 76 | for (var k in obj) { 77 | var v = obj[k]; 78 | 79 | try { 80 | bruteForceHash = appendValueHash(bruteForceHash, v); 81 | } catch (e) { 82 | console.log("Was interning o[" + k + "] for o=", obj); 83 | throw e; 84 | } 85 | } 86 | 87 | var interned = table[bruteForceHash]; 88 | if (interned) { 89 | var useCount = this.hitCounts.get(interned); 90 | this.hitCounts.set(interned, useCount + 1); 91 | 92 | var shouldDeduplicate = useCount >= deduplicationUsageThreshold 93 | if (shouldDeduplicate) { 94 | this.nodesPruned += 1; 95 | return interned; 96 | } 97 | } 98 | 99 | table[bruteForceHash] = obj; 100 | this.hitCounts.set(obj, 1); 101 | 102 | return baseClass.prototype.finalize.call(this, obj); 103 | }; 104 | 105 | return ctor; 106 | }; 107 | 108 | exports.MakeBuilder = MakeBuilder; 109 | })); -------------------------------------------------------------------------------- /astutil.js: -------------------------------------------------------------------------------- 1 | astutil = Object.create(null); 2 | 3 | 4 | astutil.Context = function () { 5 | this.isAborted = false; 6 | this.stack = []; 7 | this.parent = null; 8 | this.key = null; 9 | }; 10 | 11 | astutil.Context.prototype.abort = function () { 12 | this.isAborted = true; 13 | }; 14 | 15 | astutil.Context.prototype.onCycleDetected = function (target) { 16 | }; 17 | 18 | 19 | astutil.Result = function () { 20 | this.parent = null; 21 | this.key = null; 22 | this.node = null; 23 | }; 24 | 25 | astutil.Result.prototype.set = function (context, node) { 26 | this.parent = context.parent; 27 | this.key = context.key; 28 | this.node = node; 29 | }; 30 | 31 | astutil.Result.prototype.remove = function () { 32 | if (!this.parent || (this.key === null)) 33 | return false; 34 | 35 | if (Array.isArray(this.parent)) { 36 | this.parent.splice(this.key, 1); 37 | } else { 38 | delete this.parent[this.key]; 39 | } 40 | 41 | this.key = null; 42 | return true; 43 | }; 44 | 45 | astutil.Result.prototype.replaceWith = function (newNode) { 46 | this.parent[this.key] = newNode; 47 | }; 48 | 49 | 50 | astutil.clone = function (ast) { 51 | return JSON.parse(JSON.stringify(ast)); 52 | }; 53 | 54 | // Mutates an AST subtree, partially in-place. 55 | // Returns the new root (typically the original root, but it may have been replaced.) 56 | // If you don't want to modify any of the original nodes, make a copy first. 57 | astutil.mutate = function (root, mutator, context) { 58 | if (!context) 59 | context = new astutil.Context(); 60 | 61 | context.stack.push(root); 62 | 63 | try { 64 | var newRoot = root; 65 | if (mutator) 66 | newRoot = mutator(context, root); 67 | if (typeof (newRoot) === "undefined") 68 | newRoot = root; 69 | 70 | if (context.isAborted) 71 | return newRoot; 72 | 73 | for (var k in newRoot) { 74 | if (context.isAborted) 75 | return newRoot; 76 | 77 | if (!newRoot.hasOwnProperty(k)) 78 | continue; 79 | 80 | var v = newRoot[k]; 81 | if (typeof(v) !== "object") 82 | continue; 83 | 84 | if (context.stack.indexOf(v) >= 0) { 85 | context.onCycleDetected(v); 86 | continue; 87 | } 88 | 89 | context.parent = newRoot; 90 | context.key = k; 91 | 92 | var newValue = astutil.mutate(v, mutator, context); 93 | if (typeof (newValue) === "undefined") { 94 | /* do nothing */ ; 95 | } else if (newValue !== v) { 96 | /* replace */ 97 | newRoot[k] = newValue; 98 | } 99 | } 100 | 101 | return newRoot; 102 | } finally { 103 | context.stack.pop(); 104 | } 105 | }; 106 | 107 | astutil.find = function (root, predicate) { 108 | var result = new astutil.Result(); 109 | 110 | var _predicate = predicate; 111 | if (_predicate.length === 1) 112 | _predicate = function (context, node) { 113 | return predicate(node); 114 | }; 115 | 116 | astutil.mutate(root, function (context, node) { 117 | if (_predicate(context, node)) { 118 | result.set(context, node); 119 | context.abort(); 120 | } 121 | }); 122 | 123 | return result; 124 | }; 125 | 126 | astutil.assertNoCycles = function (root) { 127 | var context = new astutil.Context(); 128 | context.onCycleDetected = function (target) { 129 | console.log(context.stack.map(function (n) { return n.type; })); 130 | console.log("Cycle pointing at", target); 131 | throw new Error("Cycle detected in AST"); 132 | }; 133 | 134 | astutil.mutate(root, null, context); 135 | }; -------------------------------------------------------------------------------- /TestCases/break_trailing_comma_after_label.js: -------------------------------------------------------------------------------- 1 | 2 | function d3(a, b, c, d) { 3 | a = a | 0; 4 | b = b | 0; 5 | c = c | 0; 6 | d = d | 0; 7 | var e = 0, 8 | f = 0, 9 | g = 0, 10 | h = 0, 11 | j = 0, 12 | l = 0, 13 | m = 0, 14 | n = 0, 15 | o = 0, 16 | p = 0; 17 | p = r; 18 | if ((c | 0) == 0 | (a | 0) == (c | 0)) { 19 | b = 0; 20 | r = p; 21 | return b | 0 22 | } 23 | if (!((i[a + 124 >> 0] | 0) != 1 & (i[731736] | 0) == 0)) { 24 | b = 0; 25 | r = p; 26 | return b | 0 27 | } 28 | if (i[c + 124 >> 0] | 0) { 29 | b = 0; 30 | r = p; 31 | return b | 0 32 | } 33 | l = (k[178362] | 0) + 3 | 0; 34 | if ((l >>> 0 < 26 ? (67107488 >>> l & 1 | 0) != 0 : 0) ? (Hja(a + 844 | 0, c + 844 | 0) | 0) == 0 : 0) { 35 | b = 0; 36 | r = p; 37 | return b | 0 38 | } 39 | a: do 40 | if (d) { 41 | o = b + 8 | 0; 42 | b: do 43 | if ((k[o >> 2] | 0) != 2 | (d & 1 | 0) == 0 ? (j = k[c + 1376 >> 2] | 0, m = a + 1376 | 0, g = k[m >> 2] | 0, (g | 0) > 0 & (k[187324] | 0) > (g | 0)) : 0) { 44 | l = a + 1368 | 0; 45 | f = 1; 46 | h = 0; 47 | while (1) { 48 | e = k[l >> 2] | 0; 49 | if ((f ? (k[e + 20 >> 2] | 0) > 1 : 0) ? (k[k[e + 12 >> 2] >> 2] | 0) == (j | 0) : 0) break; 50 | if (Nea(a, g, j, e + 12 | 0, 712368, h) | 0) { 51 | n = 16; 52 | break 53 | } 54 | if ((h | 0) >= 2) break b; 55 | g = k[m >> 2] | 0; 56 | if (!((g | 0) > 0 & (k[187324] | 0) > (g | 0))) break b; 57 | else { 58 | f = 0; 59 | h = h + 1 | 0 60 | } 61 | } 62 | if ((n | 0) == 16) { 63 | i[b + 20 >> 0] = 0; 64 | e = k[l >> 2] | 0 65 | } 66 | f = k[c + 436 >> 2] | 0; 67 | g = k[b >> 2] | 0; 68 | if ((g | 0) == 2) 69 | if ((k[o >> 2] | 0) == 1) n = 21; 70 | else n = 22; 71 | else if ((g | 0) == 3 ? (k[o >> 2] | 0) == 0 : 0) n = 21; 72 | else n = 22; 73 | if ((n | 0) == 21) { 74 | k[b + 4 >> 2] = k[15422]; 75 | k[b + 12 >> 2] = f; 76 | k[b + 16 >> 2] = 0; 77 | i[b + 20 >> 0] = 0; 78 | break a 79 | } else if ((n | 0) == 22) { 80 | J3(e, 2, 1, f) | 0; 81 | break a 82 | } 83 | } 84 | while (0); 85 | if ((d | 0) > 2) { 86 | b = 0; 87 | r = p; 88 | return b | 0 89 | } 90 | } 91 | while (0); 92 | f = k[a + 1368 >> 2] | 0; 93 | g = f + 48 | 0; 94 | e = k[c + 436 >> 2] | 0; 95 | if ((k[g >> 2] | 0) == (e | 0)) { 96 | b = 1; 97 | r = p; 98 | return b | 0 99 | } 100 | b = k[15422] | 0; 101 | k[f + 56 >> 2] = b; 102 | k[f + 52 >> 2] = b; 103 | k[g >> 2] = e; 104 | b = 1; 105 | r = p; 106 | return b | 0 107 | } -------------------------------------------------------------------------------- /TestCases/trailing_while_parse_error.js: -------------------------------------------------------------------------------- 1 | 2 | function gca() { 3 | var a = 0, 4 | b = 0, 5 | c = 0, 6 | d = 0, 7 | e = 0, 8 | f = 0, 9 | g = 0, 10 | h = 0, 11 | j = 0, 12 | l = 0, 13 | m = 0, 14 | n = 0, 15 | o = 0; 16 | o = r; 17 | r = r + 16 | 0; 18 | n = o; 19 | d = (i[738672] | 0) == 0; 20 | if (d) k[184662] = (k[184662] | 0) + (k[15418] | 0); 21 | e = k[184660] | 0; 22 | b = e + 3 | 0; 23 | a = b >>> 0 < 26; 24 | do 25 | if (!(a & (e | 0) == -1)) { 26 | if (d) { 27 | if ((a ? (23 >>> b & 1 | 0) == 0 : 0) ? (k[184662] | 0) >= (k[184664] | 0) : 0) break; 28 | dca(); 29 | if ((k[15418] | 0) != 0 & (k[184848] | 0) > 0) { 30 | b = 0; 31 | do { 32 | d = k[184846] | 0; 33 | a = d + (b * 12 | 0) + 4 | 0; 34 | c = k[a >> 2] | 0; 35 | do 36 | if (c) { 37 | e = c - (k[15418] | 0) | 0; 38 | k[a >> 2] = e; 39 | if ((e | 0) < 1) { 40 | k[a >> 2] = 0; 41 | i[d + (b * 12 | 0) + 8 >> 0] = 1; 42 | k[n >> 2] = 24; 43 | k[n + 4 >> 2] = b; 44 | iW(-1, 1, 743096, n) | 0; 45 | break 46 | } 47 | if ((e | 0) < 10001 & (c | 0) > 1e4 ? (f = k[d + (b * 12 | 0) >> 2] | 0, (f | 0) == 15 | (f | 0) == 18) : 0) { 48 | k[n >> 2] = 67; 49 | k[n + 4 >> 2] = f; 50 | iW(-1, 1, 743096, n) | 0 51 | } 52 | } 53 | while (0); 54 | b = b + 1 | 0 55 | } while ((b | 0) < (k[184848] | 0)) 56 | } 57 | hca(); 58 | a = k[185510] | 0; 59 | if (a) vk[k[(k[a >> 2] | 0) + 48 >> 2] & 1023](a) 60 | } 61 | } else Fba(); 62 | while (0); 63 | a = k[184756] | 0; 64 | a: do 65 | if (a) 66 | do { 67 | b = k[184754] | 0; 68 | if (((k[b >> 2] | 0) - (k[15420] | 0) | 0) <= 144e5) break a; 69 | if ((a | 0) > 1) { 70 | e = b + 8 | 0; 71 | f = k[e + 4 >> 2] | 0; 72 | a = b; 73 | k[a >> 2] = k[e >> 2]; 74 | k[a + 4 >> 2] = f; 75 | a = k[184756] | 0; 76 | if ((a | 0) > 2) { 77 | d = 2; 78 | b = 1; 79 | while (1) { 80 | e = k[184754] | 0; 81 | f = e + (d << 3) | 0; 82 | a = k[f + 4 >> 2] | 0; 83 | b = e + (b << 3) | 0; 84 | k[b >> 2] = k[f >> 2]; 85 | k[b + 4 >> 2] = a; 86 | b = d + 1 | 0; 87 | a = k[184756] | 0; 88 | if ((b | 0) < (a | 0)) { 89 | f = d; 90 | d = b; 91 | b = f 92 | } else break 93 | } 94 | } 95 | } 96 | a = a + -1 | 0; 97 | k[184756] = a 98 | } while ((a | 0) != 0); 99 | while (0); 100 | 101 | return 102 | } -------------------------------------------------------------------------------- /configuration.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | (function (root, factory) { 4 | if (typeof define === 'function' && define.amd) { 5 | define(['exports'], factory); 6 | } else if (typeof exports !== 'undefined') { 7 | factory(exports); 8 | } else { 9 | factory((root.configuration = {})); 10 | } 11 | }(this, function (exports) { 12 | exports.Template = function () { 13 | // Prune duplicate objects before serializing a module. 14 | this.DeduplicateObjects = false; 15 | 16 | // At least this many identical nodes must exist for a node 17 | // to be deduplicated. 18 | this.DeduplicationUsageThreshold = 1; 19 | 20 | // Write indices as LEB128 32-bit uints instead of 4-byte uints 21 | this.EnableVarints = false; 22 | 23 | // Sorts the object table to reduce the average size of varints, 24 | // and potentially improve stream compression in general. 25 | this.SortTables = false; 26 | // Two-pass object table sort. Highest frequency use objects 27 | // at the front of the table, then lower frequency use objects 28 | // sorted sequentially to improve locality (and compression?) 29 | this.LocalityAwareSorting = false; 30 | // Tables over this size use the larger locality cutoff. 31 | // Roughly, we want to use this as a heuristic for cases 32 | // where indexes would otherwise frequently be 3 bytes 33 | // and hitcount sorting can reduce them to 2. 34 | this.LargeTableThreshold = 1 << 13; 35 | // How many items at the front of the table (hitcount sorted) 36 | // Small is used for 'small' tables (see above), large etc 37 | this.LocalityCutoffSmall = 128; 38 | this.LocalityCutoffLarge = 1 << 13; 39 | // How low can the minimum hitcount be 40 | this.LocalityMinimumThreshold = 3; 41 | 42 | // If set to an integer, objects with this # of uses or 43 | // less are encoded inline. 44 | this.InlineUseCountThreshold = 10; 45 | 46 | // If an object's estimated size is <= this, always inline it 47 | this.InlineObjectSizeThreshold = null; // 7; 48 | 49 | // See above 50 | this.ConditionalInlining = false; 51 | 52 | // If conditional inlining is active, writes inlined nodes 53 | // into their value streams instead of into the current stream 54 | this.PartitionedInlining = false; 55 | 56 | // When doing partitioned inlining, don't force primitive values 57 | // (ints, floats) out of their streams 58 | this.NoOverridingPrimitiveStream = false; 59 | 60 | // Separate sequential stream for all type tags 61 | this.TypeTagStream = false; 62 | 63 | // Separate sequential streams of values, partitioned by type. 64 | this.ValueStreamPerType = false; 65 | 66 | // If varints are disabled, writes indices as 3-byte uints 67 | this.ThreeByteIndices = false; 68 | 69 | // Null-terminated strings instead of length headers 70 | this.NullTerminatedStrings = false; 71 | 72 | // Maintains a scope chain and replaces symbols with numbered per-scope indices 73 | this.InternedSymbols = false; 74 | 75 | // Orders local symbols before global symbols in the index space. 76 | this.LocalSymbolsBeforeGlobals = false; 77 | 78 | // Packs the inlining flag into index values instead of writing it separately 79 | // This makes object references larger on average, but eliminates inline flags 80 | this.PackedInliningFlags = false; 81 | 82 | // When packed inlining is enables, moves packed indices out of individual streams 83 | // into a single stream 84 | this.PackedIndexStream = false; 85 | 86 | // Transform if statements into labels and br_if 87 | this.BrIf = false; 88 | }; 89 | 90 | exports.FromDictionary = function (dict) { 91 | var result = new exports.Template(); 92 | 93 | for (var k in dict) 94 | result[k] = dict[k]; 95 | 96 | if (result.RelativeIndexes) 97 | throw new Error("Relative indexes no longer supported"); 98 | 99 | return result; 100 | }; 101 | 102 | exports.FromJson = function (json) { 103 | var dict = JSON.parse(json); 104 | return exports.FromDictionary(dict); 105 | }; 106 | 107 | exports.ToJson = function (configuration) { 108 | return JSON.stringify(configuration, null, 2); 109 | }; 110 | 111 | })); -------------------------------------------------------------------------------- /shapes-jsontree.json: -------------------------------------------------------------------------------- 1 | { 2 | "shapeKey": "type", 3 | "shapes": { 4 | "MemberAccess": { 5 | "lhs": "Expression", 6 | "memberName": "symbol" 7 | }, 8 | "ComputedMemberAccess": { 9 | "lhs": "Expression", 10 | "rhs": "Expression" 11 | }, 12 | "Identifier": { 13 | "identifier": "symbol" 14 | }, 15 | "RegExpLiteral": { 16 | "pattern": "string", 17 | "flags": "string" 18 | }, 19 | "BooleanLiteral": { 20 | "value": "boolean" 21 | }, 22 | "StringLiteral": { 23 | "value": "string" 24 | }, 25 | "IntegerLiteral": { 26 | "value": "integer" 27 | }, 28 | "DoubleLiteral": { 29 | "value": "double" 30 | }, 31 | "ObjectLiteral": { 32 | "pairs": ["Pair"] 33 | }, 34 | "ArrayLiteral": { 35 | "elements": ["Expression"] 36 | }, 37 | "StringPair": { 38 | "key": "string", 39 | "value": "Expression" 40 | }, 41 | "NumberPair": { 42 | "key": "double", 43 | "value": "Expression" 44 | }, 45 | "NullStatement": { 46 | }, 47 | "ExpressionStatement": { 48 | "expression": "Expression" 49 | }, 50 | "BreakStatement": { 51 | "label": "symbol?" 52 | }, 53 | "ContinueStatement": { 54 | "label": "symbol?" 55 | }, 56 | "ReturnStatement": { 57 | "expression": "Expression?" 58 | }, 59 | "ThrowStatement": { 60 | "expression": "Expression?" 61 | }, 62 | "Comma": { 63 | "expressions": ["Expression"] 64 | }, 65 | "LabelStatement": { 66 | "label": "symbol", 67 | "labelled": "Statement | Expression" 68 | }, 69 | "BrIfFalse": { 70 | "condition": "Expression", 71 | "label": "symbol" 72 | }, 73 | "MultiLabelStatement": { 74 | "labels": ["symbol"], 75 | "labelled": "Statement | Expression" 76 | }, 77 | "ForStatement": { 78 | "initialize": "Expression?", 79 | "update": "Expression?", 80 | "condition": "Expression?", 81 | "body": "Block | Statement" 82 | }, 83 | "WhileStatement": { 84 | "condition": "Expression", 85 | "body": "Block | Statement" 86 | }, 87 | "DoWhileStatement": { 88 | "condition": "Expression", 89 | "body": "Block | Statement" 90 | }, 91 | "TryStatement": { 92 | "body": "Block | Statement", 93 | "catchExpression": "Expression?", 94 | "catchBlock": "Block? | Statement?", 95 | "finallyBlock": "Block? | Statement?" 96 | }, 97 | "ForInDeclaration": { 98 | "variableName": "symbol", 99 | "sequenceExpression": "Expression" 100 | }, 101 | "ForInStatement": { 102 | "declaration": "ForInDeclaration", 103 | "body": "Statement | Expression" 104 | }, 105 | "SwitchStatement": { 106 | "value": "Expression", 107 | "cases": ["SwitchCase"] 108 | }, 109 | "SwitchCase": { 110 | "value": "Expression?", 111 | "body": "Block? | Statement?" 112 | }, 113 | "PostfixMutation": { 114 | "operator": "symbol", 115 | "lhs": "Expression" 116 | }, 117 | "PrefixMutation": { 118 | "operator": "symbol", 119 | "rhs": "Expression" 120 | }, 121 | "UnaryOperator": { 122 | "operator": "symbol", 123 | "rhs": "Expression" 124 | }, 125 | "BinaryOperator": { 126 | "operator": "symbol", 127 | "lhs": "Expression", 128 | "rhs": "Expression" 129 | }, 130 | "AssignmentOperator": { 131 | "operator": "symbol", 132 | "lhs": "Expression", 133 | "rhs": "Expression" 134 | }, 135 | "TernaryOperator": { 136 | "condition": "Expression", 137 | "trueExpression": "Expression", 138 | "falseExpression": "Expression" 139 | }, 140 | "DeclarationStatement": { 141 | "declarations": ["Declaration"] 142 | }, 143 | "Declaration": { 144 | "name": "symbol", 145 | "initialValue": "Expression?" 146 | }, 147 | "Invocation": { 148 | "callee": "Expression", 149 | "argumentValues": ["Expression"] 150 | }, 151 | "Block": { 152 | "statements": ["Statement | Expression"] 153 | }, 154 | "IfStatement": { 155 | "condition": "Expression", 156 | "trueStatement": "Block | Statement", 157 | "falseStatement": "Block? | Statement?" 158 | }, 159 | "Function": { 160 | "name": "symbol?", 161 | "argumentNames": ["symbol"], 162 | "body": "Block" 163 | }, 164 | "TopLevel": { 165 | "statements": ["Statement"] 166 | }, 167 | 168 | "SignedTruncation": { 169 | "expression": "Expression" 170 | }, 171 | "UnsignedTruncation": { 172 | "expression": "Expression" 173 | }, 174 | "ToDouble": { 175 | "expression": "Expression" 176 | }, 177 | "UseAsmStatement": { 178 | } 179 | } 180 | } -------------------------------------------------------------------------------- /TestCases/break_eats_semicolons.js: -------------------------------------------------------------------------------- 1 | 2 | function D1() { 3 | var a = 0, 4 | b = 0, 5 | c = 0, 6 | d = 0, 7 | e = 0, 8 | f = 0, 9 | g = 0, 10 | h = 0, 11 | l = 0, 12 | m = 0.0, 13 | n = 0, 14 | p = 0, 15 | q = 0, 16 | s = 0.0, 17 | t = 0.0, 18 | u = 0.0, 19 | v = 0.0; 20 | q = r; 21 | r = r + 32 | 0; 22 | p = q + 8 | 0; 23 | n = q + 4 | 0; 24 | if (!(i[74792] | 0)) { 25 | lu(4, 708400, q); 26 | r = q; 27 | return 28 | } 29 | b = k[177216] | 0; 30 | if ((k[177094] | 0) == 0 | (b | 0) == 0) { 31 | r = q; 32 | return 33 | } 34 | k[177112] = 0; 35 | m = +(k[18661] | 0) / +(k[177212] | 0); 36 | a: do 37 | if ((b | 0) > 0) { 38 | d = p + 4 | 0; 39 | c = p + 8 | 0; 40 | h = 0; 41 | while (1) { 42 | l = k[177214] | 0; 43 | g = l + (h * 24 | 0) | 0; 44 | k[p + 0 >> 2] = k[g + 0 >> 2]; 45 | k[p + 4 >> 2] = k[g + 4 >> 2]; 46 | k[p + 8 >> 2] = k[g + 8 >> 2]; 47 | u = m * +o[d >> 2]; 48 | s = m * +o[c >> 2]; 49 | v = +(k[18656] | 0); 50 | t = +(k[18657] | 0); 51 | o[p >> 2] = +(k[18655] | 0) + m * +o[p >> 2]; 52 | o[d >> 2] = v + u; 53 | o[c >> 2] = t + s; 54 | if (z1(1, p, 0, j[l + (h * 24 | 0) + 12 >> 1] | 0, j[l + (h * 24 | 0) + 14 >> 1] | 0, j[l + (h * 24 | 0) + 16 >> 1] | 0, j[l + (h * 24 | 0) + 18 >> 1] | 0, j[l + (h * 24 | 0) + 20 >> 1] | 0, n) | 0) { 55 | l = k[n >> 2] | 0; 56 | i[708368] = 1; 57 | g = k[177112] | 0; 58 | if ((g | 0) == (k[177111] | 0)) { 59 | e = g + 1 | 0; 60 | if (!g) f = (e | 0) < 8 ? 8 : e; 61 | else { 62 | f = g; 63 | do f = f << 1; while ((f | 0) < (e | 0)) 64 | } 65 | k[177111] = f; 66 | if ((f | 0) > (g | 0)) { 67 | f = Zia(f << 2) | 0; 68 | if (!f) break; 69 | if ((g | 0) > 0 ? (a = k[177110] | 0, mka(f | 0, a | 0, g << 2 | 0) | 0, (a | 0) != 0) : 0) { 70 | _ia(a); 71 | g = k[177112] | 0 72 | } 73 | k[177110] = f 74 | } 75 | } 76 | k[(k[177110] | 0) + (g << 2) >> 2] = l; 77 | k[177112] = (k[177112] | 0) + 1; 78 | g = k[177204] | 0; 79 | l = l + 1 | 0; 80 | k[177204] = (g | 0) > (l | 0) ? g : l 81 | } 82 | h = h + 1 | 0; 83 | if ((h | 0) >= (k[177216] | 0)) break a 84 | } 85 | Cj() 86 | } 87 | while (0); 88 | k[177204] = 0; 89 | a1(); 90 | a = k[177082] | 0; 91 | if ((a | 0) > 0) { 92 | d = k[177084] | 0; 93 | if ((d | 0) <= -1) { 94 | r = q; 95 | return 96 | } 97 | e = M7() | 0; 98 | e = k[(k[e >> 2] | 0) + (d << 2) >> 2] | 0; 99 | g = e + 22 | 0; 100 | f = i[g >> 0] | 0; 101 | b2(2, d); 102 | b = i[(k[177214] | 0) + 22 >> 0] | 0; 103 | i[g >> 0] = b; 104 | if (f << 24 >> 24 != b << 24 >> 24) { 105 | c = e + 56 | 0; 106 | a = k[c >> 2] | 0; 107 | if (a) { 108 | k[a + 56 >> 2] = 0; 109 | k[c >> 2] = 0 110 | } 111 | } else b = f; 112 | if (b << 24 >> 24 != 0 ? (b2(3, d), f << 24 >> 24 != (i[g >> 0] | 0)) : 0) b1(e); 113 | r8(d, 1); 114 | r = q; 115 | return 116 | } 117 | a = a + 1 | 0; 118 | k[177082] = a; 119 | p = k[177084] | 0; 120 | b = k[177112] | 0; 121 | if ((b | 0) > 0) { 122 | l = 0; 123 | g = 0; 124 | while (1) { 125 | h = k[(k[177110] | 0) + (l << 2) >> 2] | 0; 126 | k[177084] = h; 127 | if ((h | 0) > -1) { 128 | d = M7() | 0; 129 | d = k[(k[d >> 2] | 0) + (h << 2) >> 2] | 0; 130 | b = d + 22 | 0; 131 | c = i[b >> 0] | 0; 132 | b2(2, h); 133 | a = g + 1 | 0; 134 | f = i[(k[177214] | 0) + (g * 24 | 0) + 22 >> 0] | 0; 135 | i[b >> 0] = f; 136 | if (c << 24 >> 24 != f << 24 >> 24) { 137 | e = d + 56 | 0; 138 | g = k[e >> 2] | 0; 139 | if (g) { 140 | k[g + 56 >> 2] = 0; 141 | k[e >> 2] = 0 142 | } 143 | } else f = c; 144 | if (f << 24 >> 24 != 0 ? (b2(3, h), c << 24 >> 24 != (i[b >> 0] | 0)) : 0) b1(d); 145 | r8(h, 1); 146 | b = k[177112] | 0 147 | } else a = g; 148 | l = l + 1 | 0; 149 | if ((l | 0) >= (b | 0)) break; 150 | else g = a 151 | } 152 | a = k[177082] | 0 153 | } 154 | k[177084] = p; 155 | k[177082] = a + -1; 156 | r = q; 157 | return 158 | } 159 | -------------------------------------------------------------------------------- /parse/asm-expressionchain.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | (function (root, factory) { 4 | if (typeof define === 'function' && define.amd) { 5 | define(['exports'], factory); 6 | } else if (typeof exports !== 'undefined') { 7 | factory(exports); 8 | } else { 9 | factory((root.asmExpressionChain = {})); 10 | } 11 | }(this, function (exports) { 12 | var BinaryPrecedences = ([ 13 | ["*", "/", "%"], 14 | ["+", "-"], 15 | ["<<", ">>", ">>>"], 16 | ["<", "<=", ">", ">=", "in", "instanceof"], 17 | ["==", "!=", "===", "!=="], 18 | ["&"], ["^"], ["|"], ["&&"], ["||"] 19 | ]).map(function (p) { 20 | var result = Object.create(null); 21 | for (var i = 0, l = p.length; i < l; i++) 22 | result[p[i]] = true; 23 | return result; 24 | }); 25 | 26 | 27 | function ChainExpressionNode (e) { 28 | this.expression = e; 29 | }; 30 | 31 | function ChainOperatorNode (o) { 32 | this.operator = o; 33 | }; 34 | 35 | 36 | function ExpressionChain (treeBuilder, trace) { 37 | this.items = []; 38 | this.builder = treeBuilder; 39 | this.trace = trace || false; 40 | }; 41 | 42 | ExpressionChain.prototype.abort = function (msg) { 43 | this.log(true); 44 | throw new Error(msg); 45 | }; 46 | 47 | ExpressionChain.prototype.pushExpression = function (e) { 48 | this.items.push(new ChainExpressionNode(e)); 49 | }; 50 | 51 | ExpressionChain.prototype.pushOperator = function (o) { 52 | this.items.push(new ChainOperatorNode(o)); 53 | }; 54 | 55 | ExpressionChain.prototype.isExpression = function (i) { 56 | var n = this.items[i]; 57 | return (n instanceof ChainExpressionNode); 58 | }; 59 | 60 | ExpressionChain.prototype.isOperator = function (i) { 61 | var n = this.items[i]; 62 | return (n instanceof ChainOperatorNode); 63 | }; 64 | 65 | ExpressionChain.prototype.at = function (i) { 66 | var n = this.items[i]; 67 | 68 | if (n instanceof ChainExpressionNode) 69 | return n.expression; 70 | else if (n instanceof ChainOperatorNode) 71 | return n.operator; 72 | else 73 | return null; 74 | }; 75 | 76 | ExpressionChain.prototype.replaceWithExpression = function (first, last, expression) { 77 | var count = (last - first) + 1; 78 | var node = new ChainExpressionNode(expression); 79 | this.items.splice(first, count, node); 80 | }; 81 | 82 | ExpressionChain.prototype.log = function (force) { 83 | if (this.trace || force) 84 | console.log("chain", this.items); 85 | } 86 | 87 | ExpressionChain.prototype.applyDecrementAndIncrement = function () { 88 | this.log(); 89 | 90 | for (var i = 0; i < this.length; i++) { 91 | switch (this.at(i)) { 92 | case "++": 93 | case "--": 94 | var newExpression; 95 | var isPrefix = this.isExpression(i + 1); 96 | var isPostfix = this.isExpression(i - 1); 97 | 98 | // FIXME: This doesn't detect and reject scenarios where the ++/-- 99 | // operators are being used on a non-identifier, but that's probably fine 100 | 101 | if (isPostfix) { 102 | newExpression = this.builder.makePostfixMutationExpression( 103 | this.at(i), 104 | this.at(i - 1) 105 | ); 106 | this.replaceWithExpression(i - 1, i, newExpression) 107 | i -= 1; 108 | } else if (isPrefix) { 109 | newExpression = this.builder.makePrefixMutationExpression( 110 | this.at(i), 111 | this.at(i + 1) 112 | ); 113 | this.replaceWithExpression(i, i + 1, newExpression) 114 | } else { 115 | return this.abort("Found a '" + this.at(i) + "' surrounded by operators"); 116 | } 117 | 118 | break; 119 | } 120 | } 121 | }; 122 | 123 | ExpressionChain.prototype.applyUnaryOperators = function () { 124 | this.log(); 125 | 126 | for (var i = this.length - 2; i >= 0; i--) { 127 | switch (this.at(i)) { 128 | case "+": 129 | case "-": 130 | if (this.isExpression(i - 1) && 131 | this.isExpression(i + 1)) { 132 | // This is binary arithmetic, so don't process it here 133 | break; 134 | } else { 135 | // Fall-through 136 | } 137 | 138 | case "!": 139 | case "~": 140 | case "typeof": 141 | case "void": 142 | case "delete": 143 | case "new": 144 | if (!this.isExpression(i + 1)) 145 | return this.abort("Found a prefix operator before a non-expression"); 146 | 147 | var rhs = this.at(i + 1); 148 | var newExpression = this.builder.makeUnaryOperatorExpression( 149 | this.at(i), 150 | this.at(i + 1) 151 | ); 152 | 153 | this.replaceWithExpression(i, i + 1, newExpression); 154 | 155 | break; 156 | } 157 | } 158 | }; 159 | 160 | ExpressionChain.prototype.applyBinaryOperators = function () { 161 | this.log(); 162 | 163 | for (var p = 0; p < BinaryPrecedences.length; p++) { 164 | var table = BinaryPrecedences[p]; 165 | 166 | for (var i = 1; i < (this.length - 1); i++) { 167 | if (!this.isOperator(i)) 168 | continue; 169 | 170 | if (table[this.at(i)]) { 171 | if ( 172 | !this.isExpression(i - 1) || 173 | !this.isExpression(i + 1) 174 | ) 175 | return this.abort("Found a binary operator without a lhs & rhs"); 176 | 177 | var lhs = this.at(i - 1); 178 | var rhs = this.at(i + 1); 179 | var newExpression = this.builder.makeBinaryOperatorExpression( 180 | this.at(i), 181 | lhs, rhs 182 | ); 183 | 184 | this.replaceWithExpression(i - 1, i + 1, newExpression); 185 | i -= 1; 186 | } 187 | } 188 | 189 | } 190 | }; 191 | 192 | /* 193 | A 194 | ? a 195 | : b 196 | 197 | [A ? a : b] 198 | 199 | A 200 | ? 201 | B 202 | ? a 203 | : b 204 | : c 205 | 206 | [A ? B ? a : b : c] 207 | */ 208 | 209 | ExpressionChain.prototype.applyTernaryOperator = function () { 210 | this.log(); 211 | 212 | for (var i = this.length - 2; i >= 0; i--) { 213 | if (!this.isOperator(i)) 214 | continue; 215 | 216 | var op = this.at(i); 217 | 218 | if (op === ":") { 219 | // Scan for an ? and parse there to properly apply associativity 220 | continue; 221 | } 222 | 223 | if (op === "?") { 224 | var condition = this.at(i - 1); 225 | var trueValue = this.at(i + 1); 226 | var colon = this.at(i + 2); 227 | var falseValue = this.at(i + 3); 228 | 229 | if (colon !== ":") { 230 | console.log(i, this.items); 231 | return this.abort("Expected : in ternary expression but found " + colon); 232 | } 233 | 234 | var newExpression = this.builder.makeTernaryOperatorExpression( 235 | condition, trueValue, falseValue 236 | ); 237 | this.replaceWithExpression(i - 1, i + 3, newExpression); 238 | 239 | // FIXME: Not sure about this 240 | i -= 1; 241 | } 242 | } 243 | }; 244 | 245 | ExpressionChain.prototype.applyAssignmentOperators = function () { 246 | this.log(); 247 | 248 | for (var i = 1; i < (this.length - 1); i++) { 249 | switch (this.at(i)) { 250 | case "=": 251 | case "+=": 252 | case "-=": 253 | case "*=": 254 | case "/=": 255 | case "%=": 256 | case "<<=": 257 | case ">>=": 258 | case ">>>=": 259 | case "&=": 260 | case "^=": 261 | case "|=": 262 | if ( 263 | !this.isExpression(i - 1) || 264 | !this.isExpression(i + 1) 265 | ) 266 | return this.abort("Found an assignment operator without a lhs & rhs"); 267 | 268 | // TODO: Assert that LHS is an identifier? 269 | 270 | var lhs = this.at(i - 1); 271 | var rhs = this.at(i + 1); 272 | var newExpression = this.builder.makeAssignmentOperatorExpression( 273 | this.at(i), 274 | lhs, rhs 275 | ); 276 | 277 | this.replaceWithExpression(i - 1, i + 1, newExpression); 278 | i -= 1; 279 | break; 280 | } 281 | } 282 | }; 283 | 284 | ExpressionChain.prototype.applyCommaOperator = function () { 285 | if (this.length === 1) 286 | return; 287 | 288 | this.log(); 289 | 290 | var expressions = []; 291 | 292 | for (var i = 0; i < this.length; i++) { 293 | if (!this.isOperator(i)) { 294 | var expr = this.at(i); 295 | 296 | expressions.push(expr); 297 | } else { 298 | var op = this.at(i); 299 | 300 | if (op !== ",") { 301 | // Bail out; we hit a non-expression that isn't a , operator 302 | // The caller will notice it has an unresolved chain and abort 303 | return; 304 | } 305 | } 306 | } 307 | 308 | var newExpression = this.builder.makeCommaExpression(expressions); 309 | this.replaceWithExpression(0, this.length - 1, newExpression); 310 | }; 311 | 312 | Object.defineProperty(ExpressionChain.prototype, "length", { 313 | enumerable: true, 314 | configurable: false, 315 | get: function () { 316 | return this.items.length; 317 | }, 318 | set: function (l) { 319 | this.items.length = l; 320 | } 321 | }); 322 | 323 | 324 | exports.ExpressionChain = ExpressionChain; 325 | })); -------------------------------------------------------------------------------- /third_party/encoding/encoding.js: -------------------------------------------------------------------------------- 1 | /// portions ganked from JSIL.Bootstrap.Text.js (see JSIL_LICENSE) 2 | /// utf8 decode/encode partially based on tidy (see TIDY_LICENSE) 3 | /// fromCharCode / charCodeAt based on MDN reference implementations, 4 | /// (MIT license due to predating Aug 20, 2010), (see MDN_LICENSE) 5 | 6 | encoding = Object.create(null); 7 | encoding.UTF8 = Object.create(null); 8 | 9 | 10 | encoding.fromCharCode = function fixedFromCharCode (codePt) { 11 | // https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects/String/fromCharCode 12 | if (codePt > 0xFFFF) { 13 | codePt -= 0x10000; 14 | return String.fromCharCode(0xD800 + (codePt >> 10), 0xDC00 + (codePt & 0x3FF)); 15 | } else { 16 | return String.fromCharCode(codePt); 17 | } 18 | }; 19 | 20 | 21 | encoding.charCodeAt = function fixedCharCodeAt (str, idx) { 22 | // https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects/String/charCodeAt 23 | 24 | idx = idx || 0; 25 | var code = str.charCodeAt(idx); 26 | var hi, low; 27 | 28 | if (0xD800 <= code && code <= 0xDBFF) { 29 | // High surrogate (could change last hex to 0xDB7F to treat high private surrogates as single characters) 30 | hi = code; 31 | low = str.charCodeAt(idx+1); 32 | if (isNaN(low)) 33 | throw new Error("High surrogate not followed by low surrogate"); 34 | 35 | return ((hi - 0xD800) * 0x400) + (low - 0xDC00) + 0x10000; 36 | } 37 | 38 | if (0xDC00 <= code && code <= 0xDFFF) { 39 | // Low surrogate 40 | // We return false to allow loops to skip this iteration since should have already handled high surrogate above in the previous iteration 41 | return false; 42 | } 43 | 44 | return code; 45 | }; 46 | 47 | 48 | /// makeByteWriter(Uint8Array buffer, offset) 49 | /// getResult -> (Uint8Array view on used region of buffer) 50 | /// makeByteWriter() 51 | /// getResult -> (Uint8Array containing written bytes) 52 | encoding.makeByteWriter = function (outputBytes, outputIndex) { 53 | if (arguments.length === 2) { 54 | var i = outputIndex | 0; 55 | var count = 0; 56 | 57 | return { 58 | write: function (byte) { 59 | if (i >= outputBytes.length) 60 | throw new Error("End of buffer"); 61 | 62 | outputBytes[i] = byte; 63 | i++; 64 | count++; 65 | }, 66 | getPosition: function () { 67 | return count; 68 | }, 69 | getResult: function () { 70 | return outputBytes.slice(outputIndex, outputIndex + count); 71 | } 72 | }; 73 | } else { 74 | var resultBytes = new Array(); 75 | 76 | return { 77 | write: function (byte) { 78 | resultBytes.push(byte); 79 | }, 80 | getPosition: function () { 81 | return resultBytes.length; 82 | }, 83 | getResult: function () { 84 | if (typeof (Uint8Array) !== "undefined") 85 | return new Uint8Array(resultBytes); 86 | else 87 | return resultBytes; 88 | } 89 | }; 90 | } 91 | }; 92 | 93 | 94 | encoding.makeByteReader = function (bytes, index, count) { 95 | var position = (typeof(index) === "number") ? index : 0; 96 | var endpoint; 97 | 98 | if (typeof (count) === "number") 99 | endpoint = (position + count); 100 | else 101 | endpoint = (bytes.length - position); 102 | 103 | var peek = function peek (offset) { 104 | offset |= 0; 105 | if (position + offset >= endpoint) 106 | return false; 107 | 108 | return bytes[position + offset]; 109 | }; 110 | 111 | var result = { 112 | peek: peek, 113 | read: function () { 114 | var result = peek(0); 115 | position += 1; 116 | return result; 117 | }, 118 | getPosition: function () { 119 | return position; 120 | }, 121 | skip: function (distance) { 122 | position += distance; 123 | } 124 | }; 125 | 126 | Object.defineProperty(result, "eof", { 127 | get: function () { 128 | return (position >= endpoint); 129 | }, 130 | configurable: true, 131 | enumerable: true 132 | }); 133 | 134 | return result; 135 | }; 136 | 137 | 138 | encoding.makeCharacterReader = function (str) { 139 | var position = 0, length = str.length; 140 | var cca = encoding.charCodeAt; 141 | 142 | var peek = function peek (offset) { 143 | offset |= 0; 144 | if (position + offset >= length) 145 | return false; 146 | 147 | return cca(str, position + offset); 148 | }; 149 | 150 | var result = { 151 | peek: peek, 152 | read: function () { 153 | var result = peek(0); 154 | position += 1; 155 | return result; 156 | }, 157 | getPosition: function () { 158 | return position; 159 | }, 160 | skip: function (distance) { 161 | position += distance; 162 | } 163 | }; 164 | 165 | Object.defineProperty(result, "eof", { 166 | get: function () { 167 | return (position >= length); 168 | }, 169 | configurable: true, 170 | enumerable: true 171 | }); 172 | 173 | return result; 174 | }; 175 | 176 | 177 | /// encode(str, outputBytes, outputOffset) -> numBytesWritten 178 | /// encode(str, outputWriter) -> numBytesWritten 179 | /// encode(str) -> Uint8Array 180 | encoding.UTF8.encode = function (string, output, outputIndex) { 181 | // http://tidy.sourceforge.net/cgi-bin/lxr/source/src/utf8.c 182 | 183 | var UTF8ByteSwapNotAChar = 0xFFFE; 184 | var UTF8NotAChar = 0xFFFF; 185 | 186 | var writer; 187 | if ((arguments.length === 3) && output.buffer) { 188 | writer = encoding.makeByteWriter(output, outputIndex); 189 | } else if (arguments.length === 2) { 190 | if (output && output.write && output.getResult) 191 | writer = output; 192 | else 193 | throw new Error("Expected 2nd arg to be a writer"); 194 | } else if (arguments.length === 1) { 195 | writer = encoding.makeByteWriter(); 196 | } 197 | 198 | if (typeof (string) !== "string") 199 | throw new Error("String expected"); 200 | else if (!writer) 201 | throw new Error("No writer available"); 202 | 203 | var reader = encoding.makeCharacterReader(string), ch; 204 | 205 | var hasError = false; 206 | 207 | while (!reader.eof) { 208 | ch = reader.read(); 209 | 210 | if (ch === false) 211 | continue; 212 | 213 | if (ch <= 0x7F) { 214 | writer.write( ch ); 215 | } else if (ch <= 0x7FF) { 216 | writer.write( 0xC0 | (ch >> 6) ); 217 | writer.write( 0x80 | (ch & 0x3F) ); 218 | } else if (ch <= 0xFFFF) { 219 | writer.write( 0xE0 | (ch >> 12) ); 220 | writer.write( 0x80 | ((ch >> 6) & 0x3F) ); 221 | writer.write( 0x80 | (ch & 0x3F) ); 222 | } else if (ch <= 0x1FFFF) { 223 | writer.write( 0xF0 | (ch >> 18) ); 224 | writer.write( 0x80 | ((ch >> 12) & 0x3F) ); 225 | writer.write( 0x80 | ((ch >> 6) & 0x3F) ); 226 | writer.write( 0x80 | (ch & 0x3F) ); 227 | 228 | if ((ch === UTF8ByteSwapNotAChar) || (ch === UTF8NotAChar)) 229 | hasError = true; 230 | } else if (ch <= 0x3FFFFFF) { 231 | writer.write( 0xF0 | (ch >> 24) ); 232 | writer.write( 0x80 | ((ch >> 18) & 0x3F) ); 233 | writer.write( 0x80 | ((ch >> 12) & 0x3F) ); 234 | writer.write( 0x80 | ((ch >> 6) & 0x3F) ); 235 | writer.write( 0x80 | (ch & 0x3F) ); 236 | 237 | hasError = true; 238 | } else if (ch <= 0x7FFFFFFF) { 239 | writer.write( 0xF0 | (ch >> 30) ); 240 | writer.write( 0x80 | ((ch >> 24) & 0x3F) ); 241 | writer.write( 0x80 | ((ch >> 18) & 0x3F) ); 242 | writer.write( 0x80 | ((ch >> 12) & 0x3F) ); 243 | writer.write( 0x80 | ((ch >> 6) & 0x3F) ); 244 | writer.write( 0x80 | (ch & 0x3F) ); 245 | 246 | hasError = true; 247 | } else { 248 | hasError = true; 249 | } 250 | } 251 | 252 | return writer.getResult(); 253 | }; 254 | 255 | 256 | encoding.UTF8.decode = function (bytes, index, count) { 257 | // http://tidy.sourceforge.net/cgi-bin/lxr/source/src/utf8.c 258 | 259 | var UTF8ByteSwapNotAChar = 0xFFFE; 260 | var UTF8NotAChar = 0xFFFF; 261 | 262 | var reader = encoding.makeByteReader(bytes, index, count), firstByte; 263 | var result = ""; 264 | 265 | while (!reader.eof) { 266 | var accumulator = 0, extraBytes = 0, hasError = false; 267 | firstByte = reader.read(); 268 | 269 | if (firstByte === false) 270 | continue; 271 | 272 | if (firstByte <= 0x7F) { 273 | accumulator = firstByte; 274 | } else if ((firstByte & 0xE0) === 0xC0) { 275 | accumulator = firstByte & 31; 276 | extraBytes = 1; 277 | } else if ((firstByte & 0xF0) === 0xE0) { 278 | accumulator = firstByte & 15; 279 | extraBytes = 2; 280 | } else if ((firstByte & 0xF8) === 0xF0) { 281 | accumulator = firstByte & 7; 282 | extraBytes = 3; 283 | } else if ((firstByte & 0xFC) === 0xF8) { 284 | accumulator = firstByte & 3; 285 | extraBytes = 4; 286 | hasError = true; 287 | } else if ((firstByte & 0xFE) === 0xFC) { 288 | accumulator = firstByte & 3; 289 | extraBytes = 5; 290 | hasError = true; 291 | } else { 292 | accumulator = firstByte; 293 | hasError = false; 294 | } 295 | 296 | while (extraBytes > 0) { 297 | var extraByte = reader.read(); 298 | extraBytes--; 299 | 300 | if (extraByte === false) { 301 | hasError = true; 302 | break; 303 | } 304 | 305 | if ((extraByte & 0xC0) !== 0x80) { 306 | hasError = true; 307 | break; 308 | } 309 | 310 | accumulator = (accumulator << 6) | (extraByte & 0x3F); 311 | } 312 | 313 | if ((accumulator === UTF8ByteSwapNotAChar) || (accumulator === UTF8NotAChar)) 314 | hasError = true; 315 | 316 | var characters; 317 | if (!hasError) 318 | characters = encoding.fromCharCode(accumulator); 319 | 320 | if (hasError || (characters === false)) { 321 | throw new Error("Invalid character in UTF8 text"); 322 | } else 323 | result += characters; 324 | } 325 | 326 | return result; 327 | }; -------------------------------------------------------------------------------- /parse/asmlike-json-treebuilder.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | (function (root, factory) { 4 | if (typeof define === 'function' && define.amd) { 5 | define(['exports'], factory); 6 | } else if (typeof exports !== 'undefined') { 7 | factory(exports); 8 | } else { 9 | factory((root.asmlikeJsonTreeBuilder = {})); 10 | } 11 | }(this, function (exports) { 12 | var jsonTreeBuilder = require("./json-treebuilder.js"); 13 | var JsonTreeBuilder = jsonTreeBuilder.Builder; 14 | 15 | var baseClass = JsonTreeBuilder.prototype; 16 | 17 | function Symbol (text, getBase, id) { 18 | this.text = text; 19 | this.getBase = getBase || null; 20 | this.id = id | 0; 21 | }; 22 | 23 | Symbol.prototype.type = "symbol"; 24 | 25 | Symbol.prototype.toString = function () { 26 | return "#" + this.id + " (" + this.text + ")"; 27 | }; 28 | 29 | Symbol.prototype.valueOf = function () { 30 | var base = 0; 31 | if (this.getBase) 32 | base = this.getBase() | 0; 33 | 34 | var result = base + this.id; 35 | 36 | return result; 37 | }; 38 | 39 | 40 | function AsmlikeJsonTreeBuilder () { 41 | JsonTreeBuilder.call(this); 42 | 43 | this.operatorTable = Object.create(null); 44 | this.operatorCount = 0; 45 | 46 | this.externalSymbols = Object.create(null); 47 | this.externalSymbolCount = 0; 48 | 49 | this.scopeChain = [ 50 | Object.create(null) 51 | ]; 52 | 53 | this.scopeChain[0].$$count = 0; 54 | this.maxLocalSymbolCount = 0; 55 | 56 | this.internSymbols = false; 57 | this.localSymbolsBeforeGlobals = false; 58 | 59 | this.brIf = false; 60 | 61 | var self = this; 62 | 63 | this.getLocalSymbolBase = function getLocalSymbolBase () { 64 | if (self.localSymbolsBeforeGlobals) 65 | return 0; 66 | else 67 | return self.externalSymbolCount; 68 | }; 69 | 70 | this.getGlobalSymbolBase = function getGlobalSymbolBase () { 71 | if (self.localSymbolsBeforeGlobals) 72 | return self.maxLocalSymbolCount; 73 | else 74 | return 0; 75 | }; 76 | }; 77 | 78 | AsmlikeJsonTreeBuilder.prototype = Object.create(JsonTreeBuilder.prototype); 79 | 80 | AsmlikeJsonTreeBuilder.prototype.pushScope = function () { 81 | var previousScope = this.scopeChain[this.scopeChain.length - 1]; 82 | var newScope = Object.create(previousScope); 83 | 84 | var count = previousScope.$$count; 85 | Object.defineProperty(newScope, "$$count", { 86 | value: count, 87 | configurable: true, 88 | enumerable: false, 89 | writable: true 90 | }); 91 | 92 | this.scopeChain.push(newScope); 93 | 94 | baseClass.pushScope.call(this); 95 | }; 96 | 97 | AsmlikeJsonTreeBuilder.prototype.popScope = function () { 98 | this.scopeChain.pop(); 99 | 100 | if (this.scopeChain.length < 1) 101 | throw new Error("Mismatched scope push/pop"); 102 | 103 | baseClass.popScope.call(this); 104 | }; 105 | 106 | // Special-case for operator names 107 | AsmlikeJsonTreeBuilder.prototype.internOperator = function (operator) { 108 | if (!this.internSymbols) 109 | return operator; 110 | 111 | var index = this.operatorTable[operator]; 112 | 113 | if (typeof (index) === "undefined") { 114 | index = this.operatorTable[operator] = 115 | new Symbol(operator, null, (this.operatorCount++) | 0); 116 | } 117 | 118 | return index; 119 | }; 120 | 121 | // FIXME: Externally visible names should have their names stored in a table somewhere 122 | // in order to accurately capture the cost of those names 123 | AsmlikeJsonTreeBuilder.prototype.internName = function (name, externallyVisible) { 124 | if (!this.internSymbols) 125 | return name; 126 | 127 | if (name === null) 128 | return 0xFFFFFFFF; 129 | 130 | if (externallyVisible) { 131 | var index = this.externalSymbols[name]; 132 | 133 | if (typeof (index) === "undefined") { 134 | index = this.externalSymbols[name] = 135 | new Symbol(name, this.getGlobalSymbolBase, (this.externalSymbolCount++) | 0); 136 | } 137 | } else { 138 | var currentScope = this.scopeChain[this.scopeChain.length - 1]; 139 | var index = currentScope[name]; 140 | 141 | if (typeof (index) === "undefined") { 142 | var i = (currentScope.$$count++); 143 | this.maxLocalSymbolCount = Math.max(i, this.maxLocalSymbolCount) | 0; 144 | 145 | index = currentScope[name] = 146 | new Symbol(name, this.getLocalSymbolBase, i | 0); 147 | } 148 | } 149 | 150 | // HACK: The shape table lists string for these... 151 | return index; 152 | }; 153 | 154 | AsmlikeJsonTreeBuilder.prototype.makeUnaryOperatorExpression = function (operator, rhs) { 155 | if (operator === "+") { 156 | var result = this.make("ToDouble"); 157 | result.expression = rhs; 158 | return this.finalize(result); 159 | } else { 160 | operator = this.internOperator(operator); 161 | 162 | return baseClass.makeUnaryOperatorExpression.call(this, operator, rhs); 163 | } 164 | }; 165 | 166 | AsmlikeJsonTreeBuilder.prototype.makeTruncation = function (isSigned, expression) { 167 | var result = this.make( 168 | isSigned 169 | ? "SignedTruncation" 170 | : "UnsignedTruncation" 171 | ); 172 | result.expression = expression; 173 | return this.finalize(result); 174 | }; 175 | 176 | AsmlikeJsonTreeBuilder.prototype.makeBinaryOperatorExpression = function (operator, lhs, rhs) { 177 | if ( 178 | ( 179 | (operator === ">>>") || 180 | (operator === "|") 181 | ) && 182 | (rhs.type === "IntegerLiteral") && 183 | (rhs.value === 0) 184 | ) { 185 | var isSigned = (operator === "|"); 186 | return this.makeTruncation(isSigned, lhs); 187 | } else { 188 | operator = this.internOperator(operator); 189 | 190 | return baseClass.makeBinaryOperatorExpression.call(this, operator, lhs, rhs); 191 | } 192 | }; 193 | 194 | AsmlikeJsonTreeBuilder.prototype.makePrefixMutationExpression = function (operator, rhs) { 195 | operator = this.internOperator(operator); 196 | 197 | return baseClass.makePrefixMutationExpression.call(this, operator, rhs); 198 | }; 199 | 200 | AsmlikeJsonTreeBuilder.prototype.makePostfixMutationExpression = function (operator, lhs) { 201 | operator = this.internOperator(operator); 202 | 203 | return baseClass.makePostfixMutationExpression.call(this, operator, lhs); 204 | }; 205 | 206 | AsmlikeJsonTreeBuilder.prototype.makeAssignmentOperatorExpression = function (operator, lhs, rhs) { 207 | operator = this.internOperator(operator); 208 | 209 | return baseClass.makeAssignmentOperatorExpression.call(this, operator, lhs, rhs); 210 | }; 211 | 212 | AsmlikeJsonTreeBuilder.prototype.makeExpressionStatement = function (expression) { 213 | if ( 214 | (expression.type === "StringLiteral") && 215 | (expression.value === "use asm") 216 | ) { 217 | var result = this.make("UseAsmStatement"); 218 | return this.finalize(result); 219 | } else { 220 | return baseClass.makeExpressionStatement.call(this, expression); 221 | } 222 | }; 223 | 224 | AsmlikeJsonTreeBuilder.prototype.makeLabelStatement = function (labels, labelled) { 225 | labels = Array.prototype.slice.call(labels); 226 | 227 | for (var i = 0; i < labels.length; i++) 228 | labels[i] = this.internName(labels[i], false); 229 | 230 | return baseClass.makeLabelStatement.call(this, labels, labelled); 231 | }; 232 | 233 | AsmlikeJsonTreeBuilder.prototype.makeDeclaration = function (name, initialValue) { 234 | name = this.internName(name, false); 235 | 236 | return baseClass.makeDeclaration.call(this, name, initialValue); 237 | }; 238 | 239 | AsmlikeJsonTreeBuilder.prototype.makeForInDeclaration = function (variableName, sequenceExpression) { 240 | variableName = this.internName(variableName, false); 241 | 242 | return baseClass.makeForInDeclaration.call(this, variableName, sequenceExpression); 243 | }; 244 | 245 | AsmlikeJsonTreeBuilder.prototype.makeIfStatement = function (condition, trueStatement, falseStatement) { 246 | if (this.brIf) { 247 | var id = this._ifCount++ | 0; 248 | var falseLabel = "$_if" + id + "_false"; 249 | var exitLabel = "$_if" + id + "_exit"; 250 | 251 | var mainBlock = this.makeBlock(); 252 | var mainBlockParent = this.makeLabelStatement([exitLabel], mainBlock); 253 | var falseBlock = this.makeBlock(); 254 | var falseBlockParent = this.makeLabelStatement([falseLabel], falseBlock); 255 | 256 | // exitLabel: { 257 | // falseLabel: { 258 | // if (cond) ; else break falseLabel; 259 | // trueStatement; 260 | // break exitLabel; 261 | // } 262 | // falseStatement; 263 | // } 264 | 265 | this.appendToBlock(mainBlock, falseBlockParent); 266 | 267 | var conditionalBranch = this.make("BrIfFalse"); 268 | conditionalBranch.condition = condition; 269 | conditionalBranch.label = this.internName(falseLabel, false); 270 | this.finalize(conditionalBranch); 271 | 272 | this.appendToBlock(falseBlock, conditionalBranch); 273 | this.appendToBlock(falseBlock, trueStatement); 274 | this.appendToBlock( 275 | falseBlock, 276 | this.makeBreakStatement(exitLabel) 277 | ); 278 | this.appendToBlock(mainBlock, falseStatement); 279 | 280 | this.finishBlock(falseBlock); 281 | this.finishBlock(mainBlock); 282 | this.finalize(falseBlock); 283 | this.finalize(mainBlock); 284 | 285 | return mainBlockParent; 286 | } else { 287 | return baseClass.makeIfStatement.call(this, condition, trueStatement, falseStatement); 288 | } 289 | }; 290 | 291 | AsmlikeJsonTreeBuilder.prototype.makeBreakStatement = function (label) { 292 | label = this.internName(label, false); 293 | 294 | return baseClass.makeBreakStatement.call(this, label); 295 | }; 296 | 297 | AsmlikeJsonTreeBuilder.prototype.makeContinueStatement = function (label) { 298 | label = this.internName(label, false); 299 | 300 | return baseClass.makeContinueStatement.call(this, label); 301 | }; 302 | 303 | AsmlikeJsonTreeBuilder.prototype.makeFunctionExpression = function (name, argumentNames, body) { 304 | // FIXME: The name should be interned in the outer scope 305 | name = this.internName(name, true); 306 | argumentNames = Array.prototype.slice.call(argumentNames); 307 | 308 | for (var i = 0; i < argumentNames.length; i++) 309 | argumentNames[i] = this.internName(argumentNames[i], false); 310 | 311 | return baseClass.makeFunctionExpression.call(this, name, argumentNames, body); 312 | }; 313 | 314 | AsmlikeJsonTreeBuilder.prototype.makePair = function (key, value) { 315 | // FIXME: Identifier vs string literal 316 | /* 317 | if (typeof (key) === "string") 318 | key = this.internName(key, true); 319 | */ 320 | 321 | return baseClass.makePair.call(this, key, value); 322 | }; 323 | 324 | AsmlikeJsonTreeBuilder.prototype.makeIdentifierExpression = function (identifier) { 325 | identifier = this.internName(identifier, false); 326 | 327 | return baseClass.makeIdentifierExpression.call(this, identifier); 328 | }; 329 | 330 | AsmlikeJsonTreeBuilder.prototype.makeMemberAccessExpression = function (lhs, memberName) { 331 | memberName = this.internName(memberName, true); 332 | 333 | return baseClass.makeMemberAccessExpression.call(this, lhs, memberName); 334 | }; 335 | 336 | AsmlikeJsonTreeBuilder.prototype.finalize = function (obj) { 337 | return obj; 338 | }; 339 | 340 | 341 | exports.Builder = AsmlikeJsonTreeBuilder; 342 | })); -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /third_party/cashew/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /parse/json-treebuilder.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | (function (root, factory) { 4 | if (typeof define === 'function' && define.amd) { 5 | define(['exports'], factory); 6 | } else if (typeof exports !== 'undefined') { 7 | factory(exports); 8 | } else { 9 | factory((root.jsonTreeBuilder = {})); 10 | } 11 | }(this, function (exports) { 12 | // If enabled, expressions are wrapped in ExpressionStatement nodes, making 13 | // it possible to satisfy Statement type requirements. 14 | // This increases the filesize of the binary representation significantly, though. 15 | var GenerateExpressionStatements = false; 16 | 17 | 18 | function JsonTreeBuilder () { 19 | this.protos = Object.create(null); 20 | }; 21 | 22 | JsonTreeBuilder.prototype.makeProto = function (type) { 23 | return new Object(); 24 | }; 25 | 26 | JsonTreeBuilder.prototype.make = function (type) { 27 | var proto = this.protos[type]; 28 | if (!proto) { 29 | this.protos[type] = proto = this.makeProto(type); 30 | proto.type = type; 31 | } 32 | 33 | var result = Object.create(proto); 34 | // HACK For debugging and JSON.stringify 35 | result.type = type; 36 | 37 | return result; 38 | }; 39 | 40 | JsonTreeBuilder.prototype.pushScope = function () { 41 | }; 42 | 43 | JsonTreeBuilder.prototype.popScope = function () { 44 | }; 45 | 46 | JsonTreeBuilder.prototype._makeBlock = function (typeTag) { 47 | this.pushScope(); 48 | 49 | var result = this.make(typeTag); 50 | result.statements = []; 51 | 52 | return result; 53 | }; 54 | 55 | JsonTreeBuilder.prototype.appendToBlock = function (block, statement) { 56 | block.statements.push(statement); 57 | }; 58 | 59 | // You must call this after you finish appending stuff to a block 60 | JsonTreeBuilder.prototype.finishBlock = function (block) { 61 | this.popScope(); 62 | 63 | return this.finalize(block); 64 | }; 65 | 66 | // Note that this isn't finalized yet. Use finishBlock. 67 | JsonTreeBuilder.prototype.makeTopLevelBlock = function () { 68 | return this._makeBlock("TopLevel"); 69 | }; 70 | 71 | // Note that this isn't finalized yet. Use finishBlock. 72 | JsonTreeBuilder.prototype.makeBlock = function () { 73 | return this._makeBlock("Block"); 74 | }; 75 | 76 | JsonTreeBuilder.prototype.finalize = function (obj) { 77 | return obj; 78 | }; 79 | 80 | JsonTreeBuilder.prototype.makeExpressionStatement = function (expression) { 81 | if ( 82 | !expression || 83 | (typeof (expression.type) !== "string") 84 | ) { 85 | console.log(expression); 86 | throw new Error("Expected an expression"); 87 | } else if (expression.type.indexOf("Statement") >= 0) { 88 | console.log(expression); 89 | throw new Error("Cannot wrap a statement in an expression statement"); 90 | } 91 | 92 | if (GenerateExpressionStatements) { 93 | var result = this.make("ExpressionStatement"); 94 | result.expression = expression; 95 | return this.finalize(result); 96 | } else { 97 | return expression; 98 | } 99 | }; 100 | 101 | JsonTreeBuilder.prototype.makeLabelStatement = function (labels, labelled) { 102 | var result; 103 | 104 | if (labels.length === 1) { 105 | result = this.make("LabelStatement"); 106 | result.label = labels[0]; 107 | } else if (labels.length === 0) { 108 | throw new Error("No labels"); 109 | } else { 110 | result = this.make("MultiLabelStatement"); 111 | result.labels = labels; 112 | } 113 | 114 | result.labelled = labelled; 115 | return this.finalize(result); 116 | }; 117 | 118 | JsonTreeBuilder.prototype.makeIfStatement = function (condition, trueStatement, falseStatement) { 119 | var result = this.make("IfStatement"); 120 | result.condition = condition; 121 | result.trueStatement = trueStatement; 122 | result.falseStatement = falseStatement; 123 | return this.finalize(result); 124 | }; 125 | 126 | JsonTreeBuilder.prototype.makeForStatement = function (initialize, update, condition, body) { 127 | var result = this.make("ForStatement"); 128 | result.initialize = initialize; 129 | result.update = update; 130 | result.condition = condition; 131 | result.body = body; 132 | return this.finalize(result); 133 | }; 134 | 135 | JsonTreeBuilder.prototype.makeForInStatement = function (declaration, body) { 136 | var result = this.make("ForInStatement"); 137 | result.declaration = declaration; 138 | result.body = body; 139 | return this.finalize(result); 140 | }; 141 | 142 | JsonTreeBuilder.prototype.makeWhileStatement = function (condition, body) { 143 | var result = this.make("WhileStatement"); 144 | result.condition = condition; 145 | result.body = body; 146 | return this.finalize(result); 147 | }; 148 | 149 | JsonTreeBuilder.prototype.makeDoWhileStatement = function (condition, body) { 150 | var result = this.make("DoWhileStatement"); 151 | result.condition = condition; 152 | result.body = body; 153 | return this.finalize(result); 154 | }; 155 | 156 | JsonTreeBuilder.prototype.makeNullStatement = function () { 157 | var result = this.make("NullStatement"); 158 | return this.finalize(result); 159 | }; 160 | 161 | JsonTreeBuilder.prototype.makeFunctionStatement = function (functionExpression) { 162 | var result = this.make("FunctionStatement"); 163 | result.functionExpression = functionExpression; 164 | return this.finalize(result); 165 | }; 166 | 167 | JsonTreeBuilder.prototype.makeDeclarationStatement = function (declarations) { 168 | var result = this.make("DeclarationStatement"); 169 | result.declarations = declarations; 170 | return this.finalize(result); 171 | }; 172 | 173 | JsonTreeBuilder.prototype.makeDeclaration = function (name, initialValue) { 174 | var result = this.make("Declaration"); 175 | result.name = name; 176 | result.initialValue = initialValue || null; 177 | return this.finalize(result); 178 | }; 179 | 180 | JsonTreeBuilder.prototype.makeForInDeclaration = function (variableName, sequenceExpression) { 181 | var result = this.make("ForInDeclaration"); 182 | result.variableName = variableName; 183 | result.sequenceExpression = sequenceExpression; 184 | return this.finalize(result); 185 | }; 186 | 187 | JsonTreeBuilder.prototype.makeReturnStatement = function (expression) { 188 | var result = this.make("ReturnStatement"); 189 | result.expression = expression; 190 | return this.finalize(result); 191 | }; 192 | 193 | JsonTreeBuilder.prototype.makeBreakStatement = function (label) { 194 | var result = this.make("BreakStatement"); 195 | result.label = label; 196 | return this.finalize(result); 197 | }; 198 | 199 | JsonTreeBuilder.prototype.makeContinueStatement = function (label) { 200 | var result = this.make("ContinueStatement"); 201 | result.label = label; 202 | return this.finalize(result); 203 | }; 204 | 205 | JsonTreeBuilder.prototype.makeThrowStatement = function (expression) { 206 | var result = this.make("ThrowStatement"); 207 | result.expression = expression; 208 | return this.finalize(result); 209 | }; 210 | 211 | JsonTreeBuilder.prototype.makeSwitchStatement = function (value, cases) { 212 | var result = this.make("SwitchStatement"); 213 | result.value = value; 214 | result.cases = cases; 215 | return this.finalize(result); 216 | }; 217 | 218 | JsonTreeBuilder.prototype.makeSwitchCase = function (value, body) { 219 | var result = this.make("SwitchCase"); 220 | result.value = value; 221 | result.body = body; 222 | return this.finalize(result); 223 | }; 224 | 225 | JsonTreeBuilder.prototype.makeTryStatement = function (body, catchExpression, catchBlock, finallyBlock) { 226 | var result = this.make("TryStatement"); 227 | result.body = body; 228 | result.catchExpression = catchExpression; 229 | result.catchBlock = catchBlock; 230 | result.finallyBlock = finallyBlock; 231 | return this.finalize(result); 232 | }; 233 | 234 | JsonTreeBuilder.prototype.makeCommaExpression = function (expressions) { 235 | var result = this.make("Comma"); 236 | result.expressions = expressions; 237 | return this.finalize(result); 238 | }; 239 | 240 | JsonTreeBuilder.prototype.makeFunctionExpression = function (name, argumentNames, body) { 241 | var result = this.make("Function"); 242 | result.name = name; 243 | result.argumentNames = argumentNames; 244 | result.body = body; 245 | return this.finalize(result); 246 | }; 247 | 248 | JsonTreeBuilder.prototype.makeRegExpLiteralExpression = function (pattern, flags) { 249 | var result = this.make("RegExpLiteral"); 250 | result.pattern = pattern; 251 | result.flags = flags; 252 | return this.finalize(result); 253 | }; 254 | 255 | JsonTreeBuilder.prototype.makeLiteralExpression = function (type, value) { 256 | var titleCaseType = type[0].toUpperCase() + type.substr(1); 257 | if (type === "regexp") 258 | throw new Error("Invalid"); 259 | 260 | var result = this.make(titleCaseType + "Literal"); 261 | result.value = value; 262 | return this.finalize(result); 263 | }; 264 | 265 | JsonTreeBuilder.prototype.makeArrayLiteralExpression = function (elements) { 266 | var result = this.make("ArrayLiteral"); 267 | result.elements = elements; 268 | return this.finalize(result); 269 | }; 270 | 271 | JsonTreeBuilder.prototype.makeObjectLiteralExpression = function (pairs) { 272 | var result = this.make("ObjectLiteral"); 273 | result.pairs = pairs; 274 | return this.finalize(result); 275 | }; 276 | 277 | JsonTreeBuilder.prototype.makePair = function (key, value) { 278 | var keyType = typeof(key); 279 | keyType = keyType[0].toUpperCase() + keyType.substr(1); 280 | 281 | var result = this.make(keyType + "Pair"); 282 | result.key = key; 283 | result.value = value; 284 | return this.finalize(result); 285 | }; 286 | 287 | JsonTreeBuilder.prototype.makeIdentifierExpression = function (identifier) { 288 | var result = this.make("Identifier"); 289 | result.identifier = identifier; 290 | return this.finalize(result); 291 | }; 292 | 293 | JsonTreeBuilder.prototype.makePrefixMutationExpression = function (operator, rhs) { 294 | var result = this.make("PrefixMutation"); 295 | result.operator = operator; 296 | result.rhs = rhs; 297 | return this.finalize(result); 298 | }; 299 | 300 | JsonTreeBuilder.prototype.makePostfixMutationExpression = function (operator, lhs) { 301 | var result = this.make("PostfixMutation"); 302 | result.operator = operator; 303 | result.lhs = lhs; 304 | return this.finalize(result); 305 | }; 306 | 307 | JsonTreeBuilder.prototype.makeUnaryOperatorExpression = function (operator, rhs) { 308 | var result = this.make("UnaryOperator"); 309 | result.operator = operator; 310 | result.rhs = rhs; 311 | return this.finalize(result); 312 | }; 313 | 314 | JsonTreeBuilder.prototype.makeBinaryOperatorExpression = function (operator, lhs, rhs) { 315 | var result = this.make("BinaryOperator"); 316 | result.operator = operator; 317 | result.lhs = lhs; 318 | result.rhs = rhs; 319 | return this.finalize(result); 320 | }; 321 | 322 | JsonTreeBuilder.prototype.makeAssignmentOperatorExpression = function (operator, lhs, rhs) { 323 | var result = this.make("AssignmentOperator"); 324 | result.operator = operator; 325 | result.lhs = lhs; 326 | result.rhs = rhs; 327 | return this.finalize(result); 328 | }; 329 | 330 | JsonTreeBuilder.prototype.makeComputedMemberAccessExpression = function (lhs, rhs) { 331 | var result = this.make("ComputedMemberAccess"); 332 | result.lhs = lhs; 333 | result.rhs = rhs; 334 | return this.finalize(result); 335 | }; 336 | 337 | JsonTreeBuilder.prototype.makeTernaryOperatorExpression = function (condition, trueExpression, falseExpression) { 338 | var result = this.make("TernaryOperator"); 339 | result.condition = condition; 340 | result.trueExpression = trueExpression; 341 | result.falseExpression = falseExpression; 342 | return this.finalize(result); 343 | }; 344 | 345 | JsonTreeBuilder.prototype.makeMemberAccessExpression = function (lhs, memberName) { 346 | var result = this.make("MemberAccess"); 347 | result.lhs = lhs; 348 | result.memberName = memberName; 349 | return this.finalize(result); 350 | }; 351 | 352 | JsonTreeBuilder.prototype.makeInvocationExpression = function (callee, argumentValues) { 353 | var result = this.make("Invocation"); 354 | result.callee = callee; 355 | result.argumentValues = argumentValues; 356 | return this.finalize(result); 357 | }; 358 | 359 | 360 | exports.Builder = JsonTreeBuilder; 361 | })); -------------------------------------------------------------------------------- /third_party/cashew/asm-tokenizer.js: -------------------------------------------------------------------------------- 1 | // Partially based on cashew asm.js parser (see Upstream/cashew/LICENSE) 2 | 3 | 'use strict'; 4 | 5 | (function (root, factory) { 6 | if (typeof define === 'function' && define.amd) { 7 | define(['exports'], factory); 8 | } else if (typeof exports !== 'undefined') { 9 | factory(exports); 10 | } else { 11 | factory((root.asmTokenizer = {})); 12 | } 13 | }(this, function (exports) { 14 | var Keywords = "break do in typeof " + 15 | "case else instanceof var " + 16 | "catch export new void " + 17 | "class extends return while " + 18 | "const finally super with " + 19 | "continue for switch yield " + 20 | "debugger function " /* this " */ + 21 | "default if throw " + 22 | "delete import try " + 23 | "enum await " + 24 | "implements package protected " + 25 | "interface private public " + 26 | "true false"; 27 | 28 | var KeywordLookup = Object.create(null); 29 | Keywords.split(" ").forEach(function (kw) { KeywordLookup[kw] = true; }); 30 | 31 | var KeywordOperators = { 32 | "delete": true, 33 | "void": true, 34 | "typeof": true, 35 | "new": true, 36 | "in": true, 37 | "instanceof": true 38 | }; 39 | 40 | var _0 = "0".charCodeAt(0), _9 = "9".charCodeAt(0); 41 | var _a = "a".charCodeAt(0), _z = "z".charCodeAt(0); 42 | var _A = "A".charCodeAt(0), _Z = "Z".charCodeAt(0); 43 | var __ = "_".charCodeAt(0), _$ = "$".charCodeAt(0); 44 | 45 | var ForwardSlash = "/".charCodeAt(0); 46 | var BackSlash = "\\".charCodeAt(0); 47 | var Asterisk = "*".charCodeAt(0); 48 | var DoubleQuote = "\"".charCodeAt(0); 49 | var SingleQuote = "\'".charCodeAt(0); 50 | var Period = ".".charCodeAt(0); 51 | var LessThan = "<".charCodeAt(0); 52 | var GreaterThan = ">".charCodeAt(0); 53 | var Equal = "=".charCodeAt(0); 54 | var Minus = "-".charCodeAt(0); 55 | var Plus = "+".charCodeAt(0); 56 | var Exclamation = "!".charCodeAt(0); 57 | var Ampersand = "&".charCodeAt(0); 58 | var Pipe = "|".charCodeAt(0); 59 | 60 | var Tab = 9, CR = 10, LF = 13, Space = 32; 61 | 62 | var DigitChars = Array.prototype.slice.call("0123456789abcdefxABCDEFX.") 63 | .map(function (ch) { return ch.charCodeAt(0); }); 64 | 65 | var OperatorInitialChars = Array.prototype.slice.call("!%&*+,-./:<=>?^|~") 66 | .map(function (ch) { return ch.charCodeAt(0); }); 67 | 68 | var MutationAssignmentChars = Array.prototype.slice.call("!%&*+-/^|~") 69 | .map(function (ch) { return ch.charCodeAt(0); }); 70 | 71 | var Separators = Array.prototype.slice.call("([];{})") 72 | .map(function (ch) { return ch.charCodeAt(0); }); 73 | 74 | function isWhitespace (ch) { 75 | return (ch === Space) || 76 | (ch === Tab) || 77 | (ch === CR) || 78 | (ch === LF); 79 | }; 80 | 81 | function isDigit (ch) { 82 | return (ch >= _0) && (ch <= _9); 83 | }; 84 | 85 | function is32Bit (x) { 86 | return (x === (x | 0)) /* || (x === (x >>> 0)) */; 87 | }; 88 | 89 | function isIdentifierPrefix (ch) { 90 | return ((ch >= _a) && (ch <= _z)) || 91 | ((ch >= _A) && (ch <= _Z)) || 92 | (ch === __) || 93 | (ch === _$); 94 | }; 95 | 96 | function isIdentifierBody (ch) { 97 | return isIdentifierPrefix(ch) || 98 | ((ch >= _0) && (ch <= _9)); 99 | } 100 | 101 | // Devour whitespace & comments from the reader 102 | function skipDeadSpace (reader) { 103 | while (!reader.eof) { 104 | var ch = reader.peek(0); 105 | 106 | if (isWhitespace(ch)) { 107 | reader.read(); 108 | continue; 109 | } else if (ch === ForwardSlash) { 110 | var ch2 = reader.peek(1); 111 | 112 | if (ch2 === ForwardSlash) { 113 | // Greedily parse single-line comment 114 | reader.skip(2); 115 | 116 | while ( 117 | !reader.eof && 118 | (ch !== CR) && 119 | (ch !== LF) 120 | ) { 121 | ch = reader.read(); 122 | }; 123 | 124 | continue; 125 | } else if (ch2 === Asterisk) { 126 | reader.skip(2); 127 | 128 | while ( 129 | !reader.eof && 130 | ( 131 | (ch !== Asterisk) || 132 | (ch2 !== ForwardSlash) 133 | ) 134 | ) { 135 | ch = reader.read(); 136 | ch2 = reader.peek(0); 137 | }; 138 | 139 | reader.read(); 140 | continue; 141 | } 142 | } 143 | 144 | break; 145 | } 146 | }; 147 | 148 | 149 | function Token (type, value) { 150 | if (arguments.length !== 2) 151 | throw new Error("Expected (type, value)"); 152 | else if (!type) 153 | throw new Error("Expected type"); 154 | 155 | this.type = type; 156 | this.value = value; 157 | }; 158 | 159 | 160 | // parses an input character stream into a stream of tokens 161 | // input is a ByteReader (see encoding.js) 162 | function Tokenizer (input) { 163 | this.reader = input; 164 | this._previous = null; 165 | }; 166 | 167 | Tokenizer.prototype.getPosition = function () { 168 | return this.reader.getPosition(); 169 | } 170 | 171 | Tokenizer.prototype.makeResult = function (type, value) { 172 | var result = new Token(type, value); 173 | // HACK 174 | this._previous = result; 175 | return result; 176 | }; 177 | 178 | Tokenizer.prototype.assert = function (cond) { 179 | if (!cond) 180 | throw new Error("Assertion failed"); 181 | }; 182 | 183 | Tokenizer.prototype.getPrevious = function () { 184 | return this._previous; 185 | } 186 | 187 | // Reads a single token from the stream. 188 | // Return value is reused between calls, so deep-copy it if you wish to retain it 189 | Tokenizer.prototype.read = function () { 190 | skipDeadSpace(this.reader); 191 | 192 | if (this.reader.eof) 193 | return false; 194 | 195 | var ch = this.reader.peek(0), 196 | ch2 = this.reader.peek(1); 197 | 198 | this.assert(!isWhitespace(ch)); 199 | 200 | if (isIdentifierPrefix(ch)) { 201 | return this.readIdentifier(ch); 202 | } else if ( 203 | (ch === SingleQuote) || 204 | (ch === DoubleQuote) 205 | ) { 206 | return this.readStringLiteral(ch); 207 | } else if ( 208 | isDigit(ch) || 209 | ((ch === Period) && isDigit(ch2)) 210 | ) { 211 | return this.readNumberLiteral(ch, ch2); 212 | } else if ( 213 | OperatorInitialChars.indexOf(ch) >= 0 214 | ) { 215 | return this.readOperator(ch, ch2); 216 | } else if ( 217 | Separators.indexOf(ch) >= 0 218 | ) { 219 | return this.readSeparator(ch); 220 | } else { 221 | var ch$2 = this.reader.peek(-2), 222 | ch$1 = this.reader.peek(-1), 223 | ch3 = this.reader.peek(2); 224 | 225 | console.log( 226 | "Initial character not implemented: '" + String.fromCharCode(ch) + 227 | "' (" + ch + ") at offset " + 228 | this.reader.getPosition() + ", surrounding: '" + 229 | String.fromCharCode(ch$2, ch$1, ch, ch2, ch3) + "'" 230 | ); 231 | return false; 232 | } 233 | 234 | return false; 235 | }; 236 | 237 | Tokenizer.prototype.readOperator = function (ch, ch2) { 238 | var length = 1; 239 | var ch3 = this.reader.peek(2), ch4 = this.reader.peek(3); 240 | 241 | switch (ch) { 242 | case LessThan: 243 | if (ch2 === LessThan) { 244 | if (ch3 === Equal) { 245 | length = 3; 246 | } else { 247 | length = 2; 248 | } 249 | 250 | } else if (ch2 === Equal) { 251 | length = 2; 252 | } else { 253 | length = 1; 254 | } 255 | 256 | break; 257 | 258 | case GreaterThan: 259 | if (ch2 === GreaterThan) { 260 | if (ch3 === Equal) { 261 | length = 3; 262 | 263 | } else if (ch3 === GreaterThan) { 264 | if (ch4 === Equal) { 265 | length = 4; 266 | } else { 267 | length = 3; 268 | } 269 | 270 | } else { 271 | length = 2; 272 | } 273 | 274 | } else if (ch2 === Equal) { 275 | length = 2; 276 | } else { 277 | length = 1; 278 | } 279 | 280 | break; 281 | 282 | case Equal: 283 | length = ( 284 | (ch2 === Equal) && 285 | (ch3 === Equal) 286 | ) ? 3 : (ch2 === Equal) ? 2 : 1; 287 | 288 | break; 289 | 290 | case Minus: 291 | length = ( 292 | (ch2 === Minus) || 293 | (ch2 === Equal) 294 | ) ? 2 : 1; 295 | 296 | break; 297 | 298 | case Plus: 299 | length = ( 300 | (ch2 === Plus) || 301 | (ch2 === Equal) 302 | ) ? 2 : 1; 303 | 304 | break; 305 | 306 | case Ampersand: 307 | length = ( 308 | (ch2 === Ampersand) || 309 | (ch2 === Equal) 310 | ) ? 2 : 1; 311 | 312 | break; 313 | 314 | case Pipe: 315 | length = ( 316 | (ch2 === Pipe) || 317 | (ch2 === Equal) 318 | ) ? 2 : 1; 319 | 320 | break; 321 | 322 | case Exclamation: 323 | length = ( 324 | (ch2 === Equal) && 325 | (ch3 === Equal) 326 | ) ? 3 : (ch2 === Equal) ? 2 : 1; 327 | 328 | break; 329 | 330 | case ForwardSlash: 331 | // HACK: Heuristically figure out whether this is a regexp. UGH 332 | if ( 333 | (this._previous.type === "operator") || 334 | ( 335 | (this._previous.type === "separator") && 336 | (this._previous.value !== ")") && 337 | (this._previous.value !== "]") 338 | ) 339 | ) { 340 | return this.readRegExpLiteral(); 341 | } 342 | 343 | // Fall through 344 | 345 | default: 346 | if ( 347 | (MutationAssignmentChars.indexOf(ch) >= 0) && 348 | (ch2 === Equal) 349 | ) 350 | length = 2; 351 | 352 | break; 353 | } 354 | 355 | var text = String.fromCharCode(ch); 356 | 357 | for (var i = 1; i < length; i++) 358 | text += String.fromCharCode(this.reader.peek(i)); 359 | 360 | this.reader.skip(length); 361 | 362 | return this.makeResult("operator", text); 363 | }; 364 | 365 | Tokenizer.prototype.readIdentifier = function (ch) { 366 | var temp = String.fromCharCode(ch); 367 | this.reader.skip(1); 368 | 369 | while ((ch = this.reader.peek(0)) && isIdentifierBody(ch)) { 370 | temp += String.fromCharCode(ch); 371 | this.reader.skip(1); 372 | } 373 | 374 | var typeString = (KeywordOperators[temp] === true) 375 | ? "operator" 376 | : (KeywordLookup[temp] === true) 377 | ? "keyword" 378 | : "identifier"; 379 | 380 | return this.makeResult(typeString, temp); 381 | }; 382 | 383 | Tokenizer.prototype.readStringLiteral = function (quote) { 384 | var result = ""; 385 | var ch; 386 | 387 | this.reader.skip(1); 388 | 389 | while (((ch = this.reader.read()) !== quote) && ch) { 390 | result += String.fromCharCode(ch); 391 | 392 | // HACK: Ensure \' and \" are read in their entirety 393 | // TODO: Actually parse out the escape sequences into regular chars? 394 | if (ch === BackSlash) { 395 | ch = this.reader.read(); 396 | result += String.fromCharCode(ch); 397 | } 398 | } 399 | 400 | return this.makeResult("string", result); 401 | }; 402 | 403 | Tokenizer.prototype.readNumberLiteral = function (ch, ch2) { 404 | // UGH 405 | var temp = ""; 406 | var isDouble = false; 407 | for (var i = 0; i < 16; i++) { 408 | var ch = this.reader.peek(i); 409 | 410 | if (ch === false) 411 | break; 412 | else if (DigitChars.indexOf(ch) < 0) 413 | break; 414 | 415 | if (ch === Period) 416 | isDouble = true; 417 | 418 | temp += String.fromCharCode(ch); 419 | } 420 | 421 | var value; 422 | 423 | if (!isDouble) { 424 | value = parseInt(temp); 425 | } else { 426 | value = parseFloat(temp); 427 | } 428 | 429 | this.reader.skip(temp.length); 430 | 431 | if (is32Bit(value) && !isDouble) 432 | return this.makeResult("integer", value); 433 | else 434 | return this.makeResult("double", value); 435 | }; 436 | 437 | Tokenizer.prototype.readRegExpLiteral = function () { 438 | var ch; 439 | 440 | // Skip opening / 441 | this.reader.skip(1); 442 | 443 | // Read body of the regexp pattern 444 | var body = ""; 445 | while (((ch = this.reader.read()) !== ForwardSlash) && ch) { 446 | if (ch === BackSlash) { 447 | // Read escaped character 448 | // FIXME: \x00 and \u0000 449 | 450 | body += "\\"; 451 | ch = this.reader.read(); 452 | } 453 | 454 | body += String.fromCharCode(ch); 455 | } 456 | 457 | // Read regexp flags 458 | // FIXME: Is this right? 459 | var flags = ""; 460 | while ((ch = this.reader.peek()) !== false) { 461 | if ( 462 | ((ch >= _a) && (ch <= _z)) || 463 | ((ch >= _A) && (ch <= _Z)) 464 | ) { 465 | flags += String.fromCharCode(ch); 466 | this.reader.read(); 467 | } else 468 | break; 469 | } 470 | 471 | try { 472 | var result = this.makeResult("regexp", new RegExp(body, flags)); 473 | } catch (exc) { 474 | console.log("Expression body was " + JSON.stringify(body) + ", flags were " + JSON.stringify(flags)); 475 | throw exc; 476 | } 477 | 478 | return result; 479 | }; 480 | 481 | Tokenizer.prototype.readSeparator = function (ch) { 482 | this.reader.skip(1); 483 | 484 | return this.makeResult("separator", String.fromCharCode(ch)); 485 | }; 486 | 487 | 488 | exports.Tokenizer = Tokenizer; 489 | 490 | exports.skipDeadSpace = skipDeadSpace; 491 | exports.isIdentifierPrefix = isIdentifierPrefix; 492 | exports.isIdentifierBody = isIdentifierBody; 493 | })); -------------------------------------------------------------------------------- /ast-common.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | (function (root, factory) { 4 | if (typeof define === 'function' && define.amd) { 5 | define(['exports'], factory); 6 | } else if (typeof exports !== 'undefined') { 7 | factory(exports); 8 | } else { 9 | factory((root.astCommon = {})); 10 | } 11 | }(this, function (exports) { 12 | function NamedTableId (entry) { 13 | if (!entry) 14 | throw new Error("Id must have an entry"); 15 | 16 | this.entry = entry; 17 | this.isRedirected = false; 18 | }; 19 | 20 | NamedTableId.prototype.equals = function (rhs) { 21 | return this.entry === rhs.entry; 22 | }; 23 | 24 | NamedTableId.prototype.checkInvariants = function () { 25 | if (this.entry.isInvalidated) { 26 | try { 27 | var dupe = Object.create(null); 28 | for (var k in this.entry) { 29 | if (k === "id") 30 | continue; 31 | 32 | dupe[k] = this.entry[k]; 33 | } 34 | 35 | console.log("Invalidated entry", dupe); 36 | } catch (ex) { 37 | console.log(ex); 38 | } 39 | 40 | throw new Error("Invalidated entry"); 41 | } 42 | }; 43 | 44 | NamedTableId.prototype.get_semantic = function () { 45 | this.checkInvariants(); 46 | 47 | return this.entry.table.semantic; 48 | }; 49 | 50 | NamedTableId.prototype.get_name = function () { 51 | this.checkInvariants(); 52 | 53 | return this.entry.name; 54 | }; 55 | 56 | NamedTableId.prototype.get_value = function () { 57 | this.checkInvariants(); 58 | 59 | return this.entry.value; 60 | }; 61 | 62 | NamedTableId.prototype.get_index = function () { 63 | this.checkInvariants(); 64 | 65 | return this.entry.index; 66 | }; 67 | 68 | NamedTableId.prototype.get_hit_count = function () { 69 | this.checkInvariants(); 70 | 71 | return this.entry.hitCount; 72 | }; 73 | 74 | NamedTableId.prototype.set_hit_count = function (value) { 75 | this.checkInvariants(); 76 | 77 | this.entry.hitCount = value | 0; 78 | }; 79 | 80 | NamedTableId.prototype.is_omitted = function () { 81 | this.checkInvariants(); 82 | 83 | return this.entry.isOmitted; 84 | }; 85 | 86 | NamedTableId.prototype.get_global_index = function () { 87 | this.checkInvariants(); 88 | 89 | if (typeof (this.entry.table.globalBaseIndex) !== "number") 90 | throw new Error("No base index assigned to table " + this.entry.table.semantic); 91 | 92 | return this.entry.table.globalBaseIndex + this.entry.index; 93 | }; 94 | 95 | NamedTableId.prototype.toString = function () { 96 | var index = this.get_index(); 97 | var name = this.get_name(); 98 | var prefix = "<" + this.entry.table.semantic + " #"; 99 | 100 | if (typeof (index) !== "number") 101 | index = "?"; 102 | else if (index === name) 103 | return prefix + index + ">"; 104 | else 105 | return prefix + index + " '" + name + "'>"; 106 | } 107 | 108 | 109 | function NamedTableEntry (name, value, table) { 110 | this.table = table; 111 | 112 | this.name = name; 113 | this.value = value; 114 | this.index = undefined; 115 | this.order = undefined; 116 | this.hitCount = 1; 117 | this.isOmitted = false; 118 | 119 | this.id = new NamedTableId(this); 120 | }; 121 | 122 | 123 | function NamedTable (semantic) { 124 | if (!semantic) 125 | throw new Error("Semantic name required"); 126 | 127 | this.entries = Object.create(null); 128 | this.count = 0; 129 | this.nextOrder = 0; 130 | this.semantic = semantic; 131 | this.isFinalized = false; 132 | this.globalBaseIndex = null; 133 | }; 134 | 135 | NamedTable.prototype.add = function (name, value, throwOnDivergence) { 136 | if (this.isFinalized) 137 | throw new Error("Table already finalized"); 138 | 139 | var existing = this.entries[name]; 140 | 141 | if (typeof (existing) !== "undefined") { 142 | if ( 143 | (throwOnDivergence !== false) && 144 | (existing.value !== value) 145 | ) 146 | throw new Error("A different value already exists with this name"); 147 | 148 | existing.hitCount += 1; 149 | return existing.id; 150 | } 151 | 152 | var entry = new NamedTableEntry(name, value, this); 153 | entry.order = this.nextOrder++; 154 | 155 | this.count++; 156 | this.entries[name] = entry; 157 | return entry.id; 158 | }; 159 | 160 | NamedTable.prototype.get = function (name) { 161 | var entry = this.entries[name]; 162 | 163 | if (entry) 164 | return entry.value; 165 | else 166 | return; 167 | } 168 | 169 | NamedTable.prototype.get_id = function (name) { 170 | var entry = this.entries[name]; 171 | 172 | if (entry) 173 | return entry.id; 174 | else 175 | return; 176 | } 177 | 178 | NamedTable.prototype.get_index = function (name) { 179 | var entry = this.entries[name]; 180 | 181 | if (entry) { 182 | if (typeof (entry.index) === "number") 183 | return entry.index; 184 | else 185 | throw new Error("Table not finalized"); 186 | } else { 187 | throw new Error("No '" + this.semantic + "' table entry for '" + name + "'"); 188 | } 189 | } 190 | 191 | NamedTable.prototype.get_global_index = function (name) { 192 | if (typeof (this.globalBaseIndex) !== "number") 193 | throw new Error("No base index set"); 194 | 195 | return this.get_index(name) + this.globalBaseIndex; 196 | } 197 | 198 | NamedTable.prototype.get_count = function () { 199 | return this.count; 200 | }; 201 | 202 | NamedTable.prototype.forEach = function (callback) { 203 | for (var k in this.entries) { 204 | var entry = this.entries[k]; 205 | 206 | // Skip over dedupe sources 207 | if (entry.isOmitted) 208 | continue; 209 | 210 | callback(entry.id); 211 | } 212 | }; 213 | 214 | // Makes source's table entry a copy of target's. 215 | NamedTable.prototype.omit = function (value) { 216 | var entry; 217 | 218 | if ( 219 | value instanceof NamedTableId 220 | ) 221 | entry = value.entry; 222 | else 223 | entry = this.entries[value]; 224 | 225 | if (!entry) 226 | throw new Error("value must be in the table"); 227 | 228 | if (entry.isOmitted) 229 | return; 230 | 231 | entry.isOmitted = true; 232 | this.count -= 1; 233 | }; 234 | 235 | NamedTable.prototype.finalize = function (baseIndex) { 236 | var result = new Array(this.count); 237 | var i = 0; 238 | 239 | baseIndex |= 0; 240 | 241 | this.forEach(function (id) { 242 | result[i++] = id; 243 | }); 244 | 245 | if (result.length !== this.count) { 246 | console.log(result.length, this.count); 247 | throw new Error("Count mismatch"); 248 | } 249 | 250 | if (result.length === 0) { 251 | this.isFinalized = true; 252 | return result; 253 | } 254 | 255 | if (exports.SortTables) { 256 | // First pass: sort by usage count so most frequently 257 | // used objects have low (small) indices. 258 | var hitCountPredicate = function (lhs, rhs) { 259 | return (rhs.entry.hitCount - lhs.entry.hitCount); 260 | }; 261 | 262 | result.sort(hitCountPredicate); 263 | 264 | if (exports.LocalityAwareSorting) { 265 | // Second pass: Sort objects below a usage count threshold 266 | // by their ordering in the source file to increase locality. 267 | 268 | var isLargeTable = 269 | this.count >= exports.LargeTableThreshold; 270 | 271 | var cutoff = 272 | isLargeTable 273 | ? exports.LocalityCutoffLarge 274 | : exports.LocalityCutoffSmall; 275 | if (cutoff < 0) 276 | cutoff = 0; 277 | 278 | var thresholdIndex = Math.min( 279 | result.length - 1, 280 | cutoff - 1 281 | ); 282 | 283 | var hitCountThreshold = result[thresholdIndex].entry.hitCount; 284 | if (hitCountThreshold < exports.LocalityMinimumThreshold) 285 | hitCountThreshold = exports.LocalityMinimumThreshold; 286 | 287 | var orderPredicate = function (lhs, rhs) { 288 | return (lhs.entry.order - rhs.entry.order); 289 | }; 290 | 291 | result.sort(function (lhs, rhs) { 292 | var lhsBelowThreshold = lhs.entry.hitCount < hitCountThreshold; 293 | var rhsBelowThreshold = rhs.entry.hitCount < hitCountThreshold; 294 | 295 | if (lhsBelowThreshold && rhsBelowThreshold) 296 | return orderPredicate(lhs, rhs); 297 | else if (lhsBelowThreshold) 298 | return 1; 299 | else if (rhsBelowThreshold) 300 | return -1; 301 | else 302 | return hitCountPredicate(lhs, rhs); 303 | }); 304 | 305 | if (this.semantic === "object") { 306 | if (false) { 307 | console.log("object table threshold (index=" + cutoff + ") = " + hitCountThreshold); 308 | 309 | for (var i = 0; i < result.length; i++) { 310 | console.log(i, result[i].entry.hitCount, result[i].entry.order); 311 | } 312 | } 313 | } 314 | } 315 | } 316 | 317 | if (this.isFinalized) 318 | return result; 319 | 320 | for (i = 0; i < result.length; i++) { 321 | result[i].entry.index = baseIndex + i; 322 | 323 | if (exports.LogTables) 324 | console.log(this.semantic, result[i].get_name(), result[i].entry.hitCount, result[i].entry.index); 325 | } 326 | 327 | this.isFinalized = true; 328 | 329 | return result; 330 | }; 331 | 332 | NamedTable.prototype.setGlobalBaseIndex = function (value) { 333 | this.globalBaseIndex = value | 0; 334 | }; 335 | 336 | 337 | function UniqueTable (nameFromValue, semantic) { 338 | if (typeof (nameFromValue) !== "function") 339 | throw new Error("Name provider required"); 340 | else 341 | this.nameFromValue = nameFromValue; 342 | 343 | NamedTable.call(this, semantic); 344 | }; 345 | 346 | UniqueTable.prototype = Object.create(NamedTable.prototype); 347 | 348 | UniqueTable.prototype.add = function (value) { 349 | var name = this.nameFromValue(value); 350 | return NamedTable.prototype.add.call(this, name, value); 351 | }; 352 | 353 | // No-op 354 | UniqueTable.prototype.get = function (value) { 355 | return value; 356 | }; 357 | 358 | UniqueTable.prototype.get_id = function (value) { 359 | var name = this.nameFromValue(value); 360 | return NamedTable.prototype.get_id.call(this, name); 361 | }; 362 | 363 | UniqueTable.prototype.get_index = function (value) { 364 | var name = this.nameFromValue(value); 365 | return NamedTable.prototype.get_index.call(this, name); 366 | }; 367 | 368 | // Makes source's table entry a copy of target's. 369 | UniqueTable.prototype.dedupe = function (source, target) { 370 | var sourceName, targetName; 371 | 372 | if (source instanceof NamedTableId) 373 | sourceName = source; 374 | else 375 | sourceName = this.nameFromValue(source); 376 | 377 | if (target instanceof NamedTableId) 378 | targetName = target; 379 | else 380 | targetName = this.nameFromValue(target); 381 | 382 | return NamedTable.prototype.dedupe.call(this, sourceName, targetName); 383 | }; 384 | 385 | 386 | function StringTable (semantic) { 387 | UniqueTable.call(this, function (s) { 388 | if (typeof (s) !== "string") 389 | throw new Error("StringTable entries must be strings"); 390 | else 391 | return s; 392 | }, semantic); 393 | }; 394 | 395 | StringTable.prototype = Object.create(UniqueTable.prototype); 396 | 397 | 398 | var GetObjectId_table = new WeakMap(); 399 | var GetObjectId_nextId = 0; 400 | 401 | function NextObjectId () { 402 | return GetObjectId_nextId++; 403 | }; 404 | 405 | function GetObjectId (obj) { 406 | if (typeof (obj.__id__) === "number") 407 | return obj.__id__; 408 | 409 | if (typeof (obj) !== "object") 410 | throw new Error("GetObjectId expected object, got '" + typeof (obj) + "'"); 411 | else if (obj === null) 412 | // HACK 413 | return -1; 414 | 415 | var existing = GetObjectId_table.get(obj); 416 | if (typeof (existing) === "number") 417 | return existing; 418 | 419 | var result = NextObjectId(); 420 | GetObjectId_table.set(obj, result); 421 | 422 | return result; 423 | }; 424 | 425 | 426 | function ObjectTable (semantic) { 427 | UniqueTable.call(this, GetObjectId, semantic); 428 | }; 429 | 430 | ObjectTable.prototype = Object.create(UniqueTable.prototype); 431 | 432 | 433 | function ShapeTable (shapeKey) { 434 | this.shapeKey = shapeKey; 435 | 436 | NamedTable.call(this, "Shape"); 437 | }; 438 | 439 | ShapeTable.fromJson = function (json) { 440 | var parsed = JSON.parse(json); 441 | 442 | var result = new ShapeTable(parsed.shapeKey); 443 | 444 | for (var k in parsed.shapes) { 445 | var definition = new ShapeDefinition(k); 446 | var fields = parsed.shapes[k]; 447 | 448 | for (var j in fields) { 449 | var fieldType = fields[j]; 450 | 451 | var isOptional = false; 452 | if (!Array.isArray(fieldType)) { 453 | isOptional = (fieldType.indexOf("?") >= 0); 454 | fieldType = fieldType.replace("?", ""); 455 | } 456 | 457 | var fd = new FieldDefinition(j, fieldType, isOptional); 458 | definition.fields.push(fd); 459 | } 460 | 461 | result.add(k, definition); 462 | } 463 | 464 | return result; 465 | }; 466 | 467 | ShapeTable.prototype = Object.create(NamedTable.prototype); 468 | 469 | 470 | function FieldDefinition (name, type, optional) { 471 | this.name = name; 472 | this.type = type; 473 | this.optional = optional; 474 | }; 475 | 476 | 477 | function ShapeDefinition (key) { 478 | this.key = key; 479 | this.fields = []; 480 | }; 481 | 482 | 483 | var nags = Object.create(null); 484 | 485 | function pickTagForField (field, getTableForTypeTag) { 486 | var declaredType = field.type; 487 | if (Array.isArray(declaredType)) 488 | declaredType = "array"; 489 | 490 | if ( 491 | (declaredType !== "any") && 492 | (declaredType !== "object") && 493 | (declaredType !== "string") && 494 | (declaredType !== "symbol") && 495 | (declaredType !== "array") && 496 | !exports.TagIsPrimitive[declaredType] 497 | ) { 498 | var table = getTableForTypeTag(declaredType); 499 | 500 | if (!table) { 501 | // HACK: Type (virtual base?) without shape 502 | if (false && !nags[declaredType]) { 503 | nags[declaredType] = true; 504 | console.log("'object' fallback for " + declaredType); 505 | } 506 | 507 | declaredType = "object"; 508 | } 509 | } 510 | 511 | return declaredType; 512 | }; 513 | 514 | 515 | function writeLEBUint32 (byteWriter, value) { 516 | var v = value; 517 | 518 | var b = 0; 519 | value |= 0; 520 | 521 | do { 522 | b = value & 0x7F; 523 | value >>>= 7; 524 | if (value) 525 | b |= 0x80; 526 | 527 | byteWriter.write(b); 528 | } while (value); 529 | }; 530 | 531 | function readLEBUint32 (byteReader) { 532 | var result = 0, shift = 0; 533 | while (true) { 534 | var b = byteReader.read() | 0; 535 | var shifted = (b & 0x7F) << shift; 536 | result |= shifted; 537 | 538 | if ((b & 0x80) === 0) 539 | break; 540 | 541 | shift += 7; 542 | } 543 | 544 | result >>>= 0; 545 | return result; 546 | }; 547 | 548 | function writeLEBInt32 (byteWriter, value) { 549 | var v = value; 550 | 551 | var b = 0; 552 | value |= 0; 553 | 554 | do { 555 | b = value & 0x7F; 556 | value >>= 7; 557 | 558 | var signBit = (b & 0x40) !== 0; 559 | 560 | if ( 561 | ((value === 0) && !signBit) || 562 | ((value === -1) && signBit) 563 | ) { 564 | byteWriter.write(b); 565 | break; 566 | } else { 567 | b |= 0x80; 568 | byteWriter.write(b); 569 | } 570 | } while (true); 571 | }; 572 | 573 | function readLEBInt32 (byteReader) { 574 | var result = 0, shift = 0, b = 0; 575 | while (true) { 576 | b = byteReader.read() | 0; 577 | var shifted = (b & 0x7F) << shift; 578 | result |= shifted; 579 | shift += 7; 580 | 581 | if ((b & 0x80) === 0) 582 | break; 583 | } 584 | 585 | if (b & 0x40) 586 | result |= (-1 << shift); 587 | 588 | return result; 589 | }; 590 | 591 | exports.writeLEBUint32 = writeLEBUint32; 592 | exports.readLEBUint32 = readLEBUint32; 593 | exports.writeLEBInt32 = writeLEBInt32; 594 | exports.readLEBInt32 = readLEBInt32; 595 | 596 | exports.Magic = new Uint8Array([ 597 | 0x89, 598 | 87, 101, 98, 65, 83, 77, 599 | 0x0D, 0x0A, 0x1A, 0x0A 600 | ]); 601 | 602 | // Type tags for which a value is emitted directly instead of 603 | // referred to as a table index 604 | exports.TagIsPrimitive = { 605 | "null" : true, 606 | "false" : true, 607 | "true" : true, 608 | "integer": true, 609 | "double" : true, 610 | "boolean": true, 611 | "name" : true 612 | }; 613 | 614 | // Dumps information on the sorted tables & hit counts 615 | exports.LogTables = false; 616 | 617 | // Expected and decoded json ASTs are pretty printed. 618 | // Can't be on by default because JSON.stringify in node is 619 | // super busted for large objects. 620 | exports.PrettyJson = false; 621 | 622 | // Disable this for ASTs too large for JSON.stringify 623 | exports.DumpJson = true; 624 | 625 | // At the end of encoding, log the size of any large streams 626 | exports.LogStreamSizes = false; 627 | 628 | 629 | exports.ShapeDefinition = ShapeDefinition; 630 | 631 | exports.NamedTable = NamedTable; 632 | exports.UniqueTable = UniqueTable; 633 | exports.StringTable = StringTable; 634 | exports.ObjectTable = ObjectTable; 635 | exports.ShapeTable = ShapeTable; 636 | 637 | exports.GetObjectId = GetObjectId; 638 | exports.NextObjectId = NextObjectId; 639 | 640 | exports.pickTagForField = pickTagForField; 641 | })); -------------------------------------------------------------------------------- /ast-decoder.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | (function (root, factory) { 4 | if (typeof define === 'function' && define.amd) { 5 | define(['exports'], factory); 6 | } else if (typeof exports !== 'undefined') { 7 | factory(exports); 8 | } else { 9 | factory((root.astDecoder = {})); 10 | } 11 | }(this, function (exports) { 12 | var common = require("./ast-common.js"); 13 | var Configuration = require("./configuration.js"); 14 | 15 | var NamedTable = common.NamedTable, 16 | UniqueTable = common.UniqueTable, 17 | StringTable = common.StringTable, 18 | ObjectTable = common.ObjectTable; 19 | 20 | 21 | var IoTrace = false; 22 | var TraceInlining = false; 23 | var TraceInlinedTypeCount = 0; 24 | 25 | 26 | function ValueReader (bytes, index, count, configuration, description) { 27 | if (!configuration) 28 | throw new Error("Configuration required"); 29 | 30 | this.bytes = bytes; 31 | this.byteReader = encoding.makeByteReader(bytes, index, count); 32 | this.scratchBytes = new Uint8Array (128); 33 | this.scratchU32 = new Uint32Array (this.scratchBytes.buffer); 34 | this.scratchI32 = new Int32Array (this.scratchBytes.buffer); 35 | this.scratchF64 = new Float64Array(this.scratchBytes.buffer); 36 | this.configuration = configuration; 37 | this.description = description || null; 38 | } 39 | 40 | ValueReader.prototype.peekByte = function (offset) { 41 | return this.byteReader.peek(offset); 42 | }; 43 | 44 | ValueReader.prototype.readByte = function () { 45 | var result = this.byteReader.read(); 46 | 47 | if (IoTrace) 48 | console.log(this.description + " read byte", result.toString(16)); 49 | 50 | return result; 51 | }; 52 | 53 | ValueReader.prototype.readBytes = function (buffer, offset, count) { 54 | if (arguments.length === 1) { 55 | var temp = new Uint8Array(buffer | 0); 56 | if (this.readBytes(temp, 0, buffer | 0)) 57 | return temp; 58 | else 59 | return false; 60 | } 61 | 62 | for (var i = 0; i < count; i++) { 63 | var b = this.byteReader.read(); 64 | 65 | if (b === false) 66 | return false; 67 | 68 | buffer[offset + i] = b; 69 | } 70 | 71 | return true; 72 | }; 73 | 74 | ValueReader.prototype.readScratchBytes = function (count) { 75 | return this.readBytes(this.scratchBytes, 0, count); 76 | }; 77 | 78 | ValueReader.prototype.readUint32 = function () { 79 | var b1 = this.byteReader.peek(0), 80 | b2 = this.byteReader.peek(1), 81 | b3 = this.byteReader.peek(2); 82 | 83 | if (!this.readScratchBytes(4)) 84 | return false; 85 | 86 | var result = this.scratchU32[0]; 87 | if (IoTrace) 88 | console.log(this.description + " read uint", result.toString(16), "[" + b1.toString(16) + " " + b2.toString(16) + " " + b3.toString(16) + "]"); 89 | return result; 90 | }; 91 | 92 | ValueReader.prototype.readUint24 = function () { 93 | this.scratchU32[0] = 0; 94 | 95 | if (!this.readScratchBytes(3)) 96 | return false; 97 | 98 | var result = this.scratchU32[0]; 99 | if (IoTrace) 100 | console.log(this.description + " read uint24", result.toString(16)); 101 | return result; 102 | }; 103 | 104 | ValueReader.prototype.readInt32 = function () { 105 | if (!this.readScratchBytes(4)) 106 | return false; 107 | 108 | var result = this.scratchI32[0]; 109 | if (IoTrace) 110 | console.log(this.description + " read int", result.toString(16)); 111 | return result; 112 | }; 113 | 114 | ValueReader.prototype.readVarUint32 = function () { 115 | var b1 = this.byteReader.peek(0), 116 | b2 = this.byteReader.peek(1), 117 | b3 = this.byteReader.peek(2); 118 | 119 | if (!this.configuration.EnableVarints) { 120 | if (this.configuration.ThreeByteIndices) 121 | return this.readUint24(); 122 | else 123 | return this.readUint32(); 124 | } 125 | 126 | var result = common.readLEBUint32(this.byteReader); 127 | if (IoTrace) 128 | console.log(this.description + " read varuint", result.toString(16), "[" + b1.toString(16) + " " + b2.toString(16) + " " + b3.toString(16) + "]"); 129 | return result; 130 | }; 131 | 132 | ValueReader.prototype.readVarInt32 = function () { 133 | if (!this.configuration.EnableVarints) 134 | return this.readInt32(); 135 | 136 | var result = common.readLEBInt32(this.byteReader); 137 | if (IoTrace) 138 | console.log(this.description + " read varint", result.toString(16)); 139 | return result; 140 | }; 141 | 142 | ValueReader.prototype.readIndex = function () { 143 | var indexRaw = this.readVarUint32(); 144 | 145 | if (indexRaw === 0) 146 | return 0xFFFFFFFF; 147 | else 148 | return indexRaw - 1; 149 | }; 150 | 151 | ValueReader.prototype.readFloat64 = function () { 152 | if (!this.readScratchBytes(8)) 153 | return false; 154 | 155 | var result = this.scratchF64[0]; 156 | if (IoTrace) 157 | console.log(this.description + " read float64", result.toFixed(4)); 158 | return result; 159 | }; 160 | 161 | ValueReader.prototype.readUtf8String = function () { 162 | var length = 0, position; 163 | 164 | if (!this.configuration.NullTerminatedStrings) { 165 | length = this.readIndex(); 166 | if (length === false) 167 | return false; 168 | 169 | // HACK So we can encode null distinct from "" 170 | if (length === 0xFFFFFFFF) 171 | return null; 172 | 173 | if (length === 0) 174 | return ""; 175 | 176 | position = this.byteReader.getPosition(); 177 | 178 | } else { 179 | // HACK So we can encode null distinct from "" 180 | if (this.peekByte() === 0xFF) { 181 | this.readByte(); 182 | return null; 183 | } 184 | 185 | var b; 186 | 187 | position = this.byteReader.getPosition(); 188 | while (Number(b = this.readByte()) > 0) 189 | length++; 190 | } 191 | 192 | var result = encoding.UTF8.decode(this.bytes, position, length); 193 | 194 | if (!this.configuration.NullTerminatedStrings) 195 | this.byteReader.skip(length); 196 | 197 | return result; 198 | }; 199 | 200 | ValueReader.prototype.readSubstream = function () { 201 | var prior = IoTrace; 202 | IoTrace = false; 203 | 204 | var description = this.readUtf8String(); 205 | var length = this.readUint32(); 206 | 207 | var result = new ValueReader(this.bytes, this.byteReader.getPosition(), length, this.configuration, description); 208 | 209 | this.byteReader.skip(length); 210 | 211 | var length2 = this.readUint32(); 212 | if (length2 !== length) 213 | throw new Error("Length footer didn't match length header"); 214 | 215 | IoTrace = prior; 216 | return result; 217 | }; 218 | 219 | ValueReader.prototype.skip = function (distance) { 220 | this.byteReader.skip(distance); 221 | }; 222 | 223 | 224 | function JsAstModule (configuration, shapes) { 225 | this.configuration = configuration; 226 | this.shapes = shapes; 227 | 228 | this.tags = null; 229 | this.objects = null; 230 | this.valueStreams = Object.create(null); 231 | 232 | this.typeTagStream = null; 233 | this.inliningStream = null; 234 | this.packedIndexStream = null; 235 | 236 | this.root = null; 237 | }; 238 | 239 | 240 | function _decodeTypeTag (module, tagIndex) { 241 | if (false) 242 | console.log( 243 | "read type tag " + tagIndex + 244 | " (" + module.tags[tagIndex] + ") " + 245 | " as varuint from " + reader.description 246 | ); 247 | 248 | var tag = module.tags[tagIndex]; 249 | if (typeof (tag) !== "string") 250 | throw new Error("Invalid tag index: " + tagIndex); 251 | 252 | return tag; 253 | }; 254 | 255 | 256 | function readTypeTag (reader, module) { 257 | if (module.configuration.TypeTagStream) 258 | reader = module.typeTagStream; 259 | 260 | var tagIndex = reader.readIndex(); 261 | if (tagIndex === false) 262 | throw new Error("Truncated file"); 263 | 264 | return _decodeTypeTag(module, tagIndex); 265 | }; 266 | 267 | 268 | function getTableEntry (table, index) { 269 | if (!table) 270 | throw new Error("Table expected"); 271 | 272 | if (index === 0xFFFFFFFF) 273 | return null; 274 | 275 | if ((index < 0) || (index >= table.length)) 276 | throw new Error("Invalid index " + index); 277 | 278 | var result = table[index]; 279 | if (typeof (result) === "undefined") 280 | throw new Error("Uninitialized at index " + index); 281 | 282 | return result; 283 | }; 284 | 285 | 286 | function readInliningFlag (reader, module) { 287 | var flag = module.inliningStream.readByte(); 288 | return flag; 289 | }; 290 | 291 | 292 | var nullBundle = { 293 | isInlined: true, 294 | isNull: true, 295 | tag: "null", 296 | index: 0xFFFFFFFF 297 | }; 298 | 299 | function readInliningBundle (reader, module) { 300 | if (module.configuration.PackedInliningFlags) { 301 | if (module.configuration.PackedIndexStream) 302 | reader = module.packedIndexStream; 303 | 304 | var packedIndex = reader.readIndex(); 305 | 306 | if (packedIndex === 0xFFFFFFFF) { 307 | // console.log("unpacked FFFFFFFF -> null"); 308 | return nullBundle; 309 | } 310 | 311 | var flag = packedIndex & 0x1; 312 | var index = packedIndex >> 1; 313 | // console.log("unpacked", packedIndex.toString(16), "->", flag, index); 314 | 315 | if (flag) { 316 | return { 317 | isInlined: true, 318 | tag: _decodeTypeTag(module, index) 319 | }; 320 | } else { 321 | return { 322 | isInlined: false, 323 | index: index 324 | } 325 | } 326 | 327 | } else { 328 | var flag = readInliningFlag(reader, module); 329 | 330 | if (flag === 0xFF) 331 | return nullBundle; 332 | 333 | if (flag) { 334 | return { 335 | isInlined: true, 336 | tag: readTypeTag(reader, module) 337 | } 338 | } else { 339 | var index = reader.readIndex(); 340 | return { 341 | isInlined: false, 342 | index: index 343 | } 344 | } 345 | } 346 | }; 347 | 348 | 349 | function deserializeObjectReference (reader, module, tag) { 350 | var isUntypedObject = (tag === "object"); 351 | if (!isUntypedObject) { 352 | var shape = module.shapes.get(tag); 353 | if (!shape) 354 | throw new Error("Unhandled value type " + tag + " with no shape"); 355 | } 356 | 357 | var objectTable; 358 | if (IoTrace) 359 | console.log(reader.description + " read object"); 360 | 361 | objectTable = module.objects; 362 | 363 | var shouldConditionalInline = 364 | reader.configuration.ConditionalInlining && 365 | (tag !== "any"); 366 | var index; 367 | 368 | if (shouldConditionalInline) { 369 | var bundle = readInliningBundle(reader, module); 370 | 371 | if (bundle.isNull) { 372 | return null; 373 | 374 | } else if (bundle.isInlined) { 375 | var result = new Object(); 376 | 377 | deserializeObjectContents(reader, module, result, bundle.isInlined, bundle.tag); 378 | 379 | return result; 380 | 381 | } else { 382 | index = bundle.index; 383 | } 384 | 385 | } else { 386 | index = reader.readIndex(); 387 | } 388 | 389 | return getTableEntry(objectTable, index); 390 | }; 391 | 392 | function deserializeValueWithKnownTag (reader, module, tag) { 393 | switch (tag) { 394 | case "any": { 395 | tag = readTypeTag(reader, module); 396 | if (tag === "any") 397 | throw new Error("Found 'any' type tag when reading any-tag"); 398 | 399 | if (IoTrace) 400 | console.log(reader.description + " read any ->"); 401 | return deserializeValueWithKnownTag(reader, module, tag); 402 | } 403 | 404 | case "symbol": 405 | if (IoTrace) 406 | console.log(reader.description + " read symbol"); 407 | 408 | if (module.configuration.InternedSymbols) { 409 | var index = reader.readIndex(); 410 | return index; 411 | } else { 412 | var string = reader.readUtf8String(); 413 | return string; 414 | } 415 | 416 | case "string": 417 | if (IoTrace) 418 | console.log(reader.description + " read string"); 419 | var string = reader.readUtf8String(); 420 | return string; 421 | 422 | case "array": 423 | var length = reader.readVarUint32(); 424 | var array = new Array(length); 425 | 426 | if (length > 0) { 427 | var elementTag = readTypeTag(reader, module); 428 | if (IoTrace) 429 | console.log(reader.description + " read array of type " + elementTag + " with " + length + " element(s)"); 430 | 431 | for (var i = 0; i < length; i++) { 432 | var element = deserializeValueWithKnownTag(reader, module, elementTag); 433 | array[i] = element; 434 | } 435 | } else { 436 | if (IoTrace) 437 | console.log(reader.description + " read empty array"); 438 | } 439 | 440 | return array; 441 | 442 | case "boolean": 443 | return Boolean(reader.readByte()); 444 | 445 | case "integer": 446 | return reader.readInt32(); 447 | 448 | case "double": 449 | return reader.readFloat64(); 450 | 451 | default: 452 | case "object": 453 | return deserializeObjectReference(reader, module, tag); 454 | } 455 | 456 | throw new Error("unexpected"); 457 | }; 458 | 459 | 460 | function getReaderForField (defaultReader, module, field, tag) { 461 | if (module.configuration.ValueStreamPerType) { 462 | var reader = module.valueStreams[tag]; 463 | if (!reader) 464 | throw new Error("No value stream for tag '" + tag + "'"); 465 | 466 | return reader; 467 | } else { 468 | return defaultReader; 469 | } 470 | }; 471 | 472 | 473 | function deserializeFieldValue (reader, module, shape, field, overrideReader) { 474 | var tag = common.pickTagForField(field, function (t) { 475 | var shape = module.shapes.get(t); 476 | return shape; 477 | }); 478 | 479 | if ( 480 | overrideReader && 481 | reader.configuration.NoOverridingPrimitiveStream && 482 | common.TagIsPrimitive[tag] 483 | ) 484 | overrideReader = false; 485 | 486 | if (!overrideReader) { 487 | var oldReader = reader; 488 | reader = getReaderForField(reader, module, field, tag); 489 | 490 | if (IoTrace) { 491 | if (reader === oldReader) 492 | console.log("field " + field.name + " did not pick reader -> " + reader.description); 493 | else 494 | console.log("field " + field.name + " picked reader " + oldReader.description + " -> " + reader.description); 495 | } 496 | } else if (IoTrace) { 497 | console.log("field " + field.name + " reader forced " + reader.description); 498 | } 499 | 500 | var value = deserializeValueWithKnownTag(reader, module, tag); 501 | return value; 502 | } 503 | 504 | 505 | function deserializeObjectContents (reader, module, obj, isInline, inlinedTypeTag) { 506 | var shapeName; 507 | 508 | if (typeof (inlinedTypeTag) === "string") 509 | shapeName = inlinedTypeTag; 510 | else 511 | shapeName = readTypeTag(reader, module); 512 | 513 | var shouldOverride = false; 514 | if (isInline) { 515 | var trace = TraceInlining || (TraceInlinedTypeCount-- > 0); 516 | 517 | if ( 518 | module.configuration.ValueStreamPerType && 519 | module.configuration.PartitionedInlining 520 | ) { 521 | reader = module.valueStreams[shapeName]; 522 | shouldOverride = true; 523 | if (trace) 524 | console.log("Reading inlined " + shapeName + " from " + reader.description); 525 | } else { 526 | if (trace) 527 | console.log("reading inlined " + shapeName); 528 | } 529 | } 530 | 531 | var shape = module.shapes.get(shapeName); 532 | if (!shape) 533 | throw new Error("Could not find shape '" + shapeName + "'"); 534 | 535 | obj[module.shapes.shapeKey] = shapeName; 536 | 537 | for (var i = 0, l = shape.fields.length; i < l; i++) { 538 | var fd = shape.fields[i]; 539 | 540 | try { 541 | var value = deserializeFieldValue(reader, module, shape, fd, shouldOverride); 542 | } catch (e) { 543 | console.log( 544 | "Failed while reading field " + fd.name + 545 | " of an " + shapeName 546 | ); 547 | throw e; 548 | } 549 | 550 | obj[fd.name] = value; 551 | if (IoTrace) 552 | console.log("// " + fd.name + " =", value); 553 | } 554 | 555 | if (IoTrace) 556 | console.log(obj); 557 | }; 558 | 559 | 560 | function deserializeTable (reader, payloadReader) { 561 | var count = reader.readUint32(); 562 | if (count === false) 563 | throw new Error("Truncated file"); 564 | 565 | var result = new Array(count); 566 | 567 | for (var i = 0; i < count; i++) { 568 | var item = payloadReader(reader); 569 | result[i] = item; 570 | } 571 | 572 | return result; 573 | }; 574 | 575 | 576 | function deserializeObjectTable (reader, module) { 577 | var count = reader.readUint32(); 578 | if (count === false) 579 | throw new Error("Truncated file"); 580 | 581 | var table = module.objects; 582 | 583 | if (count !== table.length) 584 | throw new Error("Read " + count + " object(s) into table of length " + table.length); 585 | 586 | for (var i = 0; i < count; i++) { 587 | var obj = table[i]; 588 | deserializeObjectContents(reader, module, obj, false); 589 | } 590 | }; 591 | 592 | 593 | function allocateObjectTable (module, count) { 594 | var table = new Array(count); 595 | for (var i = 0; i < count; i++) { 596 | var o = new Object(); 597 | 598 | table[i] = o; 599 | }; 600 | 601 | table.baseIndex = 0; 602 | 603 | if (module.objects) 604 | throw new Error("Object table already allocated"); 605 | else 606 | module.objects = table; 607 | }; 608 | 609 | 610 | function bytesToModule (configuration, shapes, bytes) { 611 | var reader = new ValueReader(bytes, 0, bytes.length, configuration, "module"); 612 | 613 | var magic = reader.readBytes(common.Magic.length); 614 | if (JSON.stringify(magic) !== JSON.stringify(common.Magic)) { 615 | console.log(magic, common.Magic); 616 | throw new Error("Magic header does not match"); 617 | } 618 | 619 | var result = new JsAstModule(configuration, shapes); 620 | 621 | // The lengths are stored in front of the tables themselves, 622 | // this simplifies table deserialization... 623 | var tagCount = reader.readUint32(); 624 | var objectCount = reader.readUint32(); 625 | 626 | 627 | var readUtf8String = function (_) { 628 | var text = _.readUtf8String(); 629 | if (text === false) 630 | throw new Error("Truncated file"); 631 | return text; 632 | }; 633 | 634 | var tagReader = reader.readSubstream(); 635 | result.tags = deserializeTable(tagReader, readUtf8String); 636 | 637 | 638 | if ( 639 | configuration.ConditionalInlining && 640 | !configuration.PackedInliningFlags 641 | ) 642 | result.inliningStream = reader.readSubstream(); 643 | 644 | if (configuration.TypeTagStream) 645 | result.typeTagStream = reader.readSubstream(); 646 | 647 | if ( 648 | configuration.PackedInliningFlags && 649 | configuration.PackedIndexStream 650 | ) 651 | result.packedIndexStream = reader.readSubstream(); 652 | 653 | 654 | if (configuration.ValueStreamPerType) 655 | for (var i = 0; i < result.tags.length; i++) { 656 | var tagIndex = reader.readIndex(); 657 | var tag = result.tags[tagIndex]; 658 | 659 | var valueStream = reader.readSubstream(); 660 | result.valueStreams[tag] = valueStream; 661 | } 662 | 663 | 664 | allocateObjectTable(result, objectCount); 665 | 666 | 667 | var objectReader = reader.readSubstream(); 668 | 669 | deserializeObjectTable(objectReader, result); 670 | 671 | 672 | var rootReader = reader.readSubstream(); 673 | 674 | result.root = deserializeValueWithKnownTag(rootReader, result, "any"); 675 | if (!result.root) 676 | throw new Error("Failed to retrieve root from module"); 677 | 678 | return result; 679 | }; 680 | 681 | 682 | exports.PrettyJson = common.PrettyJson; 683 | 684 | exports.ShapeTable = common.ShapeTable; 685 | exports.ValueReader = ValueReader; 686 | 687 | exports.bytesToModule = bytesToModule; 688 | })); -------------------------------------------------------------------------------- /ast-encoder.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | (function (root, factory) { 4 | if (typeof define === 'function' && define.amd) { 5 | define(['exports'], factory); 6 | } else if (typeof exports !== 'undefined') { 7 | factory(exports); 8 | } else { 9 | factory((root.astEncoder = {})); 10 | } 11 | }(this, function (exports) { 12 | var common = require("./ast-common.js"); 13 | var Configuration = require("./configuration.js"); 14 | var treeBuilder = require("./parse/treebuilder.js"); 15 | 16 | var AsmlikeJsonTreeBuilder = treeBuilder.AsmlikeJSON; 17 | var NamedTable = common.NamedTable, 18 | UniqueTable = common.UniqueTable, 19 | StringTable = common.StringTable, 20 | ObjectTable = common.ObjectTable, 21 | GetObjectId = common.GetObjectId, 22 | NextObjectId = common.NextObjectId; 23 | 24 | var IoTrace = false; 25 | var TraceInlining = false; 26 | var TraceInlinedTypeCount = 0; 27 | 28 | function ValueWriter (configuration, description, parent) { 29 | // HACK: Default max size 128mb because growable buffers are effort 30 | var capacity = (1024 * 1024) * 128; 31 | if (parent) 32 | capacity /= 4; 33 | 34 | this.bytes = new Uint8Array(capacity); 35 | this.position = 0; 36 | this.view = new DataView(this.bytes.buffer); 37 | 38 | if (parent) 39 | this.varintSizes = parent.varintSizes; 40 | else 41 | this.varintSizes = [0, 0, 0, 0, 0]; 42 | 43 | if (!configuration) 44 | throw new Error("configuration required"); 45 | 46 | this.configuration = configuration; 47 | this.description = description; 48 | }; 49 | 50 | ValueWriter.prototype.getPosition = function () { 51 | return this.position; 52 | }; 53 | 54 | ValueWriter.prototype.skip = function (distance) { 55 | this.position += distance; 56 | }; 57 | 58 | ValueWriter.prototype.write = 59 | ValueWriter.prototype.writeByte = function (b) { 60 | if (this.position >= this.bytes.length) 61 | throw new Error("buffer full"); 62 | 63 | if (IoTrace) 64 | console.log(this.description + " write byte", b.toString(16)); 65 | 66 | this.bytes[this.position++] = b; 67 | }; 68 | 69 | ValueWriter.prototype.writeBytes = function (bytes, offset, count) { 70 | if (this.position >= this.bytes.length) 71 | throw new Error("buffer full"); 72 | 73 | if (arguments.length === 1) { 74 | this.bytes.set(bytes, this.position); 75 | this.position += bytes.length; 76 | return; 77 | } else if (arguments.length === 3) { 78 | offset |= 0; 79 | count |= 0; 80 | } else { 81 | throw new Error("Expected (bytes) or (bytes, offset, count)"); 82 | } 83 | 84 | this.bytes.set(bytes.subarray(offset, offset + count), this.position); 85 | this.position += count; 86 | }; 87 | 88 | ValueWriter.prototype.writeScratchBytes = function (count) { 89 | this.writeBytes(this.scratchBytes, 0, count); 90 | }; 91 | 92 | ValueWriter.prototype.writeUint24 = function (value) { 93 | var masked = value & 0xFFFFFF; 94 | if (masked !== value) 95 | throw new Error("Value is larger than 24 bits"); 96 | 97 | if (IoTrace) 98 | console.log(this.description + " write uint24", masked.toString(16)); 99 | 100 | this.view.setUint32(this.position, masked, true); 101 | this.position += 3; 102 | }; 103 | 104 | ValueWriter.prototype.writeUint32 = function (value) { 105 | this.view.setUint32(this.position, value, true); 106 | 107 | if (IoTrace) 108 | console.log(this.description + " write uint", value.toString(16), 109 | "[" + this.bytes[this.position].toString(16) + " " + this.bytes[this.position + 1].toString(16) + " " + this.bytes[this.position + 2].toString(16) + "]" 110 | ); 111 | 112 | this.position += 4; 113 | }; 114 | 115 | ValueWriter.prototype.writeInt32 = function (value) { 116 | if (IoTrace) 117 | console.log(this.description + " write int", value.toString(16)); 118 | 119 | this.view.setInt32(this.position, value, true); 120 | this.position += 4; 121 | }; 122 | 123 | ValueWriter.prototype.writeVarUint32 = function (value) { 124 | if (this.configuration.EnableVarints) { 125 | if (IoTrace) 126 | console.log(this.description + " write varuint", value.toString(16)); 127 | 128 | var before = this.position; 129 | common.writeLEBUint32(this, value); 130 | var after = this.position; 131 | var lengthBytes = after - before; 132 | this.varintSizes[lengthBytes - 1] += 1; 133 | 134 | // HACK for checking inlining markers 135 | return this.bytes[before]; 136 | } else if (this.configuration.ThreeByteIndices) { 137 | this.writeUint24(value); 138 | } else { 139 | this.writeUint32(value); 140 | } 141 | }; 142 | 143 | ValueWriter.prototype.writeVarInt32 = function (value) { 144 | if (this.configuration.EnableVarints) { 145 | if (IoTrace) 146 | console.log(this.description + " write varint", value.toString(16)); 147 | 148 | var before = this.position; 149 | common.writeLEBInt32(this, value); 150 | var after = this.position; 151 | var lengthBytes = after - before; 152 | this.varintSizes[lengthBytes - 1] += 1; 153 | 154 | // HACK for checking inlining markers 155 | return this.bytes[before]; 156 | } else { 157 | this.writeInt32(value); 158 | } 159 | }; 160 | 161 | ValueWriter.prototype.writeIndex = function (value) { 162 | if (typeof (value) !== "number") { 163 | if ( 164 | (typeof (value) === "object") && 165 | value && 166 | (value.type === "symbol") 167 | ) 168 | value = value.valueOf(); 169 | else 170 | throw new Error("writeIndex expected number"); 171 | } 172 | 173 | if (value === 0xFFFFFFFF) 174 | this.writeVarUint32(0); 175 | else 176 | this.writeVarUint32(value + 1); 177 | }; 178 | 179 | ValueWriter.prototype.writeFloat64 = function (value) { 180 | if (IoTrace) 181 | console.log(this.description + " write float64", value.toFixed(4)); 182 | 183 | this.view.setFloat64(this.position, value, true); 184 | this.position += 8; 185 | }; 186 | 187 | ValueWriter.prototype.writeUtf8String = function (text) { 188 | // HACK so we can encode null strings as distinct from "" 189 | if (text === null) { 190 | if (this.configuration.NullTerminatedStrings) 191 | this.writeByte(0xFF); 192 | else 193 | this.writeIndex(0xFFFFFFFF); 194 | 195 | return; 196 | } 197 | 198 | if (typeof (text) !== "string") 199 | throw new Error("writeUtf8String expected string"); 200 | 201 | var lengthBytes = 0; 202 | var counter = { 203 | write: function (byte) { 204 | lengthBytes += 1; 205 | }, 206 | getResult: function () { 207 | } 208 | }; 209 | 210 | // Encode but discard bytes to compute length 211 | encoding.UTF8.encode(text, counter); 212 | 213 | if (!this.configuration.NullTerminatedStrings) 214 | this.writeIndex(lengthBytes); 215 | 216 | encoding.UTF8.encode(text, this); 217 | 218 | if (this.configuration.NullTerminatedStrings) 219 | this.writeByte(0); 220 | }; 221 | 222 | ValueWriter.prototype.writeSubstream = function (otherWriter) { 223 | var sizeBytes = otherWriter.position; 224 | var description = otherWriter.description; 225 | 226 | if (common.LogStreamSizes && (sizeBytes >= 8 * 1024)) 227 | console.log(description + ": " + (sizeBytes / 1024).toFixed(2) + "KB"); 228 | 229 | var prior = IoTrace; 230 | IoTrace = false; 231 | 232 | this.writeUtf8String(description); 233 | this.writeUint32(sizeBytes); 234 | 235 | this.writeBytes(otherWriter.bytes, 0, sizeBytes); 236 | 237 | this.writeUint32(sizeBytes); 238 | 239 | IoTrace = prior; 240 | }; 241 | 242 | ValueWriter.prototype.getResult = 243 | ValueWriter.prototype.toArray = function () { 244 | return this.bytes.subarray(0, this.position); 245 | }; 246 | 247 | 248 | function JsAstModule (configuration, shapes) { 249 | this.configuration = configuration; 250 | this.shapes = shapes; 251 | 252 | // Used to store the inferred type tags for arrays during 253 | // the initial tree-walk 254 | this.arrayTypeTags = new WeakMap(); 255 | // Used to cache the result of the 'shape small enough to always inline' check 256 | this.isShapeSmallCache = new WeakMap(); 257 | 258 | this.tags = new StringTable("tag"); 259 | 260 | this.tags.add("any"); 261 | this.tags.add("array"); 262 | this.tags.add("object"); 263 | this.tags.add("boolean"); 264 | this.tags.add("string"); 265 | this.tags.add("symbol"); 266 | this.tags.add("integer"); 267 | this.tags.add("number"); 268 | 269 | this.objects = new ObjectTable("object"); 270 | this.valueStreams = Object.create(null); 271 | 272 | this.inliningWriter = null; 273 | this.typeTagWriter = null; 274 | this.packedIndexWriter = null; 275 | 276 | this.anyTypeValuesWritten = 0; 277 | this.typeTagsWritten = 0; 278 | this.inlinedObjects = 0; 279 | this.notInlinedObjects = 0; 280 | 281 | this._getTableForTypeTag = this.getTableForTypeTag.bind(this); 282 | }; 283 | 284 | 285 | JsAstModule.prototype.isShapeSmall = function (shape) { 286 | if (typeof (this.configuration.InlineObjectSizeThreshold) !== "number") 287 | return false; 288 | 289 | var result = this.isShapeSmallCache.get(shape); 290 | if (typeof (result) === "boolean") 291 | return result; 292 | 293 | var estimatedIndexSize = 294 | this.configuration.EnableVarints 295 | ? 3 // FIXME: 2? 296 | : 4; 297 | 298 | // Type tag 299 | var size = estimatedIndexSize; 300 | // Inlining flag 301 | if (this.configuration.ConditionalInlining) 302 | size += 1; 303 | 304 | for (var i = 0, l = shape.shape.fields.length; i < l; i++) { 305 | var fd = shape.shape.fields[i]; 306 | 307 | switch (fd.type) { 308 | case "integer": 309 | size += 4; 310 | break; 311 | 312 | case "double": 313 | size += 8; 314 | break; 315 | 316 | case "boolean": 317 | size += 1; 318 | break; 319 | 320 | default: 321 | size += estimatedIndexSize; 322 | break; 323 | } 324 | } 325 | 326 | result = (size <= this.configuration.InlineObjectSizeThreshold); 327 | this.isShapeSmallCache.set(shape, result); 328 | return result; 329 | }; 330 | 331 | 332 | JsAstModule.prototype.getObjectTable = function (nodeOrShape) { 333 | if (nodeOrShape === null) 334 | return null; 335 | 336 | return this.objects; 337 | }; 338 | 339 | 340 | JsAstModule.prototype.getShapeForObject = function (obj) { 341 | var shapeName = obj[this.shapes.shapeKey]; 342 | 343 | if (typeof (shapeName) !== "string") { 344 | // HACK so js RegExp instances can fit into this shape model 345 | if ( 346 | (typeof (obj) === "object") && 347 | (Object.getPrototypeOf(obj) === RegExp.prototype) 348 | ) 349 | shapeName = "RegExp"; 350 | else 351 | throw new Error("Unsupported object " + obj + " with shape name " + JSON.stringify(shapeName)) 352 | } 353 | 354 | var shape = this.shapes.get(shapeName); 355 | if (!shape) { 356 | console.log(shapeName, obj, Object.getPrototypeOf(obj), obj.toString()); 357 | throw new Error("Unknown shape " + shapeName); 358 | } 359 | 360 | var shapeTagIndex = null; 361 | if (this.tags.isFinalized) { 362 | shapeTagIndex = this.tags.get_index(shapeName); 363 | } else { 364 | this.tags.add(shapeName); 365 | } 366 | 367 | return { 368 | name: shapeName, 369 | shape: shape, 370 | tagIndex: shapeTagIndex 371 | }; 372 | }; 373 | 374 | 375 | JsAstModule.prototype.createValueStreams = function (writer) { 376 | var self = this; 377 | 378 | this.tags.forEach(function (entry) { 379 | var tag = entry.get_name(); 380 | 381 | self.valueStreams[tag] = new ValueWriter(self.configuration, "values[" + tag + "]", writer); 382 | }); 383 | }; 384 | 385 | 386 | JsAstModule.prototype.getValueWriterForField = function (defaultWriter, field, tag) { 387 | if (this.configuration.ValueStreamPerType) { 388 | var writer = this.valueStreams[tag]; 389 | if (!writer) 390 | throw new Error("No value stream for tag '" + tag + "'"); 391 | 392 | return writer; 393 | } else { 394 | return defaultWriter; 395 | } 396 | }; 397 | 398 | 399 | JsAstModule.prototype.writeTypeTag = function (defaultWriter, tag) { 400 | var writer = defaultWriter; 401 | if (this.configuration.TypeTagStream) 402 | writer = this.typeTagWriter; 403 | 404 | writer.writeIndex(tag); 405 | 406 | this.typeTagsWritten += 1; 407 | }; 408 | 409 | 410 | JsAstModule.prototype._writeInliningFlag = function (flag) { 411 | this.inliningWriter.writeByte(flag); 412 | }; 413 | 414 | 415 | JsAstModule.prototype._writePackedIndex = function (writer, flag, index) { 416 | if (this.configuration.PackedIndexStream) 417 | writer = this.packedIndexWriter; 418 | 419 | if ((flag === 0xFF) || (index === 0xFFFFFFFF)) { 420 | // console.log("packed", flag, index, "-> FFFFFFFF"); 421 | // LEB 0xFFFFFFFF == 0 422 | writer.writeIndex(0xFFFFFFFF); 423 | return; 424 | } 425 | 426 | var packed = index << 1; 427 | packed |= (flag & 0x1); 428 | 429 | // console.log("packed", flag, index, "->", packed.toString(16)); 430 | writer.writeIndex(packed); 431 | }; 432 | 433 | 434 | JsAstModule.prototype.writeInliningFlagAndIndex = function (writer, flag, index) { 435 | if (this.configuration.PackedInliningFlags) { 436 | this._writePackedIndex(writer, flag, index); 437 | } else { 438 | this._writeInliningFlag(flag); 439 | if (flag === 0xFF) 440 | return; 441 | 442 | writer.writeIndex(index); 443 | } 444 | }; 445 | 446 | 447 | JsAstModule.prototype.writeInliningFlagAndTypeTag = function (writer, flag, typeTag) { 448 | if (this.configuration.PackedInliningFlags) { 449 | this._writePackedIndex(writer, flag, typeTag); 450 | this.typeTagsWritten += 1; 451 | } else { 452 | this._writeInliningFlag(flag); 453 | if (flag === 0xFF) 454 | return; 455 | 456 | this.writeTypeTag(writer, typeTag); 457 | } 458 | }; 459 | 460 | 461 | JsAstModule.prototype.serializeFieldValue = function (writer, shape, field, value, overrideWriter) { 462 | // FIXME: Hack together field definition type -> tag conversion 463 | var tag = common.pickTagForField(field, this._getTableForTypeTag); 464 | 465 | if ( 466 | overrideWriter && 467 | this.configuration.NoOverridingPrimitiveStream && 468 | common.TagIsPrimitive[tag] 469 | ) 470 | overrideWriter = false; 471 | 472 | try { 473 | if (!overrideWriter) { 474 | var oldWriter = writer; 475 | writer = this.getValueWriterForField(writer, field, tag); 476 | 477 | if (IoTrace) { 478 | if (writer === oldWriter) 479 | console.log("field " + field.name + " did not pick writer -> " + writer.description); 480 | else 481 | console.log("field " + field.name + " picked writer " + oldWriter.description + " -> " + writer.description); 482 | } 483 | } else if (IoTrace) { 484 | console.log("field " + field.name + " writer forced + " + writer.description); 485 | } 486 | 487 | this.serializeValueWithKnownTag(writer, value, tag); 488 | } catch (exc) { 489 | console.log("Failed while writing field " + field.name + " of type " + shape.name); 490 | throw exc; 491 | } 492 | }; 493 | 494 | 495 | JsAstModule.prototype.serializeObjectContents = function (writer, node, canBeInlined, isInline) { 496 | if (Array.isArray(node)) 497 | throw new Error("Should have serialized inline array"); 498 | 499 | var shape = this.getShapeForObject(node); 500 | if (shape.tagIndex === null) 501 | throw new Error("Tag table not finalized"); 502 | 503 | var shouldOverride = false; 504 | if (isInline) { 505 | this.writeInliningFlagAndTypeTag(writer, 1, shape.tagIndex); 506 | 507 | var trace = TraceInlining || (TraceInlinedTypeCount-- > 0); 508 | if (trace) 509 | console.log("Inlined", shape.name, shape.tagIndex.toString(16)); 510 | 511 | if ( 512 | this.configuration.ValueStreamPerType && 513 | this.configuration.PartitionedInlining 514 | ) { 515 | writer = this.valueStreams[shape.name]; 516 | shouldOverride = true; 517 | 518 | if (trace) 519 | console.log("Inlining " + shape.name + " to " + writer.description); 520 | } 521 | } else { 522 | 523 | this.writeTypeTag(writer, shape.tagIndex); 524 | } 525 | 526 | var self = this, fields = shape.shape.fields; 527 | for (var i = 0, l = fields.length; i < l; i++) { 528 | var fd = fields[i]; 529 | var value = node[fd.name]; 530 | 531 | if (typeof (value) === "undefined") 532 | value = null; 533 | 534 | this.serializeFieldValue( 535 | writer, shape, fd, value, shouldOverride 536 | ); 537 | 538 | if (IoTrace) 539 | console.log("// " + fd.name + " =", value); 540 | } 541 | }; 542 | 543 | 544 | JsAstModule.prototype.getTypeTagForValue = function (value) { 545 | var jsType = typeof (value); 546 | 547 | if (value === null) 548 | return "null"; 549 | 550 | // FIXME: names? 551 | 552 | switch (jsType) { 553 | case "string": 554 | return "string"; 555 | 556 | case "boolean": 557 | return value ? "true" : "false"; 558 | 559 | case "number": 560 | var i = value | 0; 561 | if (i === value) 562 | return "integer"; 563 | else 564 | return "double"; 565 | 566 | case "object": 567 | if (Array.isArray(value)) { 568 | return "array"; 569 | } else if (value.type === "symbol") { 570 | return "symbol"; 571 | } else { 572 | var shape = this.getShapeForObject(value); 573 | return shape.name; 574 | } 575 | 576 | default: { 577 | throw new Error("Unhandled value type " + jsType); 578 | } 579 | } 580 | }; 581 | 582 | 583 | JsAstModule.prototype.getIndexForTypeTag = function (tag) { 584 | return this.tags.get_index(tag); 585 | }; 586 | 587 | 588 | JsAstModule.prototype.getTableForTypeTag = function (tag, actualValueTag) { 589 | if (actualValueTag === "null") 590 | return null; 591 | 592 | if ((tag === "string") || (tag === "symbol")) 593 | return null; 594 | else { 595 | var shape = this.shapes.get(tag); 596 | if (shape) { 597 | return this.getObjectTable(shape); 598 | } else if (tag === "object") { 599 | return this.getObjectTable("object"); 600 | } else { 601 | return null; 602 | } 603 | } 604 | }; 605 | 606 | 607 | JsAstModule.prototype.serializeArray = function (writer, value) { 608 | writer.writeVarUint32(value.length); 609 | if (value.length === 0) { 610 | if (IoTrace) 611 | console.log(" Wrote empty array to " + writer.description); 612 | return; 613 | } 614 | 615 | // The tree-walker figured out whether we need to use 'any' earlier 616 | var elementTag = this.arrayTypeTags.get(value); 617 | var elementTagIndex = this.getIndexForTypeTag(elementTag); 618 | this.writeTypeTag(writer, elementTagIndex, false); 619 | 620 | if (IoTrace) 621 | console.log(" Writing array of type " + elementTag + " with " + value.length + " item(s) to " + writer.description); 622 | 623 | var elementsInlined = 0; 624 | try { 625 | for (var i = 0; i < value.length; i++) { 626 | var element = value[i]; 627 | var wasInlined = this.serializeValueWithKnownTag(writer, element, elementTag); 628 | if (wasInlined) 629 | elementsInlined += 1; 630 | } 631 | } catch (e) { 632 | console.log("Failed while writing array with tag", elementTag); 633 | throw e; 634 | } 635 | 636 | if (false) 637 | console.log(elementsInlined + "/" + value.length + " element(s) of tag " + elementTag + " inlined"); 638 | }; 639 | 640 | 641 | // Or unknown tag, if you pass 'any', because why not. 642 | JsAstModule.prototype.serializeValueWithKnownTag = function (writer, value, tag) { 643 | switch (tag) { 644 | case "true": 645 | case "false": 646 | case "null": 647 | // no-op. value is encoded by the type tag. 648 | return true; 649 | 650 | case "boolean": 651 | writer.writeByte(value ? 1 : 0); 652 | return true; 653 | 654 | case "integer": 655 | // TODO: varint? 656 | writer.writeInt32(value); 657 | return true; 658 | 659 | case "double": 660 | writer.writeFloat64(value); 661 | return true; 662 | 663 | case "string": 664 | writer.writeUtf8String(value); 665 | return true; 666 | 667 | case "symbol": 668 | if (this.configuration.InternedSymbols) { 669 | writer.writeIndex(value); 670 | } else { 671 | writer.writeUtf8String(value); 672 | } 673 | 674 | return true; 675 | 676 | case "any": { 677 | if (IoTrace) 678 | console.log(writer.description + " write any ->"); 679 | 680 | // FIXME: gross. 681 | tag = this.getTypeTagForValue(value); 682 | if (tag === "any") 683 | throw new Error("Couldn't identify a tag for 'any' value"); 684 | 685 | this.anyTypeValuesWritten += 1; 686 | var tagIndex = this.getIndexForTypeTag(tag); 687 | this.writeTypeTag(writer, tagIndex, false); 688 | 689 | return this.serializeValueWithKnownTag(writer, value, tag); 690 | } 691 | 692 | case "array": 693 | this.serializeArray(writer, value); 694 | return true; 695 | 696 | case "object": 697 | default: 698 | break; 699 | } 700 | 701 | return this.serializeObjectReference(writer, value, tag); 702 | } 703 | 704 | 705 | // Returns true if the reference was inlined 706 | JsAstModule.prototype.serializeObjectReference = function (writer, value, tag) { 707 | var shouldConditionalInline = 708 | this.configuration.ConditionalInlining && 709 | (tag !== "any"); 710 | 711 | if (value === null) { 712 | if ( 713 | shouldConditionalInline 714 | ) { 715 | if (TraceInlining) 716 | console.log("INLINED=null " + tag); 717 | 718 | this.inlinedObjects += 1; 719 | this.writeInliningFlagAndIndex(writer, 0xFF, 0xFFFFFFFF); 720 | } else { 721 | this.notInlinedObjects += 1; 722 | writer.writeIndex(0xFFFFFFFF); 723 | } 724 | 725 | return true; 726 | } 727 | 728 | var actualValueTag = this.getTypeTagForValue(value); 729 | var table = this.getTableForTypeTag(tag, actualValueTag); 730 | if (!table) 731 | throw new Error("No table for value with tag '" + tag + "'"); 732 | 733 | var isUntypedObject = (tag === "object"); 734 | 735 | if ((actualValueTag !== tag) && !isUntypedObject) 736 | throw new Error("Shape information specified type '" + tag + "' but actual type is '" + actualValueTag + "'"); 737 | 738 | if (IoTrace) { 739 | if (tag !== actualValueTag) 740 | console.log(writer.description + " write " + tag + " -> " + actualValueTag); 741 | else 742 | console.log(writer.description + " write " + tag); 743 | } 744 | 745 | try { 746 | var id, index; 747 | try { 748 | id = table.get_id(value); 749 | index = id.get_index(); 750 | } catch (e) { 751 | console.log("Failed to get id for value", value, "with tag", actualValueTag, "(expected", tag, ") from table", table.semantic); 752 | throw e; 753 | } 754 | 755 | if (shouldConditionalInline) { 756 | 757 | if (id.is_omitted()) { 758 | if (TraceInlining) 759 | console.log("INLINED=1 " + tag); 760 | 761 | var name = id.get_name(); 762 | var shape = this.getShapeForObject(value); 763 | 764 | if (shape) { 765 | // HACK 766 | var prior = IoTrace; 767 | if (TraceInlining || (TraceInlinedTypeCount > 0)) 768 | IoTrace = false; 769 | 770 | this.inlinedObjects += 1; 771 | // serializeObjectContents calls writeInliningFlag 772 | this.serializeObjectContents(writer, value, true, true); 773 | IoTrace = prior; 774 | 775 | return true; 776 | 777 | } else { 778 | throw new Error("Object without shape was omitted"); 779 | } 780 | 781 | } else { 782 | if (TraceInlining) 783 | console.log("INLINED=0 " + tag); 784 | 785 | this.notInlinedObjects += 1; 786 | 787 | this.writeInliningFlagAndIndex(writer, 0, index); 788 | } 789 | 790 | } else { 791 | writer.writeIndex(index); 792 | } 793 | 794 | } catch (err) { 795 | console.log("Failed while writing '" + tag + "'", value); 796 | throw err; 797 | } 798 | 799 | return false; 800 | }; 801 | 802 | 803 | JsAstModule.prototype.serializeTable = function (writer, table, ordered, serializeEntry) { 804 | var finalized = table.finalize(0); 805 | 806 | writer.writeUint32(finalized.length); 807 | 808 | for (var i = 0, l = finalized.length; i < l; i++) { 809 | var id = finalized[i]; 810 | var value = id.get_value(); 811 | 812 | // gross 813 | serializeEntry.call(this, writer, value); 814 | } 815 | }; 816 | 817 | 818 | JsAstModule.prototype.finalize = function () { 819 | this.tags .finalize(0); 820 | this.objects.finalize(0); 821 | }; 822 | 823 | 824 | function JsAstModuleBuilder (configuration, shapes) { 825 | AsmlikeJsonTreeBuilder.call(this); 826 | 827 | this.configuration = configuration; 828 | this.shapes = shapes; 829 | 830 | this.result = new JsAstModule(configuration, shapes); 831 | 832 | this.internSymbols = configuration.InternedSymbols; 833 | this.localSymbolsBeforeGlobals = configuration.LocalSymbolsBeforeGlobals; 834 | 835 | this.walkedCount = 0; 836 | this.progressInterval = 20000; 837 | }; 838 | 839 | JsAstModuleBuilder.prototype = Object.create(AsmlikeJsonTreeBuilder.prototype); 840 | var _make = AsmlikeJsonTreeBuilder.prototype.make; 841 | 842 | JsAstModuleBuilder.prototype.make = function (key) { 843 | var result = _make.call(this, key); 844 | 845 | // FIXME: This is a slow operation. RIP 846 | Object.defineProperty(result, "__id__", { 847 | configurable: false, 848 | enumerable: false, 849 | value: NextObjectId() 850 | }); 851 | 852 | return result; 853 | }; 854 | 855 | JsAstModuleBuilder.prototype.finalizeArray = function (array) { 856 | if (array.length === 0) 857 | return; 858 | 859 | var commonTypeTag = null; 860 | 861 | for (var i = 0, l = array.length; i < l; i++) { 862 | var item = array[i]; 863 | var tag = this.result.getTypeTagForValue(item); 864 | 865 | // HACK: We know that we don't have primitives mixed in 866 | // with objects in arrays, so we can cheat here. 867 | if ( 868 | (tag !== "string") && 869 | (tag !== "integer") && 870 | (tag !== "double") && 871 | (tag !== "symbol") && 872 | !Array.isArray(item) 873 | ) { 874 | // console.log(tag, "-> object"); 875 | tag = "object"; 876 | } 877 | 878 | if (commonTypeTag === null) { 879 | commonTypeTag = tag; 880 | } else if (commonTypeTag !== tag) { 881 | console.log("tag mismatch", commonTypeTag, tag); 882 | commonTypeTag = null; 883 | break; 884 | } 885 | } 886 | 887 | if (commonTypeTag === null) { 888 | console.log("any tag fallback for", array); 889 | commonTypeTag = "any"; 890 | } 891 | 892 | this.result.arrayTypeTags.set(array, commonTypeTag); 893 | 894 | for (var i = 0, l = array.length; i < l; i++) { 895 | var element = array[i]; 896 | 897 | if (Array.isArray(element)) 898 | this.finalizeArray(element); 899 | else if (typeof (element) !== "object") 900 | this.finalizeValue(element); 901 | } 902 | 903 | return array; 904 | }; 905 | 906 | JsAstModuleBuilder.prototype.finalizeValue = function (value) { 907 | var tag = this.result.getTypeTagForValue(value); 908 | this.result.tags.add(tag); 909 | 910 | var table = this.result.getTableForTypeTag(tag); 911 | if (table) 912 | table.add(value); 913 | 914 | return value; 915 | }; 916 | 917 | JsAstModuleBuilder.prototype.finalize = function (node) { 918 | if (!node) 919 | return; 920 | 921 | this.walkedCount++; 922 | if ((this.walkedCount % this.progressInterval) === 0) { 923 | if (typeof (process) === "object") 924 | process.stderr.write("."); 925 | } 926 | 927 | if (Array.isArray(node)) { 928 | return this.finalizeArray(node); 929 | } else { 930 | for (var k in node) { 931 | if (!node.hasOwnProperty(k)) 932 | continue; 933 | 934 | var value = node[k]; 935 | 936 | // Handle non-object properties 937 | if (typeof (value) !== "object") 938 | this.finalizeValue(value); 939 | else if (Array.isArray(value)) 940 | this.finalizeArray(value); 941 | 942 | // Otherwise, it's been finalized since it was returned 943 | // by the builder. 944 | } 945 | 946 | return this.finalizeValue(node); 947 | } 948 | }; 949 | 950 | JsAstModuleBuilder.prototype.finish = function (root) { 951 | if (this.getHitCount) { 952 | var self = this; 953 | this.result.objects.forEach(function (id) { 954 | var hitCount = self.getHitCount(id.get_value()); 955 | id.set_hit_count(hitCount); 956 | }); 957 | } 958 | 959 | var rootTag = this.result.getTypeTagForValue(root); 960 | var rootTable = this.result.getTableForTypeTag(rootTag); 961 | rootTable.add(root); 962 | 963 | this.result.root = root; 964 | this.result.externalSymbolCount = this.externalSymbolCount; 965 | this.result.maxLocalSymbolCount = this.maxLocalSymbolCount; 966 | 967 | return this.result; 968 | }; 969 | 970 | 971 | // Converts a JsAstModule into bytes and writes them into byteWriter. 972 | function serializeModule (module) { 973 | var writer = new ValueWriter(module.configuration, "module"); 974 | 975 | writer.writeBytes(common.Magic); 976 | 977 | var omitCount = 0, totalCount = module.objects.get_count(); 978 | if (module.configuration.ConditionalInlining) { 979 | var self = this; 980 | var maybeOmitCallback = function (id) { 981 | var hitCount = id.get_hit_count(); 982 | var shape = module.getShapeForObject(id.get_value()); 983 | var omitDueToHitCount = hitCount <= module.configuration.InlineUseCountThreshold, 984 | omitDueToSize = false; 985 | 986 | if (shape) 987 | omitDueToSize = module.isShapeSmall(shape); 988 | 989 | if (omitDueToHitCount || omitDueToSize) { 990 | module.objects.omit(id); 991 | omitCount += 1; 992 | } 993 | }; 994 | 995 | module.objects.forEach(maybeOmitCallback); 996 | } 997 | 998 | module.finalize(); 999 | 1000 | // We write out the lengths in advance of the (length-prefixed) tables. 1001 | // This allows a decoder to preallocate space for all the tables and 1002 | // use that to reconstruct relationships in a single pass. 1003 | 1004 | var tagCount = module.tags.get_count(); 1005 | var objectCount = module.objects.get_count(); 1006 | 1007 | writer.writeUint32(tagCount); 1008 | writer.writeUint32(objectCount); 1009 | 1010 | var tagWriter = new ValueWriter(module.configuration, "tag table", writer); 1011 | module.serializeTable(tagWriter, module.tags, true, function (_, value) { 1012 | _.writeUtf8String(value); 1013 | }); 1014 | 1015 | if ( 1016 | module.configuration.ConditionalInlining && 1017 | !module.configuration.PackedInliningFlags 1018 | ) 1019 | module.inliningWriter = new ValueWriter(module.configuration, "inlining flags", writer); 1020 | 1021 | if (module.configuration.TypeTagStream) 1022 | module.typeTagWriter = new ValueWriter(module.configuration, "type tags", writer); 1023 | 1024 | if ( 1025 | module.configuration.PackedInliningFlags && 1026 | module.configuration.PackedIndexStream 1027 | ) 1028 | module.packedIndexWriter = new ValueWriter(module.configuration, "packed indices", writer); 1029 | 1030 | if (module.configuration.ValueStreamPerType) 1031 | module.createValueStreams(writer); 1032 | 1033 | var objectWriter = new ValueWriter(module.configuration, "object table", writer); 1034 | module.serializeTable(objectWriter, module.objects, true, module.serializeObjectContents); 1035 | 1036 | 1037 | var rootWriter = new ValueWriter(module.configuration, "root node", writer); 1038 | module.serializeValueWithKnownTag(rootWriter, module.root, "any"); 1039 | 1040 | 1041 | writer.writeSubstream(tagWriter); 1042 | 1043 | 1044 | if (module.inliningWriter) 1045 | writer.writeSubstream(module.inliningWriter); 1046 | 1047 | if (module.typeTagWriter) 1048 | writer.writeSubstream(module.typeTagWriter); 1049 | 1050 | if (module.packedIndexWriter) 1051 | writer.writeSubstream(module.packedIndexWriter); 1052 | 1053 | 1054 | if (module.configuration.ValueStreamPerType) 1055 | for (var key in module.valueStreams) { 1056 | var valueStream = module.valueStreams[key]; 1057 | writer.writeIndex(module.tags.get_index(key)); 1058 | writer.writeSubstream(valueStream); 1059 | } 1060 | 1061 | 1062 | writer.writeSubstream(objectWriter); 1063 | writer.writeSubstream(rootWriter); 1064 | 1065 | if (module.configuration.ConditionalInlining) { 1066 | var total = module.notInlinedObjects + module.inlinedObjects; 1067 | 1068 | console.log( 1069 | module.inlinedObjects + "/" + 1070 | total + " object references written inline (" + 1071 | (module.inlinedObjects * 100 / total).toFixed(1) + "%)" 1072 | ); 1073 | } 1074 | 1075 | if (omitCount > 0) { 1076 | console.log( 1077 | omitCount + "/" + totalCount + 1078 | " objects omitted from object table (" + (omitCount * 100 / totalCount).toFixed(1) + "%)" 1079 | ); 1080 | } 1081 | 1082 | if (module.configuration.TypeTagStream) 1083 | console.log((module.typeTagsWritten / 1000).toFixed(2) + "k type tags written"); 1084 | 1085 | if (module.configuration.EnableVarints) { 1086 | var varintSizes = ""; 1087 | var totalVarintSize = 0; 1088 | for (var i = 0; i < writer.varintSizes.length; i++) { 1089 | totalVarintSize += (i + 1) * writer.varintSizes[i]; 1090 | 1091 | if (writer.varintSizes[i]) { 1092 | if (varintSizes.length) 1093 | varintSizes += ", "; 1094 | 1095 | varintSizes += (i + 1) + "b " + (writer.varintSizes[i] / 1000).toFixed(2) + "k"; 1096 | } 1097 | } 1098 | 1099 | console.log( 1100 | (totalVarintSize / 1024).toFixed(1) + "KiB varints written (" + 1101 | varintSizes + ")" 1102 | ); 1103 | } 1104 | 1105 | if (module.configuration.InternedSymbols) { 1106 | console.log( 1107 | module.externalSymbolCount + " externally visible symbols, " + 1108 | module.maxLocalSymbolCount + " max local symbols" 1109 | ); 1110 | } 1111 | 1112 | return writer.toArray(); 1113 | }; 1114 | 1115 | 1116 | exports.PrettyJson = common.PrettyJson; 1117 | exports.ShapeTable = common.ShapeTable; 1118 | exports.JsAstModuleBuilder = JsAstModuleBuilder; 1119 | 1120 | exports.serializeModule = serializeModule; 1121 | })); --------------------------------------------------------------------------------