├── .npmignore ├── assets └── powers-dre.png ├── rollup.config.js ├── package.json ├── LICENSE ├── .gitignore ├── index.d.ts ├── README.md ├── tests └── test.js └── index.js /.npmignore: -------------------------------------------------------------------------------- 1 | # Dependency directories 2 | node_modules/ 3 | -------------------------------------------------------------------------------- /assets/powers-dre.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kriszyp/ordered-binary/master/assets/powers-dre.png -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | export default [ 2 | { 3 | input: "index.js", 4 | output: [ 5 | { 6 | file: "dist/index.cjs", 7 | format: "cjs" 8 | } 9 | ] 10 | } 11 | ]; 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ordered-binary", 3 | "author": "Kris Zyp", 4 | "version": "1.6.0", 5 | "description": "Conversion of JavaScript primitives to and from Buffer with binary order matching natural primitive order", 6 | "license": "MIT", 7 | "repository": { 8 | "type": "git", 9 | "url": "http://github.com/kriszyp/ordered-binary" 10 | }, 11 | "scripts": { 12 | "build": "rollup -c", 13 | "prepare": "rollup -c", 14 | "test": "mocha tests -u tdd" 15 | }, 16 | "type": "module", 17 | "main": "dist/index.cjs", 18 | "module": "index.js", 19 | "exports": { 20 | ".": { 21 | "require": "./dist/index.cjs", 22 | "import": "./index.js" 23 | }, 24 | "./index.js": { 25 | "require": "./dist/index.cjs", 26 | "import": "./index.js" 27 | } 28 | }, 29 | "typings": "./index.d.ts", 30 | "optionalDependencies": {}, 31 | "devDependencies": { 32 | "@types/node": "latest", 33 | "chai": "^4", 34 | "mocha": "^9.2.0", 35 | "rollup": "^2.61.1" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Kris Zyp 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | dist 8 | 9 | # Runtime data 10 | pids 11 | *.pid 12 | *.seed 13 | *.pid.lock 14 | 15 | # Directory for instrumented libs generated by jscoverage/JSCover 16 | lib-cov 17 | 18 | # Coverage directory used by tools like istanbul 19 | coverage 20 | 21 | # nyc test coverage 22 | .nyc_output 23 | 24 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 25 | .grunt 26 | 27 | # Bower dependency directory (https://bower.io/) 28 | bower_components 29 | 30 | # node-waf configuration 31 | .lock-wscript 32 | 33 | # Compiled binary addons (http://nodejs.org/api/addons.html) 34 | build/Release 35 | 36 | # Dependency directories 37 | node_modules/ 38 | jspm_packages/ 39 | 40 | # Typescript v1 declaration files 41 | typings/ 42 | 43 | # Optional npm cache directory 44 | .npm 45 | 46 | # Optional eslint cache 47 | .eslintcache 48 | 49 | # Optional REPL history 50 | .node_repl_history 51 | 52 | # Output of 'npm pack' 53 | *.tgz 54 | 55 | # Yarn Integrity file 56 | .yarn-integrity 57 | 58 | yarn.lock 59 | package-lock.json 60 | 61 | .idea 62 | 63 | # dotenv environment variables file 64 | .env 65 | 66 | -------------------------------------------------------------------------------- /index.d.ts: -------------------------------------------------------------------------------- 1 | type Key = Key[] | string | symbol | number | boolean | Uint8Array; 2 | /** Writes a key (a primitive value) to the target buffer, starting at the given position */ 3 | export function writeKey(key: Key, target: Uint8Array, position: number, inSequence?: boolean): number; 4 | /** Reads a key from the provided buffer, from the given range */ 5 | export function readKey(buffer: Uint8Array, start: number, end?: number, inSequence?: boolean): Key; 6 | /** Converts key to a Buffer. This is generally much slower than using writeKey since it involves a full buffer allocation, and should be avoided for performance sensitive code. */ 7 | export function toBufferKey(key: Key): Buffer; 8 | /** Converts Buffer to Key */ 9 | export function fromBufferKey(source: Buffer): Key; 10 | /** Compares two keys, returning -1 if `a` comes before `b` in the ordered binary representation of the keys, or 1 if `a` comes after `b`, or 0 if they are equivalent */ 11 | export function compareKeys(a: Key, b: Key): number; 12 | /** The minimum key, with the "first" binary representation (one byte of zero) */ 13 | export const MINIMUM_KEY: null 14 | /** A maximum key, with a binary representation after all other JS primitives (one byte of 0xff) */ 15 | export const MAXIMUM_KEY: Uint8Array 16 | /** Enables null termination, ensuring that writing keys to buffers will end with a padding of zeros at the end to complete the following 32-bit word */ 17 | export function enableNullTermination(): void; 18 | /** An object that holds the functions for encapsulation as a single encoder */ 19 | export const encoder: { 20 | writeKey: typeof writeKey, 21 | readKey: typeof readKey, 22 | enableNullTermination: typeof enableNullTermination, 23 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![npm version](https://img.shields.io/npm/dw/ordered-binary)](https://www.npmjs.org/package/ordered-binary) 2 | [![npm version](https://img.shields.io/npm/v/ordered-binary.svg?style=flat-square)](https://www.npmjs.org/package/ordered-binary) 3 | [![license](https://img.shields.io/badge/license-MIT-brightgreen)](LICENSE) 4 | 5 | The ordered-binary package provides a representation of JavaScript primitives, serialized into binary format (NodeJS Buffers or Uint8Arrays), such that the binary values are naturally ordered such that it matches the natural ordering or values. For example, since -2.0321 > -2.04, then `toBufferKey(-2.0321)` will be greater than `toBufferKey(-2.04)` as a binary representation, in left-to-right evaluation. This is particular useful for storing keys as binaries with something like LMDB or LevelDB, to avoid any custom sorting. 6 | 7 | The ordered-binary package supports strings, numbers, booleans, symbols, null, as well as an array of primitives. Here is an example of ordering of primitive values: 8 | ``` 9 | Buffer.from([0]) // buffers are left unchanged, and this is the minimum value 10 | Symbol.for('even symbols') 11 | -10 // negative supported 12 | -1.1 // decimals supported 13 | 400 14 | 3E10 15 | 'Hello' 16 | ['Hello', 'World'] 17 | 'World' 18 | 'hello' 19 | ['hello', 1, 'world'] 20 | ['hello', 'world'] 21 | Buffer.from([0xff]) 22 | ``` 23 | 24 | 25 | The main module exports these functions: 26 | 27 | `writeKey(key: string | number | boolean | null | Array, target: Buffer, position: integer, inSequence?: boolean)` - Writes the provide key to the target buffer 28 | 29 | `readKey(buffer, start, end, inSequence)` - Reads the key from the buffer, given the provided start and end, as a primitive value 30 | 31 | `toBufferKey(jsPrimitive)` - This accepts a string, number, or boolean as the argument, and returns a `Buffer`. 32 | 33 | `fromBufferKey(bufferKey, multiple)` - This accepts a Buffer and returns a JavaScript primitive value. This can also parse buffers that hold multiple values delimited by a byte `30`, by setting the second argument to true (in which case it will return an array). 34 | 35 | And these constants: 36 | 37 | `MINIMUM_KEY` - The minimum key supported (`null`, which is represented as single zero byte) 38 | `MAXIMUM_KEY` - A maximum key larger than any supported primitive (single 0xff byte) 39 | -------------------------------------------------------------------------------- /tests/test.js: -------------------------------------------------------------------------------- 1 | import { assert } from 'chai' 2 | 3 | import { toBufferKey, fromBufferKey, readKey, writeKey } from '../index.js' 4 | 5 | function assertBufferComparison(lesser, greater) { 6 | for (let i = 0; i < lesser.length; i++) { 7 | if (lesser[i] < greater[i]) { 8 | return 9 | } 10 | if (lesser[i] > (greater[i] || 0)) { 11 | assert.fail('Byte ' + i + 'should not be ' + lesser[i] + '>' + greater[i]) 12 | } 13 | } 14 | } 15 | //var inspector = require('inspector'); inspector.open(9330, null, true); debugger 16 | let seed = 0; 17 | export function random() { 18 | seed++; 19 | let a = seed * 15485863; 20 | return ((a * a * a) % 2038074743) / 2038074743; 21 | } 22 | 23 | suite('key buffers', () => { 24 | 25 | test('numbers equivalence', () => { 26 | assert.strictEqual(fromBufferKey(toBufferKey(4)), 4) 27 | assert.strictEqual(fromBufferKey(toBufferKey(-4)), -4) 28 | assert.strictEqual(fromBufferKey(toBufferKey(3.4)), 3.4) 29 | assert.strictEqual(fromBufferKey(toBufferKey(Math.PI)), Math.PI) 30 | assert.strictEqual(fromBufferKey(toBufferKey(2002225)), 2002225) 31 | assert.strictEqual(fromBufferKey(toBufferKey(9377288)), 9377288) 32 | assert.strictEqual(fromBufferKey(toBufferKey(1503579323825)), 1503579323825) 33 | assert.strictEqual(fromBufferKey(toBufferKey(1503579323825.3523532)), 1503579323825.3523532) 34 | assert.strictEqual(fromBufferKey(toBufferKey(-1503579323825)), -1503579323825) 35 | assert.strictEqual(fromBufferKey(toBufferKey(0.00005032)), 0.00005032) 36 | assert.strictEqual(fromBufferKey(toBufferKey(-0.00005032)), -0.00005032) 37 | assert.strictEqual(fromBufferKey(toBufferKey(0.00000000000000000000000005431)), 0.00000000000000000000000005431) 38 | }) 39 | test('within buffer equivalence', () => { 40 | let buffer = Buffer.alloc(1024, 0xff); 41 | let length = writeKey(2002225, buffer, 0); 42 | let position = length; 43 | buffer[position++] = 0xff; 44 | buffer[position++] = 0xff; 45 | assert.strictEqual(readKey(buffer, 0, length), 2002225); 46 | length = writeKey('hello, world', buffer, 0); 47 | assert.strictEqual(readKey(buffer, 0, length), 'hello, world'); 48 | // ensure that reading the string didn't modify the buffer after the string (or is restored at least) 49 | assert.strictEqual(buffer[length], 0xff); 50 | }); 51 | test('number comparison', () => { 52 | assertBufferComparison(toBufferKey(4), toBufferKey(5)) 53 | assertBufferComparison(toBufferKey(1503579323824), toBufferKey(1503579323825)) 54 | assertBufferComparison(toBufferKey(1.4), toBufferKey(2)) 55 | assertBufferComparison(toBufferKey(0.000000001), toBufferKey(0.00000001)) 56 | assertBufferComparison(toBufferKey(-4), toBufferKey(3)) 57 | assertBufferComparison(toBufferKey(0), toBufferKey(1)) 58 | assertBufferComparison(toBufferKey(-0.001), toBufferKey(0)) 59 | assertBufferComparison(toBufferKey(-0.001), toBufferKey(-0.000001)) 60 | assertBufferComparison(toBufferKey(-5236532532532), toBufferKey(-5236532532531)) 61 | }) 62 | test('bigint equivalence', () => { 63 | assert.strictEqual(fromBufferKey(toBufferKey(-35913040084491349n)), -35913040084491349n) 64 | assert.strictEqual(fromBufferKey(toBufferKey(6135421331404949076605986n)), 6135421331404949076605986n) 65 | assert.strictEqual(fromBufferKey(toBufferKey(0xfffffffffffffffffffffn)), 0xfffffffffffffffffffffn) 66 | assert.strictEqual(fromBufferKey(toBufferKey(12345678901234567890n)), 12345678901234567890n) 67 | assert.deepEqual(fromBufferKey(toBufferKey([12345678901234567890n, 44])), [12345678901234567890n, 44]) 68 | assert.deepEqual(fromBufferKey(toBufferKey(['hi', 12345678901234567890n])), ['hi', 12345678901234567890n]) 69 | assert.deepEqual(fromBufferKey(toBufferKey([6135421331404949076605986n, 'after'])), [6135421331404949076605986n, 'after']) 70 | assert.strictEqual(fromBufferKey(toBufferKey(132923456789012345678903533235253252353211125n)), 132923456789012345678903533235253252353211125n) 71 | assert.strictEqual(fromBufferKey(toBufferKey(352n)), 352) 72 | let num = 5325n 73 | for (let i = 0; i < 1100; i++) { 74 | num *= BigInt(Math.floor(random() * 3 + 1)); 75 | num -= BigInt(Math.floor(random() * 1000)); 76 | assert.strictEqual(BigInt(fromBufferKey(toBufferKey(num))), num) 77 | assert.strictEqual(BigInt(fromBufferKey(toBufferKey(-num))), -num) 78 | } 79 | assert.strictEqual(fromBufferKey(toBufferKey(-352n)), -352) 80 | }) 81 | test('bigint comparison', () => { 82 | assertBufferComparison(toBufferKey(0xfffffffffffffffffffffn), toBufferKey(0x100fffffffffffffffffffn)) 83 | assertBufferComparison(toBufferKey(12345678901234567890), toBufferKey(12345678901234567890n)) 84 | assertBufferComparison(toBufferKey(6135421331404949076605986n), toBufferKey(6135421331404949076605987n)) 85 | assertBufferComparison(toBufferKey(-6135421331404949076605986n), toBufferKey(-6135421331404949076605985n)) 86 | assertBufferComparison(toBufferKey(-35913040084491349n), toBufferKey(-35913040084491348n)) 87 | }) 88 | 89 | test('string equivalence', () => { 90 | assert.strictEqual(fromBufferKey(toBufferKey('4')), '4') 91 | assert.strictEqual(fromBufferKey(toBufferKey('hello')), 'hello') 92 | assert.strictEqual(fromBufferKey(toBufferKey('')), '') 93 | assert.strictEqual(fromBufferKey(toBufferKey('\x00')), '\x00') 94 | assert.strictEqual(fromBufferKey(toBufferKey('\x03test\x01\x00')), '\x03test\x01\x00') 95 | assert.strictEqual(fromBufferKey(toBufferKey('prance 🧚🏻‍♀️🩷')), 'prance 🧚🏻‍♀️🩷') 96 | }) 97 | test('string comparison', () => { 98 | assertBufferComparison(toBufferKey('4'), toBufferKey('5')) 99 | assertBufferComparison(toBufferKey('and'), toBufferKey('bad')) 100 | assertBufferComparison(toBufferKey('hello'), toBufferKey('hello2')) 101 | let buffer = Buffer.alloc(1024) 102 | let end = writeKey(['this is a test', 5.25], buffer, 0) 103 | }) 104 | test('boolean equivalence', () => { 105 | assert.strictEqual(fromBufferKey(toBufferKey(true)), true) 106 | assert.strictEqual(fromBufferKey(toBufferKey(false)), false) 107 | }) 108 | 109 | test('multipart equivalence', () => { 110 | assert.deepEqual(fromBufferKey(toBufferKey([4, 5])), 111 | [4, 5]) 112 | assert.deepEqual(fromBufferKey(toBufferKey(['hello', 5.25])), 113 | ['hello', 5.25]) 114 | assert.deepEqual(fromBufferKey(toBufferKey([5, 'hello', null, 5.25])), 115 | [5, 'hello', null, 5.25]) 116 | assert.deepEqual(fromBufferKey(toBufferKey([true, 1503579323825])), 117 | [true, 1503579323825]) 118 | assert.deepEqual(fromBufferKey(toBufferKey([-0.2525, 'sec\x00nd'])), 119 | [-0.2525, 'sec\x00nd']) 120 | assert.deepEqual(fromBufferKey(toBufferKey([-0.2525, '2nd', '3rd'])), 121 | [-0.2525, '2nd', '3rd']) 122 | }) 123 | 124 | test('multipart comparison', () => { 125 | assertBufferComparison( 126 | Buffer.concat([toBufferKey(4), Buffer.from([30]), toBufferKey(5)]), 127 | Buffer.concat([toBufferKey(5), Buffer.from([30]), toBufferKey(5)])) 128 | assertBufferComparison( 129 | Buffer.concat([toBufferKey(4), Buffer.from([30]), toBufferKey(5)]), 130 | Buffer.concat([toBufferKey(4), Buffer.from([30]), toBufferKey(6)])) 131 | assertBufferComparison( 132 | Buffer.concat([toBufferKey('and'), Buffer.from([30]), toBufferKey(5)]), 133 | Buffer.concat([toBufferKey('and2'), Buffer.from([30]), toBufferKey(5)])) 134 | assertBufferComparison( 135 | Buffer.concat([toBufferKey(4), Buffer.from([30]), toBufferKey('and')]), 136 | Buffer.concat([toBufferKey(4), Buffer.from([30]), toBufferKey('cat')])) 137 | }) 138 | test('performance', () => { 139 | let buffer = Buffer.alloc(1024) 140 | let start = process.hrtime.bigint() 141 | let end, value 142 | for (let i = 0; i < 1000000; i++) { 143 | end = writeKey('this is a test of a longer string to read and write', buffer, 0) 144 | } 145 | console.log('writeKey string time', nextTime(), end) 146 | for (let i = 0; i < 1000000; i++) { 147 | value = readKey(buffer, 0, end) 148 | } 149 | console.log('readKey string time', nextTime(), value) 150 | 151 | for (let i = 0; i < 1000000; i++) { 152 | end = writeKey(33456, buffer, 0) 153 | } 154 | console.log('writeKey number time', nextTime(), end) 155 | 156 | for (let i = 0; i < 1000000; i++) { 157 | value = readKey(buffer, 2, end) 158 | } 159 | console.log('readKey number time', nextTime(), value) 160 | 161 | for (let i = 0; i < 1000000; i++) { 162 | end = writeKey(['hello', 33456], buffer, 0) 163 | } 164 | console.log('writeKey array time', nextTime(), end, buffer.slice(0, end)) 165 | 166 | for (let i = 0; i < 1000000; i++) { 167 | value = readKey(buffer, 0, end) 168 | } 169 | console.log('readKey array time', nextTime(), value) 170 | 171 | function nextTime() { 172 | let ns = process.hrtime.bigint() 173 | let elapsed = ns - start 174 | start = ns 175 | return Number(elapsed) / 1000000 + 'ns' 176 | } 177 | }) 178 | 179 | }) 180 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /* 2 | control character types: 3 | 1 - metadata 4 | 2 - symbols 5 | 6 - false 6 | 7 - true 7 | 8- 16 - negative doubles 8 | 16-24 positive doubles 9 | 27 - String starts with a character 27 or less or is an empty string 10 | 0 - multipart separator 11 | > 27 normal string characters 12 | */ 13 | /* 14 | * Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering 15 | */ 16 | 17 | const float64Array = new Float64Array(2) 18 | const int32Array = new Int32Array(float64Array.buffer, 0, 4) 19 | const {lowIdx, highIdx} = (() => { 20 | if (new Uint8Array(new Uint32Array([0xFFEE1100]).buffer)[0] === 0xFF) { 21 | return { lowIdx: 1, highIdx: 0 }; 22 | } 23 | return { lowIdx: 0, highIdx: 1 }; 24 | })(); 25 | let nullTerminate = false 26 | let textEncoder 27 | try { 28 | textEncoder = new TextEncoder() 29 | } catch (error) {} 30 | 31 | /* 32 | * Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering 33 | */ 34 | export function writeKey(key, target, position, inSequence) { 35 | let targetView = target.dataView 36 | if (!targetView) 37 | targetView = target.dataView = new DataView(target.buffer, target.byteOffset, ((target.byteLength + 3) >> 2) << 2) 38 | switch (typeof key) { 39 | case 'string': 40 | let strLength = key.length 41 | let c1 = key.charCodeAt(0) 42 | if (!(c1 >= 28)) // escape character 43 | target[position++] = 27 44 | if (strLength < 0x40) { 45 | let i, c2 46 | for (i = 0; i < strLength; i++) { 47 | c1 = key.charCodeAt(i) 48 | if (c1 <= 4) { 49 | target[position++] = 4 50 | target[position++] = c1 51 | } else if (c1 < 0x80) { 52 | target[position++] = c1 53 | } else if (c1 < 0x800) { 54 | target[position++] = c1 >> 6 | 0xc0 55 | target[position++] = c1 & 0x3f | 0x80 56 | } else if ( 57 | (c1 & 0xfc00) === 0xd800 && 58 | ((c2 = key.charCodeAt(i + 1)) & 0xfc00) === 0xdc00 59 | ) { 60 | c1 = 0x10000 + ((c1 & 0x03ff) << 10) + (c2 & 0x03ff) 61 | i++ 62 | target[position++] = c1 >> 18 | 0xf0 63 | target[position++] = c1 >> 12 & 0x3f | 0x80 64 | target[position++] = c1 >> 6 & 0x3f | 0x80 65 | target[position++] = c1 & 0x3f | 0x80 66 | } else { 67 | target[position++] = c1 >> 12 | 0xe0 68 | target[position++] = c1 >> 6 & 0x3f | 0x80 69 | target[position++] = c1 & 0x3f | 0x80 70 | } 71 | } 72 | } else { 73 | if (target.utf8Write) 74 | position += target.utf8Write(key, position, target.byteLength - position) 75 | else 76 | position += textEncoder.encodeInto(key, target.subarray(position)).written 77 | if (position > target.length - 4) 78 | throw new RangeError('String does not fit in target buffer') 79 | } 80 | break 81 | case 'number': 82 | float64Array[0] = key 83 | let lowInt = int32Array[lowIdx] 84 | let highInt = int32Array[highIdx] 85 | let length 86 | if (key < 0) { 87 | targetView.setInt32(position + 4, ~((lowInt >>> 4) | (highInt << 28))) 88 | targetView.setInt32(position + 0, (highInt ^ 0x7fffffff) >>> 4) 89 | targetView.setInt32(position + 8, ((lowInt & 0xf) ^ 0xf) << 4, true) // just always do the null termination here 90 | return position + 9 91 | } else if ((lowInt & 0xf) || inSequence) { 92 | length = 9 93 | } else if (lowInt & 0xfffff) 94 | length = 8 95 | else if (lowInt || (highInt & 0xf)) 96 | length = 6 97 | else 98 | length = 4 99 | // switching order to go to little endian 100 | targetView.setInt32(position + 0, (highInt >>> 4) | 0x10000000) 101 | targetView.setInt32(position + 4, (lowInt >>> 4) | (highInt << 28)) 102 | // if (length == 9 || nullTerminate) 103 | targetView.setInt32(position + 8, (lowInt & 0xf) << 4, true) 104 | return position + length; 105 | case 'object': 106 | if (key) { 107 | if (Array.isArray(key)) { 108 | for (let i = 0, l = key.length; i < l; i++) { 109 | if (i > 0) 110 | target[position++] = 0 111 | position = writeKey(key[i], target, position, true) 112 | } 113 | break 114 | } else if (key instanceof Uint8Array) { 115 | target.set(key, position) 116 | position += key.length 117 | break 118 | } else { 119 | throw new Error('Unable to serialize object as a key: ' + JSON.stringify(key)) 120 | } 121 | } else // null 122 | target[position++] = 0 123 | break 124 | case 'boolean': 125 | targetView.setUint32(position++, key ? 7 : 6, true) 126 | return position 127 | case 'bigint': 128 | let asFloat = Number(key) 129 | if (BigInt(asFloat) > key) { 130 | float64Array[0] = asFloat; 131 | if (asFloat > 0) { 132 | if (int32Array[lowIdx]) 133 | int32Array[lowIdx]--; 134 | else { 135 | int32Array[highIdx]--; 136 | int32Array[lowIdx] = 0xffffffff; 137 | } 138 | } else { 139 | if (int32Array[lowIdx] < 0xffffffff) 140 | int32Array[lowIdx]++; 141 | else { 142 | int32Array[highIdx]++; 143 | int32Array[lowIdx] = 0; 144 | } 145 | } 146 | asFloat = float64Array[0]; 147 | } 148 | let difference = key - BigInt(asFloat); 149 | if (difference === 0n) 150 | return writeKey(asFloat, target, position, inSequence) 151 | writeKey(asFloat, target, position, inSequence) 152 | position += 9; // always increment by 9 if we are adding fractional bits 153 | let exponent = BigInt((int32Array[highIdx] >> 20 & 0x7ff) - 1079); 154 | let nextByte = difference >> exponent; 155 | target[position - 1] |= Number(nextByte); 156 | difference -= nextByte << exponent; 157 | let first = true; 158 | while (difference || first) { 159 | first = false; 160 | exponent -= 7n; 161 | let nextByte = difference >> exponent; 162 | target[position++] = Number(nextByte) | 0x80; 163 | difference -= nextByte << exponent; 164 | } 165 | return position; 166 | case 'undefined': 167 | return position 168 | // undefined is interpreted as the absence of a key, signified by zero length 169 | case 'symbol': 170 | target[position++] = 2 171 | return writeKey(key.description, target, position, inSequence) 172 | default: 173 | throw new Error('Can not serialize key of type ' + typeof key) 174 | } 175 | if (nullTerminate && !inSequence) 176 | targetView.setUint32(position, 0) 177 | return position 178 | } 179 | 180 | let position 181 | export function readKey(buffer, start, end, inSequence) { 182 | position = start 183 | let controlByte = buffer[position] 184 | let value 185 | if (controlByte < 24) { 186 | if (controlByte < 8) { 187 | position++ 188 | if (controlByte == 6) { 189 | value = false 190 | } else if (controlByte == 7) { 191 | value = true 192 | } else if (controlByte == 0) { 193 | value = null 194 | } else if (controlByte == 2) { 195 | value = Symbol.for(readStringSafely(buffer, end)) 196 | } else 197 | return Uint8Array.prototype.slice.call(buffer, start, end) 198 | } else { 199 | let dataView; 200 | try { 201 | dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2)) 202 | } catch(error) { 203 | // if it is write at the end of the ArrayBuffer, we may need to retry with the exact remaining bytes 204 | dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.buffer.byteLength - buffer.byteOffset)) 205 | } 206 | 207 | let highInt = dataView.getInt32(position) << 4 208 | let size = end - position 209 | let lowInt 210 | if (size > 4) { 211 | lowInt = dataView.getInt32(position + 4) 212 | highInt |= lowInt >>> 28 213 | if (size <= 6) { // clear the last bits 214 | lowInt &= -0x10000 215 | } 216 | lowInt = lowInt << 4 217 | if (size > 8) { 218 | lowInt = lowInt | buffer[position + 8] >> 4 219 | } 220 | } else 221 | lowInt = 0 222 | if (controlByte < 16) { 223 | // negative gets negated 224 | highInt = highInt ^ 0x7fffffff 225 | lowInt = ~lowInt 226 | } 227 | int32Array[highIdx] = highInt 228 | int32Array[lowIdx] = lowInt 229 | value = float64Array[0] 230 | position += 9 231 | if (size > 9 && buffer[position] > 0) { 232 | // convert the float to bigint, and then we will add precision as we enumerate through the 233 | // extra bytes 234 | value = BigInt(value); 235 | let exponent = highInt >> 20 & 0x7ff; 236 | let next_byte = buffer[position - 1] & 0xf; 237 | value += BigInt(next_byte) << BigInt(exponent - 1079); 238 | while ((next_byte = buffer[position]) > 0 && position++ < end) { 239 | value += BigInt(next_byte & 0x7f) << BigInt((start - position) * 7 + exponent - 1016); 240 | } 241 | } 242 | } 243 | } else { 244 | if (controlByte == 27) { 245 | position++ 246 | } 247 | value = readStringSafely(buffer, end) 248 | if (position < end) position-- // if have a null terminator for the string, count that as the array separator 249 | } 250 | while (position < end) { 251 | if (buffer[position] === 0) 252 | position++ 253 | if (inSequence) { 254 | encoder.position = position 255 | return value 256 | } 257 | let nextValue = readKey(buffer, position, end, true) 258 | if (value instanceof Array) { 259 | value.push(nextValue) 260 | } else 261 | value = [ value, nextValue ] 262 | } 263 | return value 264 | } 265 | export const enableNullTermination = () => nullTerminate = true 266 | 267 | export const encoder = { 268 | writeKey, 269 | readKey, 270 | enableNullTermination, 271 | } 272 | let targetBuffer = [] 273 | let targetPosition = 0 274 | const hasNodeBuffer = typeof Buffer !== 'undefined' 275 | const ByteArrayAllocate = hasNodeBuffer ? Buffer.allocUnsafeSlow : Uint8Array 276 | export const toBufferKey = (key) => { 277 | let newBuffer 278 | if (targetPosition + 100 > targetBuffer.length) { 279 | targetBuffer = new ByteArrayAllocate(8192) 280 | targetPosition = 0 281 | newBuffer = true 282 | } 283 | try { 284 | let result = targetBuffer.slice(targetPosition, targetPosition = writeKey(key, targetBuffer, targetPosition)) 285 | if (targetPosition > targetBuffer.length) { 286 | if (newBuffer) 287 | throw new Error('Key is too large') 288 | return toBufferKey(key) 289 | } 290 | return result 291 | } catch(error) { 292 | if (newBuffer) 293 | throw error 294 | targetPosition = targetBuffer.length 295 | return toBufferKey(key) 296 | } 297 | } 298 | export const fromBufferKey = (sourceBuffer) => { 299 | return readKey(sourceBuffer, 0, sourceBuffer.length) 300 | } 301 | const fromCharCode = String.fromCharCode 302 | function makeStringBuilder() { 303 | let stringBuildCode = '(source) => {' 304 | let previous = [] 305 | for (let i = 0; i < 0x30; i++) { 306 | let v = fromCharCode((i & 0xf) + 97) + fromCharCode((i >> 4) + 97) 307 | stringBuildCode += ` 308 | let ${v} = source[position++] 309 | if (${v} > 4) { 310 | if (${v} >= 0x80) ${v} = finishUtf8(${v}, source) 311 | } else { 312 | if (${v} === 4) 313 | ${v} = source[position++] 314 | else 315 | return fromCharCode(${previous}) 316 | } 317 | ` 318 | previous.push(v) 319 | if (i == 1000000) // this just exists to prevent rollup from doing dead code elimination on finishUtf8 320 | finishUtf8() 321 | } 322 | stringBuildCode += `return fromCharCode(${previous}) + readString(source)}` 323 | return stringBuildCode 324 | } 325 | 326 | let pendingSurrogate 327 | function finishUtf8(byte1, src) { 328 | if ((byte1 & 0xe0) === 0xc0) { 329 | // 2 bytes 330 | const byte2 = src[position++] & 0x3f 331 | return ((byte1 & 0x1f) << 6) | byte2 332 | } else if ((byte1 & 0xf0) === 0xe0) { 333 | // 3 bytes 334 | const byte2 = src[position++] & 0x3f 335 | const byte3 = src[position++] & 0x3f 336 | return ((byte1 & 0x1f) << 12) | (byte2 << 6) | byte3 337 | } else if ((byte1 & 0xf8) === 0xf0) { 338 | // 4 bytes 339 | if (pendingSurrogate) { 340 | byte1 = pendingSurrogate 341 | pendingSurrogate = null 342 | position += 3 343 | return byte1 344 | } 345 | const byte2 = src[position++] & 0x3f 346 | const byte3 = src[position++] & 0x3f 347 | const byte4 = src[position++] & 0x3f 348 | let unit = ((byte1 & 0x07) << 0x12) | (byte2 << 0x0c) | (byte3 << 0x06) | byte4 349 | if (unit > 0xffff) { 350 | pendingSurrogate = 0xdc00 | (unit & 0x3ff) 351 | unit = (((unit - 0x10000) >>> 10) & 0x3ff) | 0xd800 352 | position -= 4 // reset so we can return the next part of the surrogate pair 353 | } 354 | return unit 355 | } else { 356 | return byte1 357 | } 358 | } 359 | 360 | const readString = 361 | typeof process !== 'undefined' && process.isBun ? // the eval in bun doesn't properly closure on position, so we 362 | // have to manually update it 363 | (function(reading) { 364 | let { setPosition, getPosition, readString } = reading; 365 | return (source) => { 366 | setPosition(position); 367 | let value = readString(source); 368 | position = getPosition(); 369 | return value; 370 | }; 371 | })((new Function('fromCharCode', 'let position; let readString = ' + makeStringBuilder() + 372 | ';return {' + 373 | 'setPosition(p) { position = p },' + 374 | 'getPosition() { return position },' + 375 | 'readString }'))(fromCharCode)) : 376 | eval(makeStringBuilder()) 377 | function readStringSafely(source, end) { 378 | if (source[end] > 0) { 379 | let previous = source[end] 380 | try { 381 | // read string expects a null terminator, that is a 0 or undefined from reading past the end of the buffer, so we 382 | // have to ensure that, but do so safely, restoring the buffer to its original state 383 | source[end] = 0 384 | return readString(source) 385 | } finally { 386 | source[end] = previous 387 | } 388 | } else return readString(source); 389 | } 390 | export function compareKeys(a, b) { 391 | // compare with type consistency that matches binary comparison 392 | if (typeof a == 'object') { 393 | if (!a) { 394 | return b == null ? 0 : -1 395 | } 396 | if (a.compare) { 397 | if (b == null) { 398 | return 1 399 | } else if (b.compare) { 400 | return a.compare(b) 401 | } else { 402 | return -1 403 | } 404 | } 405 | let arrayComparison 406 | if (b instanceof Array) { 407 | let i = 0 408 | while((arrayComparison = compareKeys(a[i], b[i])) == 0 && i <= a.length) { 409 | i++ 410 | } 411 | return arrayComparison 412 | } 413 | arrayComparison = compareKeys(a[0], b) 414 | if (arrayComparison == 0 && a.length > 1) 415 | return 1 416 | return arrayComparison 417 | } else if (typeof a == typeof b) { 418 | if (typeof a === 'symbol') { 419 | a = Symbol.keyFor(a) 420 | b = Symbol.keyFor(b) 421 | } 422 | return a < b ? -1 : a === b ? 0 : 1 423 | } 424 | else if (typeof b == 'object') { 425 | if (b instanceof Array) 426 | return -compareKeys(b, a) 427 | return 1 428 | } else { 429 | return typeOrder[typeof a] < typeOrder[typeof b] ? -1 : 1 430 | } 431 | } 432 | const typeOrder = { 433 | symbol: 0, 434 | undefined: 1, 435 | boolean: 2, 436 | number: 3, 437 | string: 4 438 | } 439 | export const MINIMUM_KEY = null 440 | export const MAXIMUM_KEY = new Uint8Array([0xff]) --------------------------------------------------------------------------------