├── .eslintrc.cjs ├── .gitignore ├── .npmignore ├── .prettierrc ├── LICENSE ├── README.md ├── bun.lockb ├── dist ├── index.d.ts └── index.js ├── index.ts ├── package.json ├── rand ├── 5MB.json ├── 64KB.json ├── index.no-alloc.ts ├── queryOpenAI.ts ├── simplisticVanillaJSONParser.js └── speedTest.ts ├── test ├── parse.test.ts ├── spec.test.ts ├── stream.test.ts ├── tsconfig.json └── utils.test.ts └── tsconfig.json /.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | extends: ["eslint:recommended"], 4 | ignorePatterns: [".eslintrc.cjs", "*.js", "rand"], 5 | overrides: [ 6 | { 7 | files: ["**/*.ts", "**/*.tsx"], 8 | env: { browser: true, es6: true, node: true }, 9 | extends: [ 10 | "eslint:recommended", 11 | "plugin:@typescript-eslint/eslint-recommended", 12 | "plugin:@typescript-eslint/recommended", 13 | "prettier", 14 | ], 15 | globals: { Atomics: "readonly", SharedArrayBuffer: "readonly" }, 16 | parser: "@typescript-eslint/parser", 17 | parserOptions: { 18 | ecmaFeatures: { jsx: true }, 19 | ecmaVersion: 2018, 20 | sourceType: "module", 21 | project: [ 22 | "./tsconfig.json", 23 | "./test/tsconfig.json", 24 | ], 25 | }, 26 | plugins: ["@typescript-eslint"], 27 | rules: { 28 | "no-constant-condition": ["error", { checkLoops: false }], 29 | 30 | "@typescript-eslint/no-explicit-any": 0, 31 | 32 | "@typescript-eslint/no-floating-promises": "error", 33 | "@typescript-eslint/require-await": "error", 34 | "@typescript-eslint/no-misused-promises": [ 35 | "error", 36 | { 37 | checksVoidReturn: { 38 | attributes: false, 39 | }, 40 | }, 41 | ], 42 | 43 | "@typescript-eslint/restrict-plus-operands": "error", 44 | "@typescript-eslint/no-base-to-string": "error", 45 | "@typescript-eslint/restrict-template-expressions": "error", 46 | 47 | "no-unsafe-optional-chaining": [ 48 | "error", 49 | { disallowArithmeticOperators: true }, 50 | ], 51 | 52 | "@typescript-eslint/no-unused-vars": "warn", 53 | }, 54 | }, 55 | ], 56 | } 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .DS_Store 3 | .env -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | /* 2 | !/dist -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "trailingComma": "all" 4 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Jackson Kearl 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GJP-4-GPT 2 | ## Gradual JSON Parser for Generative Pretrained Transformers 3 | 4 | A package for consuming the outputs of JSON-producing LLM's live as they are delivered. Supports both a streaming mode and a `JSON.parse` drop in replacement that handles parsing as much data as possible from a not-yet-completed JSON string. 5 | 6 | See it live at https://rexipie.com and in the [LLM Book](https://marketplace.visualstudio.com/items?itemName=jaaxxx.llm-book) VS Code extension. 7 | 8 | ## Use 9 | 10 | ### Install 11 | 12 | ```bash 13 | npm install gjp-4-gpt 14 | bun install gjp-4-gpt 15 | yarn add gjp-4-gpt 16 | ``` 17 | 18 | ### Basic Parsing 19 | 20 | The `IncompleteJSON.parse` takes a string prefix of a valid JSON string and parses as much data out of it as possible, with options provided to prohibit incomplete parsing of literals (strings, numbers). Objects and arrays that have been fully defined by the prefix (those with matching closing braces/brackets) are identified by the presence of the `gjp-4-gpt.ItemDoneStreaming` symbolic key being set to `true` on them. 21 | 22 | This entry point is best suited for basic testing or when your JSON stream has already been concatenated elsewhere and it would be unfeasible to supply a `ReadableStream`. In cases where a `ReadableStream` is available, the `IncompleteJSON.fromReadable` entry should be preferred. 23 | 24 | ```ts 25 | import { IncompleteJSON, ItemDoneStreaming } from 'gjp-4-gpt' 26 | 27 | IncompleteJSON.parse(`"It was a bright cold day in April, `) 28 | > "It was a bright cold day in April, " 29 | 30 | IncompleteJSON.parse(`["this data", ["is miss`) 31 | > ["this data", ["is miss"]] 32 | 33 | IncompleteJSON.parse>(`{"key1": "myValue", "key`) 34 | > {key1: "myValue"} 35 | 36 | IncompleteJSON.parse(`["foo", "bar", "ba`, {prohibitPartialStrings: true}) 37 | > ["foo", "bar"] 38 | 39 | 40 | // type of values is Incomplete<{value: string}[]> 41 | const values = IncompleteJson.parse<{ value: string }[]>( 42 | `[{"value": "a"}, {"value": "ab`, 43 | ) 44 | 45 | values?.[ItemDoneStreaming] // false 46 | values?.[0]?.[ItemDoneStreaming] // true 47 | values?.[1]?.[ItemDoneStreaming] // false 48 | 49 | // types are coerced accordingly 50 | if (values?.[0]?.[ItemDoneStreaming]) { 51 | values[0].value // string 52 | } else { 53 | values?.[0]?.value // string | undefined 54 | } 55 | 56 | if (values?.[ItemDoneStreaming]) { 57 | // `values` and all children are coerced into their `Complete` counterparts 58 | values[1] // { value: string } 59 | } else { 60 | values?.[1] // { value?: string } | undefined 61 | } 62 | ``` 63 | 64 | > More detail on the `Incomplete`/`Complete` higher-ordered-types is provided below. 65 | 66 | ### Stream Parsing 67 | 68 | This is likely to be the main entry point used by application code. 69 | The `IncompleteJSON.fromReadable` method takes a `ReadableStream` and the same options as above, 70 | and converts it to a `ReadableStream>`, where `T` is the type of object you are expecting the stream to generate. For convenience, utilities for parsing OpenAI-style responses and working with `ReadableStream`'s in contexts without full `AsyncIterable` spec implementation are provided. 71 | 72 | The `options` object, `ItemDoneStreaming` sentinels, and `Incomplete` result object are the same as in the plain `parse` method. 73 | 74 | ```ts 75 | import { IncompleteJSON, ReadableFromOpenAIResponse } from 'gjp-4-gpt' 76 | 77 | const response = await fetch(openAIStyleEndpoint, { 78 | ... 79 | body: JSON.stringify({ 80 | stream: true, 81 | ... 82 | }) 83 | }) 84 | 85 | const plaintextReadable = ReadableFromOpenAIResponse(response) 86 | 87 | if (plaintextReadable.error) { 88 | // Handle status >=400 errors: bad auth, invalid options, request too long, etc. 89 | const details = await plaintextReadable.error 90 | } 91 | else { 92 | const objectReadable = IncompleteJSON.fromReadable(plaintextReadable, options) 93 | 94 | // Server-side and Firefox: 95 | for await (const partial of objectReadable) { 96 | // do something with the `partial: Incomplete`... 97 | } 98 | 99 | // Everywhere else: 100 | for await (const chunk of AsAsyncIterable(objectReadable)) { 101 | // do something with the `partial: Incomplete`... 102 | } 103 | } 104 | ``` 105 | 106 | > See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of and https://github.com/microsoft/TypeScript/issues/29867 for more info on working with `ReadableStreams`'s and `AsyncIterable`'s. 107 | 108 | ### `Incomplete`, `Complete`, `Symbol("gjp-4-gpt.ItemDoneStreaming")`, oh my! 109 | 110 | Streaming data is great, but there are times when it is helpful to know you've received all the data before allowing a particular operation. `Incomplete`, `Complete`, and `ItemDoneStreaming` to the rescue! 111 | 112 | All value objects returned by `IncompleteJSON` methods are of the `Incomplete` type. 113 | This is similar to a deep `Partial`, with the notable exception that array elements 114 | are not made `| undefined` (unless `T` explicitly declares them as such). 115 | 116 | Furthermore, 117 | every object and array will has the `[ItemDoneStreaming]: boolean` symbolic key tacked on. 118 | TypeScript will identify checks for this value, and in codepaths following a successful check 119 | the value and all its children (`value`, `value.whatever`, `value[3].foo.bar[4]`, etc.) are 120 | coerced from the `Incomplete` *back* to `T` (or actually `Complete`, which is 121 | the same as `T` but maintaining the ability to access `[ItemDoneStreaming]`, which will always be `true`). 122 | 123 | ```ts 124 | import { ItemDoneStreaming } from 'gjp-4-gpt' 125 | 126 | function doThingWithCompleteDataItem(item: T) { ... } 127 | 128 | function processOnlyFullyCompletedItems(data: Incomplete) { 129 | if (!data) { 130 | // no bytes received yet... 131 | return 132 | } else { 133 | for (const entry of data) { 134 | // entry is Exclude, undefined> 135 | if (entry[ItemDoneStreaming]) { 136 | // entry is Complete, which is assignable to T 137 | doThingWithCompleteDataItem(entry) 138 | } else { 139 | // entry is not undefined, but no further data is known yet... 140 | // entry.whatever could be checked for `[ItemDoneStreaming]` if needed 141 | } 142 | } 143 | } 144 | } 145 | ``` 146 | 147 | 148 | 149 | ## Development 150 | 151 | To install dependencies: 152 | 153 | ```bash 154 | bun install 155 | ``` 156 | 157 | To test: 158 | 159 | ```bash 160 | bun test 161 | ``` 162 | 163 | To lint: 164 | ```bash 165 | npx eslint . 166 | ``` 167 | 168 | The `rand` directory contains various additional scripts and sample data used for 169 | performance testing and experimentation. 170 | 171 | ## Next Steps 172 | 173 | ### Features 174 | 175 | Some features that might be worth adding in the future: 176 | 177 | - **JSONC**: Allowing for train of thought prefixes can improve a GPT's ability to produce correct results. Currently a JSON field can be devoted to response prefixing, but putting that into a comment could make sense. Also trailing commas :). 178 | - **JSONL**: A relatively simple `TransformStream` can convert a JSONL input into a plain JSON array suitable for this library (prefix with a `[` and replace `\n`'s with `,`'s), but including it here might make sense. 179 | - **prose-embedded JSON**: GPT's may return a response that embeds a JSON string inside other prose (Example: "Sure, I can do that for you, here's the data:\n```json ..."). A way to detect and parse that might make sense. 180 | - **Multiple Readers**: It may be useful in some circumstances to combine multiple JSON streams into a single object, for instance to increase throughput or to use different models for different fields. Supporting this might make sense. 181 | 182 | Happy to take additional suggestions in the Feature Requests, or +1's to any of the above if interested! 183 | 184 | ### Performance 185 | 186 | The performance is currently suitable for the outputs of LLM's (which are typically length constrained), 187 | but the goal of this project so far has been to prioritize exploration and ease of use over speed. For reference, a 64KB json string parses in ~0.1ms using the native `JSON.parse`, 6ms using `IncompleteJSON.parse`, and 2ms with a hand written vanilla JS parser. 188 | 189 | A future rewrite may develop a gradual JSON parser from scratch rather than the current approach of patching the truncated JSON 190 | and feeding it into the builtin `JSON.parse` method, which should increase the speed enough to be suitable for most applications. 191 | 192 | For reference, my testing has a 5MB json string parse in 10ms using the native `JSON.parse`, compared to 300ms using `IncompleteJSON.parse`. A simplistic hand written JSON parser takes ~60ms for the same input, which is around the lower bound for how fast a JavaScript rewrite of this functionality could possibly be. 193 | -------------------------------------------------------------------------------- /bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JacksonKearl/gjp-4-gpt/386f68ade4ce9c6145101a6f1eed74e06d4cd25f/bun.lockb -------------------------------------------------------------------------------- /dist/index.d.ts: -------------------------------------------------------------------------------- 1 | export type IncompleteJsonOptions = { 2 | /** 3 | * When enabled, strings will only be emitted when their full value has been read. 4 | */ 5 | prohibitPartialStrings?: boolean; 6 | /** 7 | * When enabled, numbers will only be emitted when their full value has been read. 8 | * 9 | * *Note*: In plain parse mode, if the JSON consists of only a single numeric value and this option is enabled, 10 | * the value will *never* be emitted, as it is impossible to tell whether the number is complete. 11 | * This is not the case in ReadableStream mode, as the ending is explicitly signaled. 12 | */ 13 | prohibitPartialNumbers?: boolean; 14 | }; 15 | export declare class IncompleteJson { 16 | private options?; 17 | /** 18 | * Parse a prefix of a JSON serialized string into as much data as possible 19 | * Options available to ensure atomic string/number parsing and mark completed objects 20 | */ 21 | static parse(string: string, options?: IncompleteJsonOptions): Incomplete; 22 | /** 23 | * Parse a ReadableStream of JSON data into a ReadableStream of 24 | * as much data as can be extracted from the stream at each moment in time. 25 | */ 26 | static fromReadable(readable: ReadableStream, options?: IncompleteJsonOptions): ReadableStream>; 27 | private consumed; 28 | private unconsumed; 29 | private inString; 30 | private inNumber; 31 | private charsNeededToClose; 32 | private context; 33 | private isDone; 34 | private truncationInfo; 35 | private internalObjectStreamComplete; 36 | private internalObjectRawLiteral; 37 | constructor(options?: IncompleteJsonOptions | undefined); 38 | /** 39 | * Add a chunk of data to the stream. 40 | * 41 | * This runs in time linear to the size of the chunk. 42 | */ 43 | addChunk(chunk: string): void; 44 | /** 45 | * Mark the stream complete. 46 | * This will force emit values that were not emitted previously due to potentially being incomplete. 47 | * 48 | * Example: a chunk that ends in a number will not be emitted because the next chunk may continue with a number, 49 | * which would be appended to the existing value (significantly changing the meaning of the represented data) 50 | * ```ts 51 | * > const ij = new IncompleteJSON() 52 | * > ij.addChunk('1234') 53 | * > ij.readValue() 54 | * undefined 55 | * > ij.addChunk('5') 56 | * > ij.readValue() 57 | * undefined 58 | * > ij.done() 59 | * > ij.readValue() 60 | * 12345 61 | * ``` 62 | */ 63 | done(): void; 64 | /** 65 | * Attempt to parse the consumed chunks into as much data as is available 66 | * 67 | * This operation runs in time linear to the length of the stream 68 | * 69 | * While modern JSON parsers are significantly faster than modern LLM's, care 70 | * should be taken on very large inputs to not call readValue more then needed 71 | */ 72 | readValue(): Incomplete; 73 | private rawParseCache; 74 | private cachedJSONParse; 75 | } 76 | /** 77 | * Given an OpenAi streaming style `Response`, convert it to either an error object 78 | * if request was unsuccessful, or a `ReadableStream` of 79 | * `*.choices[0].delta.content` values for each line received 80 | */ 81 | export declare const ReadableFromOpenAIResponse: (response: Response) => (ReadableStream & { 82 | error?: never; 83 | }) | { 84 | error: Promise<{ 85 | error: { 86 | message: string; 87 | type: string; 88 | code: string; 89 | }; 90 | }>; 91 | }; 92 | /** 93 | * Convert an `ReadableStream` to a `AsyncIterable` for use in `for await (... of ... ) { ... }` loops 94 | * 95 | * By the spec, a `ReadableStream` is already `AsyncIterable`, but most browsers to not support this (server-side support is better). 96 | * 97 | * See https://github.com/microsoft/TypeScript/issues/29867, https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of 98 | */ 99 | export declare function AsAsyncIterable(readable: ReadableStream): AsyncIterable; 100 | /** 101 | * Symbolic sentinel added to objects/arrays when the stream has completed defining the value. 102 | * 103 | * This will be present as a symbolic key with value `true` on all objects/arrays that 104 | * have finished streaming, and will be missing otherwise. 105 | * 106 | * Ex: 107 | * ```ts 108 | * const values = IncompleteJson.parse<{ value: string }[]>( 109 | * '[{"value": "a"}, {"value": "ab', 110 | * ) 111 | * 112 | * values?.[ItemDoneStreaming] // false 113 | * values?.[0]?.[ItemDoneStreaming] // true 114 | * values?.[1]?.[ItemDoneStreaming] // false 115 | * ``` 116 | */ 117 | export declare const ItemDoneStreaming: unique symbol; 118 | /** 119 | * Deep partial of `T` 120 | * 121 | * When `S` is defined, reading `[S]: true` from an object anywhere in the tree 122 | * coerces it and all its children back into a non-partial (`Complete<>`) mode. 123 | */ 124 | export type Incomplete = (T extends (infer V)[] ? Complete | ({ 125 | [ItemDoneStreaming]?: never; 126 | } & Exclude, undefined>[]) : T extends object ? Complete | ({ 127 | [ItemDoneStreaming]?: never; 128 | } & { 129 | [P in keyof T]?: Incomplete; 130 | }) : T) | undefined; 131 | /** 132 | * Deeply adds the entry `[S]: true` to every object in `T` 133 | */ 134 | export type Complete = T extends (infer V)[] ? { 135 | [ItemDoneStreaming]: true; 136 | } & Complete[] : T extends object ? { 137 | [ItemDoneStreaming]: true; 138 | } & { 139 | [P in keyof T]: Complete; 140 | } : T; 141 | -------------------------------------------------------------------------------- /dist/index.js: -------------------------------------------------------------------------------- 1 | export class IncompleteJson { 2 | options; 3 | /** 4 | * Parse a prefix of a JSON serialized string into as much data as possible 5 | * Options available to ensure atomic string/number parsing and mark completed objects 6 | */ 7 | static parse(string, options) { 8 | if (!string) 9 | return undefined; 10 | const parser = new IncompleteJson(options); 11 | parser.addChunk(string); 12 | if (!options?.prohibitPartialNumbers || !string.match(/(\d|\.)$/)) { 13 | parser.done(); 14 | } 15 | return parser.readValue(); 16 | } 17 | /** 18 | * Parse a ReadableStream of JSON data into a ReadableStream of 19 | * as much data as can be extracted from the stream at each moment in time. 20 | */ 21 | static fromReadable(readable, options) { 22 | const parser = new IncompleteJson(options); 23 | let prior; 24 | const transformer = new TransformStream({ 25 | start() { }, 26 | transform(chunk, controller) { 27 | parser.addChunk(chunk); 28 | const next = parser.readValue(); 29 | if (next !== prior) { 30 | controller.enqueue(next); 31 | prior = next; 32 | } 33 | }, 34 | flush(controller) { 35 | parser.done(); 36 | const next = parser.readValue(); 37 | if (next !== prior) { 38 | controller.enqueue(next); 39 | prior = next; 40 | } 41 | }, 42 | }); 43 | return readable.pipeThrough(transformer); 44 | } 45 | consumed = ""; 46 | unconsumed = ""; 47 | inString = false; 48 | inNumber = false; 49 | charsNeededToClose = ""; 50 | context = []; 51 | isDone = false; 52 | truncationInfo = undefined; 53 | internalObjectStreamComplete; 54 | internalObjectRawLiteral = "value"; 55 | constructor(options) { 56 | this.options = options; 57 | this.internalObjectStreamComplete = 58 | "__" + 59 | Number(String(Math.random()).slice(2)).toString(36) + 60 | Number(String(Math.random()).slice(2)).toString(36) + 61 | Number(String(Math.random()).slice(2)).toString(36); 62 | } 63 | /** 64 | * Add a chunk of data to the stream. 65 | * 66 | * This runs in time linear to the size of the chunk. 67 | */ 68 | addChunk(chunk) { 69 | if (this.isDone) 70 | throw Error("Cannot add chunk to parser marked done"); 71 | // called to save the current state as a "safe" spot to truncate and provide a result 72 | const markTruncateSpot = (delta = 0) => (this.truncationInfo = { 73 | index: this.consumed.length + delta, 74 | append: this.charsNeededToClose, 75 | }); 76 | // consume everything we didn't consume last time, then the new chunk 77 | const toConsume = this.unconsumed + chunk; 78 | this.unconsumed = ""; 79 | for (let i = 0; i < toConsume.length; i++) { 80 | const c = toConsume[i]; 81 | // atomically consume escape sequences 82 | if (this.inString && c === "\\") { 83 | // we have seen the `\` 84 | i++; 85 | const escaped = toConsume[i]; 86 | // unicode escapes are of the form \uXXXX 87 | if (escaped === "u") { 88 | if (toConsume[i + 4] !== undefined) { 89 | // if we can grab 4 chars forward, do so 90 | this.consumed += c + escaped + toConsume.slice(i + 1, i + 5); 91 | } 92 | else { 93 | // otherwise, save the rest of the string for later 94 | this.unconsumed = c + escaped + toConsume.slice(i + 1, i + 5); 95 | } 96 | // we have seen either 4 chars or until the end of the string 97 | // (if this goes over the end the loop exists normally) 98 | i += 4; 99 | } 100 | else if (escaped !== undefined) { 101 | // standard two char escape (\n, \t, etc.) 102 | this.consumed += c + escaped; 103 | } 104 | else { 105 | // end of chunk. save the \ to tack onto front of next chunk 106 | this.unconsumed = c; 107 | } 108 | // restart from after the sequence 109 | continue; 110 | } 111 | if (!this.inString && !isNaN(+c)) { 112 | this.inNumber = true; 113 | } 114 | if (this.inNumber && 115 | isNaN(+c) && 116 | c !== "-" && 117 | c !== "e" && 118 | c !== "+" && 119 | c !== "E" && 120 | c !== ".") { 121 | this.inNumber = false; 122 | } 123 | // inject completed object sentinels as required 124 | // basically, convert: 125 | // `A { B ` => `A {isComplete: false, value: { B ` 126 | // `A { B } C` => `A {isComplete: false, value: { B }, isComplete: true} C` 127 | // `A [ B ` => `A {isComplete: false, value: [ B ` 128 | // `A [ B ] C` => `A {isComplete: false, value: [ B ], isComplete: true} C` 129 | // Flattened and replaced with the IncompleteJson.ObjectStreamComplete in this.cachedJSONParse 130 | if (!this.inString && c === "}") { 131 | this.consumed += `}, "${this.internalObjectStreamComplete}": true}`; 132 | this.charsNeededToClose = this.charsNeededToClose.slice(2); 133 | markTruncateSpot(); 134 | } 135 | else if (!this.inString && c === "{") { 136 | this.consumed += `{"${this.internalObjectStreamComplete}": false, "${this.internalObjectRawLiteral}": {`; 137 | this.charsNeededToClose = "}" + this.charsNeededToClose; 138 | } 139 | else if (!this.inString && c === "[") { 140 | this.consumed += `{"${this.internalObjectStreamComplete}": false, "${this.internalObjectRawLiteral}": [`; 141 | this.charsNeededToClose = "}" + this.charsNeededToClose; 142 | } 143 | else if (!this.inString && c === "]") { 144 | this.consumed += `], "${this.internalObjectStreamComplete}": true}`; 145 | this.charsNeededToClose = this.charsNeededToClose.slice(2); 146 | markTruncateSpot(); 147 | } 148 | else { 149 | //otherwise, consume the char itself 150 | this.consumed += c; 151 | } 152 | if (this.inString && c !== '"') { 153 | // if partial strings allowed, every location in a string is a potential truncate spot 154 | // EXCEPT in key strings - the following cannot be completed: { "ab 155 | if (this.context[0] !== "key" && 156 | !this.options?.prohibitPartialStrings) { 157 | markTruncateSpot(); 158 | } 159 | // skip over the special char handling 160 | continue; 161 | } 162 | // consuming a matching closing " - pop it from the stack 163 | if (c === this.charsNeededToClose[0] && c === '"') { 164 | this.charsNeededToClose = this.charsNeededToClose.slice(1); 165 | // good place to truncate, unless we're in a key 166 | if (this.context[0] !== "key") { 167 | markTruncateSpot(); 168 | } 169 | } 170 | if (this.inNumber && !this.options?.prohibitPartialNumbers) { 171 | // symbols found in numbers 172 | if (c === "e" || c === "." || c === "E") { 173 | // unparsable as suffixes, trim them if partials allowed 174 | markTruncateSpot(-1); 175 | } 176 | } 177 | if (c === '"') { 178 | // toggle string mode 179 | this.inString = !this.inString; 180 | // if we aren't prohibiting partial strings and we are starting a new string, 181 | // note how to close this string 182 | if (!this.options?.prohibitPartialStrings && this.inString) { 183 | this.charsNeededToClose = '"' + this.charsNeededToClose; 184 | if (this.context[0] !== "key") { 185 | markTruncateSpot(); 186 | } 187 | } 188 | } 189 | if (c === ",") { 190 | // truncate right before the `,` 191 | markTruncateSpot(-1); 192 | // when parsing object, comma switches from val context to key 193 | if (this.context[0] === "val") { 194 | this.context[0] = "key"; 195 | } 196 | } 197 | // colon switches from key context to val 198 | if (c === ":") { 199 | if (this.context[0] === "key") { 200 | this.context[0] = "val"; 201 | } 202 | } 203 | // in array: strings can always be truncated 204 | if (c === "[") { 205 | this.context.unshift("arr"); 206 | this.charsNeededToClose = "]" + this.charsNeededToClose; 207 | markTruncateSpot(); 208 | } 209 | // in object: strings can be truncated in values, but not keys! 210 | if (c === "{") { 211 | this.context.unshift("key"); 212 | this.charsNeededToClose = "}" + this.charsNeededToClose; 213 | markTruncateSpot(); 214 | } 215 | // exiting our context, pop! 216 | if (c === "}" || c === "]") { 217 | this.context.shift(); 218 | } 219 | } 220 | } 221 | /** 222 | * Mark the stream complete. 223 | * This will force emit values that were not emitted previously due to potentially being incomplete. 224 | * 225 | * Example: a chunk that ends in a number will not be emitted because the next chunk may continue with a number, 226 | * which would be appended to the existing value (significantly changing the meaning of the represented data) 227 | * ```ts 228 | * > const ij = new IncompleteJSON() 229 | * > ij.addChunk('1234') 230 | * > ij.readValue() 231 | * undefined 232 | * > ij.addChunk('5') 233 | * > ij.readValue() 234 | * undefined 235 | * > ij.done() 236 | * > ij.readValue() 237 | * 12345 238 | * ``` 239 | */ 240 | done() { 241 | if (this.isDone) 242 | return; 243 | this.isDone = true; 244 | const rawData = this.consumed + this.charsNeededToClose; 245 | try { 246 | const result = this.cachedJSONParse(rawData); 247 | this.truncationInfo = { 248 | index: this.consumed.length, 249 | append: this.charsNeededToClose, 250 | result, 251 | }; 252 | } 253 | catch { 254 | // pass: this JSON parse is expected to fail in some cases, 255 | // for instance when IncompleteJSON.parse is called without a complete stream. 256 | // the existing truncationInfo will still be good 257 | } 258 | } 259 | /** 260 | * Attempt to parse the consumed chunks into as much data as is available 261 | * 262 | * This operation runs in time linear to the length of the stream 263 | * 264 | * While modern JSON parsers are significantly faster than modern LLM's, care 265 | * should be taken on very large inputs to not call readValue more then needed 266 | */ 267 | readValue() { 268 | if (!this.consumed || !this.truncationInfo) { 269 | return undefined; 270 | } 271 | if (!("result" in this.truncationInfo)) { 272 | try { 273 | this.truncationInfo.result = this.cachedJSONParse(this.consumed.slice(0, this.truncationInfo.index) + 274 | this.truncationInfo.append); 275 | } 276 | catch (e) { 277 | console.error("ERROR: readValue called with bogus internal state.", this); 278 | throw e; 279 | } 280 | } 281 | return this.truncationInfo.result; 282 | } 283 | rawParseCache = { key: "", value: undefined }; 284 | cachedJSONParse(str) { 285 | if (str !== this.rawParseCache.key) { 286 | this.rawParseCache = { 287 | key: str, 288 | value: JSON.parse(str, (k, v) => { 289 | if (typeof v === "object" && 290 | v && 291 | this.internalObjectStreamComplete in v) { 292 | const raw = v[this.internalObjectRawLiteral]; 293 | raw[ItemDoneStreaming] = v[this.internalObjectStreamComplete]; 294 | return raw; 295 | } 296 | return v; 297 | }), 298 | }; 299 | } 300 | return this.rawParseCache.value; 301 | } 302 | } 303 | const PollyfillTextDecoderStream = () => { 304 | try { 305 | return new TextDecoderStream(); 306 | } 307 | catch { 308 | const decoder = new TextDecoder(); 309 | return new TransformStream({ 310 | transform(chunk, controller) { 311 | const text = decoder.decode(chunk, { stream: true }); 312 | if (text.length !== 0) { 313 | controller.enqueue(text); 314 | } 315 | }, 316 | flush(controller) { 317 | const text = decoder.decode(); 318 | if (text.length !== 0) { 319 | controller.enqueue(text); 320 | } 321 | }, 322 | }); 323 | } 324 | }; 325 | /** 326 | * Given an OpenAi streaming style `Response`, convert it to either an error object 327 | * if request was unsuccessful, or a `ReadableStream` of 328 | * `*.choices[0].delta.content` values for each line received 329 | */ 330 | export const ReadableFromOpenAIResponse = (response) => { 331 | if (response.status !== 200) { 332 | return { error: response.json() }; 333 | } 334 | if (!response.body) { 335 | throw Error("Response is non-erroneous but has no body."); 336 | } 337 | let partial = ""; 338 | return response.body.pipeThrough(PollyfillTextDecoderStream()).pipeThrough(new TransformStream({ 339 | transform(value, controller) { 340 | const chunk = partial + value; 341 | partial = ""; 342 | const lines = chunk 343 | .split("\n\n") 344 | .map((x) => x.trim()) 345 | .filter((x) => x && x.startsWith("data: ")) 346 | .map((x) => x.slice("data: ".length)); 347 | for (const line of lines) { 348 | if (line === "[DONE]") { 349 | break; 350 | } 351 | try { 352 | const json = JSON.parse(line); 353 | const content = json.choices[0].delta.content; 354 | if (content !== undefined) { 355 | controller.enqueue(content); 356 | } 357 | } 358 | catch { 359 | // data line incomplete? 360 | partial += "data: " + line; 361 | } 362 | } 363 | }, 364 | })); 365 | }; 366 | /** 367 | * Convert an `ReadableStream` to a `AsyncIterable` for use in `for await (... of ... ) { ... }` loops 368 | * 369 | * By the spec, a `ReadableStream` is already `AsyncIterable`, but most browsers to not support this (server-side support is better). 370 | * 371 | * See https://github.com/microsoft/TypeScript/issues/29867, https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of 372 | */ 373 | export async function* AsAsyncIterable(readable) { 374 | const reader = readable.getReader(); 375 | try { 376 | while (true) { 377 | const { done, value } = await reader.read(); 378 | if (done) 379 | return; 380 | yield value; 381 | } 382 | } 383 | finally { 384 | reader.releaseLock(); 385 | } 386 | } 387 | /** 388 | * Symbolic sentinel added to objects/arrays when the stream has completed defining the value. 389 | * 390 | * This will be present as a symbolic key with value `true` on all objects/arrays that 391 | * have finished streaming, and will be missing otherwise. 392 | * 393 | * Ex: 394 | * ```ts 395 | * const values = IncompleteJson.parse<{ value: string }[]>( 396 | * '[{"value": "a"}, {"value": "ab', 397 | * ) 398 | * 399 | * values?.[ItemDoneStreaming] // false 400 | * values?.[0]?.[ItemDoneStreaming] // true 401 | * values?.[1]?.[ItemDoneStreaming] // false 402 | * ``` 403 | */ 404 | export const ItemDoneStreaming = Symbol("gjp-4-gpt.ItemDoneStreaming"); 405 | -------------------------------------------------------------------------------- /index.ts: -------------------------------------------------------------------------------- 1 | export type IncompleteJsonOptions = { 2 | /** 3 | * When enabled, strings will only be emitted when their full value has been read. 4 | */ 5 | prohibitPartialStrings?: boolean 6 | 7 | /** 8 | * When enabled, numbers will only be emitted when their full value has been read. 9 | * 10 | * *Note*: In plain parse mode, if the JSON consists of only a single numeric value and this option is enabled, 11 | * the value will *never* be emitted, as it is impossible to tell whether the number is complete. 12 | * This is not the case in ReadableStream mode, as the ending is explicitly signaled. 13 | */ 14 | prohibitPartialNumbers?: boolean 15 | } 16 | 17 | export class IncompleteJson { 18 | /** 19 | * Parse a prefix of a JSON serialized string into as much data as possible 20 | * Options available to ensure atomic string/number parsing and mark completed objects 21 | */ 22 | static parse( 23 | string: string, 24 | options?: IncompleteJsonOptions, 25 | ): Incomplete { 26 | if (!string) return undefined 27 | 28 | const parser = new IncompleteJson(options) 29 | parser.addChunk(string) 30 | if (!options?.prohibitPartialNumbers || !string.match(/(\d|\.)$/)) { 31 | parser.done() 32 | } 33 | 34 | return parser.readValue() 35 | } 36 | 37 | /** 38 | * Parse a ReadableStream of JSON data into a ReadableStream of 39 | * as much data as can be extracted from the stream at each moment in time. 40 | */ 41 | static fromReadable( 42 | readable: ReadableStream, 43 | options?: IncompleteJsonOptions, 44 | ): ReadableStream> { 45 | const parser = new IncompleteJson(options) 46 | let prior: Incomplete 47 | 48 | const transformer = new TransformStream>({ 49 | start() {}, 50 | transform(chunk, controller) { 51 | parser.addChunk(chunk) 52 | 53 | const next = parser.readValue() 54 | 55 | if (next !== prior) { 56 | controller.enqueue(next) 57 | prior = next 58 | } 59 | }, 60 | flush(controller) { 61 | parser.done() 62 | const next = parser.readValue() 63 | if (next !== prior) { 64 | controller.enqueue(next) 65 | prior = next 66 | } 67 | }, 68 | }) 69 | return readable.pipeThrough(transformer) 70 | } 71 | 72 | private consumed = "" 73 | private unconsumed = "" 74 | private inString = false 75 | private inNumber = false 76 | 77 | private charsNeededToClose: string = "" 78 | private context: ("key" | "val" | "arr")[] = [] 79 | 80 | private isDone = false 81 | 82 | private truncationInfo: 83 | | { 84 | index: number 85 | append: string 86 | result?: Incomplete 87 | } 88 | | undefined = undefined 89 | 90 | private internalObjectStreamComplete: string 91 | private internalObjectRawLiteral = "value" 92 | 93 | constructor(private options?: IncompleteJsonOptions) { 94 | this.internalObjectStreamComplete = 95 | "__" + 96 | Number(String(Math.random()).slice(2)).toString(36) + 97 | Number(String(Math.random()).slice(2)).toString(36) + 98 | Number(String(Math.random()).slice(2)).toString(36) 99 | } 100 | 101 | /** 102 | * Add a chunk of data to the stream. 103 | * 104 | * This runs in time linear to the size of the chunk. 105 | */ 106 | addChunk(chunk: string) { 107 | if (this.isDone) throw Error("Cannot add chunk to parser marked done") 108 | 109 | // called to save the current state as a "safe" spot to truncate and provide a result 110 | const markTruncateSpot = (delta = 0) => 111 | (this.truncationInfo = { 112 | index: this.consumed.length + delta, 113 | append: this.charsNeededToClose, 114 | }) 115 | 116 | // consume everything we didn't consume last time, then the new chunk 117 | const toConsume = this.unconsumed + chunk 118 | 119 | this.unconsumed = "" 120 | 121 | for (let i = 0; i < toConsume.length; i++) { 122 | const c = toConsume[i] 123 | 124 | // atomically consume escape sequences 125 | if (this.inString && c === "\\") { 126 | // we have seen the `\` 127 | i++ 128 | const escaped = toConsume[i] 129 | // unicode escapes are of the form \uXXXX 130 | if (escaped === "u") { 131 | if (toConsume[i + 4] !== undefined) { 132 | // if we can grab 4 chars forward, do so 133 | this.consumed += c + escaped + toConsume.slice(i + 1, i + 5) 134 | } else { 135 | // otherwise, save the rest of the string for later 136 | this.unconsumed = c + escaped + toConsume.slice(i + 1, i + 5) 137 | } 138 | // we have seen either 4 chars or until the end of the string 139 | // (if this goes over the end the loop exists normally) 140 | i += 4 141 | } else if (escaped !== undefined) { 142 | // standard two char escape (\n, \t, etc.) 143 | this.consumed += c + escaped 144 | } else { 145 | // end of chunk. save the \ to tack onto front of next chunk 146 | this.unconsumed = c 147 | } 148 | 149 | // restart from after the sequence 150 | continue 151 | } 152 | 153 | if (!this.inString && !isNaN(+c)) { 154 | this.inNumber = true 155 | } 156 | if ( 157 | this.inNumber && 158 | isNaN(+c) && 159 | c !== "-" && 160 | c !== "e" && 161 | c !== "+" && 162 | c !== "E" && 163 | c !== "." 164 | ) { 165 | this.inNumber = false 166 | } 167 | 168 | // inject completed object sentinels as required 169 | // basically, convert: 170 | // `A { B ` => `A {isComplete: false, value: { B ` 171 | // `A { B } C` => `A {isComplete: false, value: { B }, isComplete: true} C` 172 | // `A [ B ` => `A {isComplete: false, value: [ B ` 173 | // `A [ B ] C` => `A {isComplete: false, value: [ B ], isComplete: true} C` 174 | // Flattened and replaced with the IncompleteJson.ObjectStreamComplete in this.cachedJSONParse 175 | if (!this.inString && c === "}") { 176 | this.consumed += `}, "${this.internalObjectStreamComplete}": true}` 177 | this.charsNeededToClose = this.charsNeededToClose.slice(2) 178 | markTruncateSpot() 179 | } else if (!this.inString && c === "{") { 180 | this.consumed += `{"${this.internalObjectStreamComplete}": false, "${this.internalObjectRawLiteral}": {` 181 | this.charsNeededToClose = "}" + this.charsNeededToClose 182 | } else if (!this.inString && c === "[") { 183 | this.consumed += `{"${this.internalObjectStreamComplete}": false, "${this.internalObjectRawLiteral}": [` 184 | this.charsNeededToClose = "}" + this.charsNeededToClose 185 | } else if (!this.inString && c === "]") { 186 | this.consumed += `], "${this.internalObjectStreamComplete}": true}` 187 | this.charsNeededToClose = this.charsNeededToClose.slice(2) 188 | markTruncateSpot() 189 | } else { 190 | //otherwise, consume the char itself 191 | this.consumed += c 192 | } 193 | 194 | if (this.inString && c !== '"') { 195 | // if partial strings allowed, every location in a string is a potential truncate spot 196 | // EXCEPT in key strings - the following cannot be completed: { "ab 197 | if ( 198 | this.context[0] !== "key" && 199 | !this.options?.prohibitPartialStrings 200 | ) { 201 | markTruncateSpot() 202 | } 203 | 204 | // skip over the special char handling 205 | continue 206 | } 207 | 208 | // consuming a matching closing " - pop it from the stack 209 | if (c === this.charsNeededToClose[0] && c === '"') { 210 | this.charsNeededToClose = this.charsNeededToClose.slice(1) 211 | 212 | // good place to truncate, unless we're in a key 213 | if (this.context[0] !== "key") { 214 | markTruncateSpot() 215 | } 216 | } 217 | 218 | if (this.inNumber && !this.options?.prohibitPartialNumbers) { 219 | // symbols found in numbers 220 | if (c === "e" || c === "." || c === "E") { 221 | // unparsable as suffixes, trim them if partials allowed 222 | markTruncateSpot(-1) 223 | } 224 | } 225 | 226 | if (c === '"') { 227 | // toggle string mode 228 | this.inString = !this.inString 229 | 230 | // if we aren't prohibiting partial strings and we are starting a new string, 231 | // note how to close this string 232 | if (!this.options?.prohibitPartialStrings && this.inString) { 233 | this.charsNeededToClose = '"' + this.charsNeededToClose 234 | if (this.context[0] !== "key") { 235 | markTruncateSpot() 236 | } 237 | } 238 | } 239 | 240 | if (c === ",") { 241 | // truncate right before the `,` 242 | markTruncateSpot(-1) 243 | 244 | // when parsing object, comma switches from val context to key 245 | if (this.context[0] === "val") { 246 | this.context[0] = "key" 247 | } 248 | } 249 | 250 | // colon switches from key context to val 251 | if (c === ":") { 252 | if (this.context[0] === "key") { 253 | this.context[0] = "val" 254 | } 255 | } 256 | 257 | // in array: strings can always be truncated 258 | if (c === "[") { 259 | this.context.unshift("arr") 260 | this.charsNeededToClose = "]" + this.charsNeededToClose 261 | markTruncateSpot() 262 | } 263 | 264 | // in object: strings can be truncated in values, but not keys! 265 | if (c === "{") { 266 | this.context.unshift("key") 267 | this.charsNeededToClose = "}" + this.charsNeededToClose 268 | markTruncateSpot() 269 | } 270 | 271 | // exiting our context, pop! 272 | if (c === "}" || c === "]") { 273 | this.context.shift() 274 | } 275 | } 276 | } 277 | 278 | /** 279 | * Mark the stream complete. 280 | * This will force emit values that were not emitted previously due to potentially being incomplete. 281 | * 282 | * Example: a chunk that ends in a number will not be emitted because the next chunk may continue with a number, 283 | * which would be appended to the existing value (significantly changing the meaning of the represented data) 284 | * ```ts 285 | * > const ij = new IncompleteJSON() 286 | * > ij.addChunk('1234') 287 | * > ij.readValue() 288 | * undefined 289 | * > ij.addChunk('5') 290 | * > ij.readValue() 291 | * undefined 292 | * > ij.done() 293 | * > ij.readValue() 294 | * 12345 295 | * ``` 296 | */ 297 | done() { 298 | if (this.isDone) return 299 | this.isDone = true 300 | 301 | const rawData = this.consumed + this.charsNeededToClose 302 | try { 303 | const result = this.cachedJSONParse(rawData) 304 | this.truncationInfo = { 305 | index: this.consumed.length, 306 | append: this.charsNeededToClose, 307 | result, 308 | } 309 | } catch { 310 | // pass: this JSON parse is expected to fail in some cases, 311 | // for instance when IncompleteJSON.parse is called without a complete stream. 312 | // the existing truncationInfo will still be good 313 | } 314 | } 315 | 316 | /** 317 | * Attempt to parse the consumed chunks into as much data as is available 318 | * 319 | * This operation runs in time linear to the length of the stream 320 | * 321 | * While modern JSON parsers are significantly faster than modern LLM's, care 322 | * should be taken on very large inputs to not call readValue more then needed 323 | */ 324 | readValue(): Incomplete { 325 | if (!this.consumed || !this.truncationInfo) { 326 | return undefined 327 | } 328 | 329 | if (!("result" in this.truncationInfo)) { 330 | try { 331 | this.truncationInfo.result = this.cachedJSONParse( 332 | this.consumed.slice(0, this.truncationInfo.index) + 333 | this.truncationInfo.append, 334 | ) 335 | } catch (e) { 336 | console.error( 337 | "ERROR: readValue called with bogus internal state.", 338 | this, 339 | ) 340 | throw e 341 | } 342 | } 343 | 344 | return this.truncationInfo.result 345 | } 346 | 347 | private rawParseCache = { key: "", value: undefined as Incomplete } 348 | private cachedJSONParse(str: string): Incomplete { 349 | if (str !== this.rawParseCache.key) { 350 | this.rawParseCache = { 351 | key: str, 352 | value: JSON.parse(str, (k, v) => { 353 | if ( 354 | typeof v === "object" && 355 | v && 356 | this.internalObjectStreamComplete in v 357 | ) { 358 | const raw = v[this.internalObjectRawLiteral] 359 | raw[ItemDoneStreaming] = v[this.internalObjectStreamComplete] 360 | 361 | return raw 362 | } 363 | return v 364 | }), 365 | } 366 | } 367 | return this.rawParseCache.value 368 | } 369 | } 370 | 371 | const PollyfillTextDecoderStream = () => { 372 | try { 373 | return new TextDecoderStream() 374 | } catch { 375 | const decoder = new TextDecoder() 376 | return new TransformStream({ 377 | transform(chunk, controller) { 378 | const text = decoder.decode(chunk, { stream: true }) 379 | if (text.length !== 0) { 380 | controller.enqueue(text) 381 | } 382 | }, 383 | flush(controller) { 384 | const text = decoder.decode() 385 | if (text.length !== 0) { 386 | controller.enqueue(text) 387 | } 388 | }, 389 | }) 390 | } 391 | } 392 | 393 | /** 394 | * Given an OpenAi streaming style `Response`, convert it to either an error object 395 | * if request was unsuccessful, or a `ReadableStream` of 396 | * `*.choices[0].delta.content` values for each line received 397 | */ 398 | export const ReadableFromOpenAIResponse = ( 399 | response: Response, 400 | ): 401 | | (ReadableStream & { error?: never }) 402 | | { 403 | error: Promise<{ error: { message: string; type: string; code: string } }> 404 | } => { 405 | if (response.status !== 200) { 406 | return { error: response.json() } 407 | } 408 | if (!response.body) { 409 | throw Error("Response is non-erroneous but has no body.") 410 | } 411 | 412 | let partial = "" 413 | 414 | return response.body.pipeThrough(PollyfillTextDecoderStream()).pipeThrough( 415 | new TransformStream({ 416 | transform(value, controller) { 417 | const chunk = partial + value 418 | partial = "" 419 | 420 | const lines = chunk 421 | .split("\n\n") 422 | .map((x) => x.trim()) 423 | .filter((x) => x && x.startsWith("data: ")) 424 | .map((x) => x.slice("data: ".length)) 425 | 426 | for (const line of lines) { 427 | if (line === "[DONE]") { 428 | break 429 | } 430 | try { 431 | const json = JSON.parse(line) 432 | const content = json.choices[0].delta.content 433 | 434 | if (content !== undefined) { 435 | controller.enqueue(content) 436 | } 437 | } catch { 438 | // data line incomplete? 439 | partial += "data: " + line 440 | } 441 | } 442 | }, 443 | }), 444 | ) 445 | } 446 | 447 | /** 448 | * Convert an `ReadableStream` to a `AsyncIterable` for use in `for await (... of ... ) { ... }` loops 449 | * 450 | * By the spec, a `ReadableStream` is already `AsyncIterable`, but most browsers to not support this (server-side support is better). 451 | * 452 | * See https://github.com/microsoft/TypeScript/issues/29867, https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of 453 | */ 454 | export async function* AsAsyncIterable( 455 | readable: ReadableStream, 456 | ): AsyncIterable { 457 | const reader = readable.getReader() 458 | try { 459 | while (true) { 460 | const { done, value } = await reader.read() 461 | if (done) return 462 | yield value 463 | } 464 | } finally { 465 | reader.releaseLock() 466 | } 467 | } 468 | 469 | /** 470 | * Symbolic sentinel added to objects/arrays when the stream has completed defining the value. 471 | * 472 | * This will be present as a symbolic key with value `true` on all objects/arrays that 473 | * have finished streaming, and will be missing otherwise. 474 | * 475 | * Ex: 476 | * ```ts 477 | * const values = IncompleteJson.parse<{ value: string }[]>( 478 | * '[{"value": "a"}, {"value": "ab', 479 | * ) 480 | * 481 | * values?.[ItemDoneStreaming] // false 482 | * values?.[0]?.[ItemDoneStreaming] // true 483 | * values?.[1]?.[ItemDoneStreaming] // false 484 | * ``` 485 | */ 486 | export const ItemDoneStreaming = Symbol("gjp-4-gpt.ItemDoneStreaming") 487 | 488 | /** 489 | * Deep partial of `T` 490 | * 491 | * When `S` is defined, reading `[S]: true` from an object anywhere in the tree 492 | * coerces it and all its children back into a non-partial (`Complete<>`) mode. 493 | */ 494 | export type Incomplete = 495 | | (T extends (infer V)[] 496 | ? 497 | | Complete 498 | | ({ [ItemDoneStreaming]?: never } & Exclude< 499 | Incomplete, 500 | undefined 501 | >[]) 502 | : T extends object 503 | ? 504 | | Complete 505 | | ({ [ItemDoneStreaming]?: never } & { 506 | [P in keyof T]?: Incomplete 507 | }) 508 | : T) 509 | | undefined 510 | 511 | /** 512 | * Deeply adds the entry `[S]: true` to every object in `T` 513 | */ 514 | export type Complete = T extends (infer V)[] 515 | ? { [ItemDoneStreaming]: true } & Complete[] 516 | : T extends object 517 | ? { [ItemDoneStreaming]: true } & { 518 | [P in keyof T]: Complete 519 | } 520 | : T 521 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gjp-4-gpt", 3 | "version": "0.0.8", 4 | "license": "MIT", 5 | "module": "index.ts", 6 | "type": "module", 7 | "main": "dist/index.js", 8 | "types": "dist/index.d.ts", 9 | "devDependencies": { 10 | "bun-types": "latest", 11 | "@typescript-eslint/eslint-plugin": "^6.10.0", 12 | "@typescript-eslint/parser": "^6.10.0", 13 | "eslint": "^8.53.0", 14 | "eslint-config-prettier": "^9.0.0" 15 | } 16 | } -------------------------------------------------------------------------------- /rand/64KB.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Adeel Solangi", 4 | "language": "Sindhi", 5 | "id": "V59OF92YF627HFY0", 6 | "bio": "Donec lobortis eleifend condimentum. Cras dictum dolor lacinia lectus vehicula rutrum. Maecenas quis nisi nunc. Nam tristique feugiat est vitae mollis. Maecenas quis nisi nunc.", 7 | "version": 6.1 8 | }, 9 | { 10 | "name": "Afzal Ghaffar", 11 | "language": "Sindhi", 12 | "id": "ENTOCR13RSCLZ6KU", 13 | "bio": "Aliquam sollicitudin ante ligula, eget malesuada nibh efficitur et. Pellentesque massa sem, scelerisque sit amet odio id, cursus tempor urna. Etiam congue dignissim volutpat. Vestibulum pharetra libero et velit gravida euismod.", 14 | "version": 1.88 15 | }, 16 | { 17 | "name": "Aamir Solangi", 18 | "language": "Sindhi", 19 | "id": "IAKPO3R4761JDRVG", 20 | "bio": "Vestibulum pharetra libero et velit gravida euismod. Quisque mauris ligula, efficitur porttitor sodales ac, lacinia non ex. Fusce eu ultrices elit, vel posuere neque.", 21 | "version": 7.27 22 | }, 23 | { 24 | "name": "Abla Dilmurat", 25 | "language": "Uyghur", 26 | "id": "5ZVOEPMJUI4MB4EN", 27 | "bio": "Donec lobortis eleifend condimentum. Morbi ac tellus erat.", 28 | "version": 2.53 29 | }, 30 | { 31 | "name": "Adil Eli", 32 | "language": "Uyghur", 33 | "id": "6VTI8X6LL0MMPJCC", 34 | "bio": "Vivamus id faucibus velit, id posuere leo. Morbi vitae nisi lacinia, laoreet lorem nec, egestas orci. Suspendisse potenti.", 35 | "version": 6.49 36 | }, 37 | { 38 | "name": "Adile Qadir", 39 | "language": "Uyghur", 40 | "id": "F2KEU5L7EHYSYFTT", 41 | "bio": "Duis commodo orci ut dolor iaculis facilisis. Morbi ultricies consequat ligula posuere eleifend. Aenean finibus in tortor vel aliquet. Fusce eu ultrices elit, vel posuere neque.", 42 | "version": 1.9 43 | }, 44 | { 45 | "name": "Abdukerim Ibrahim", 46 | "language": "Uyghur", 47 | "id": "LO6DVTZLRK68528I", 48 | "bio": "Vivamus id faucibus velit, id posuere leo. Nunc aliquet sodales nunc a pulvinar. Nunc aliquet sodales nunc a pulvinar. Ut viverra quis eros eu tincidunt.", 49 | "version": 5.9 50 | }, 51 | { 52 | "name": "Adil Abro", 53 | "language": "Sindhi", 54 | "id": "LJRIULRNJFCNZJAJ", 55 | "bio": "Etiam malesuada blandit erat, nec ultricies leo maximus sed. Fusce congue aliquam elit ut luctus. Etiam malesuada blandit erat, nec ultricies leo maximus sed. Cras dictum dolor lacinia lectus vehicula rutrum. Integer vehicula, arcu sit amet egestas efficitur, orci justo interdum massa, eget ullamcorper risus ligula tristique libero.", 56 | "version": 9.32 57 | }, 58 | { 59 | "name": "Afonso Vilarchán", 60 | "language": "Galician", 61 | "id": "JMCL0CXNXHPL1GBC", 62 | "bio": "Fusce eu ultrices elit, vel posuere neque. Morbi ac tellus erat. Nunc tincidunt laoreet laoreet.", 63 | "version": 5.21 64 | }, 65 | { 66 | "name": "Mark Schembri", 67 | "language": "Maltese", 68 | "id": "KU4T500C830697CW", 69 | "bio": "Nam laoreet, nunc non suscipit interdum, justo turpis vestibulum massa, non vulputate ex urna at purus. Morbi ultricies consequat ligula posuere eleifend. Vivamus id faucibus velit, id posuere leo. Sed laoreet posuere sapien, ut feugiat nibh gravida at. Ut maximus, libero nec facilisis fringilla, ex sem sollicitudin leo, non congue tortor ligula in eros.", 70 | "version": 3.17 71 | }, 72 | { 73 | "name": "Antía Sixirei", 74 | "language": "Galician", 75 | "id": "XOF91ZR7MHV1TXRS", 76 | "bio": "Pellentesque massa sem, scelerisque sit amet odio id, cursus tempor urna. Phasellus massa ligula, hendrerit eget efficitur eget, tincidunt in ligula. Morbi finibus dui sed est fringilla ornare. Duis pellentesque ultrices convallis. Morbi ultricies consequat ligula posuere eleifend.", 77 | "version": 6.44 78 | }, 79 | { 80 | "name": "Aygul Mutellip", 81 | "language": "Uyghur", 82 | "id": "FTSNV411G5MKLPDT", 83 | "bio": "Duis commodo orci ut dolor iaculis facilisis. Nam semper gravida nunc, sit amet elementum ipsum. Donec pellentesque ultrices mi, non consectetur eros luctus non. Pellentesque massa sem, scelerisque sit amet odio id, cursus tempor urna.", 84 | "version": 9.1 85 | }, 86 | { 87 | "name": "Awais Shaikh", 88 | "language": "Sindhi", 89 | "id": "OJMWMEEQWMLDU29P", 90 | "bio": "Nunc aliquet sodales nunc a pulvinar. Ut dictum, ligula eget sagittis maximus, tellus mi varius ex, a accumsan justo tellus vitae leo. Donec pellentesque ultrices mi, non consectetur eros luctus non. Nulla finibus massa at viverra facilisis. Nunc tincidunt laoreet laoreet.", 91 | "version": 1.59 92 | }, 93 | { 94 | "name": "Ambreen Ahmed", 95 | "language": "Sindhi", 96 | "id": "5G646V7E6TJW8X2M", 97 | "bio": "Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam consequat enim lorem, at tincidunt velit ultricies et. Ut maximus, libero nec facilisis fringilla, ex sem sollicitudin leo, non congue tortor ligula in eros.", 98 | "version": 2.35 99 | }, 100 | { 101 | "name": "Celtia Anes", 102 | "language": "Galician", 103 | "id": "Z53AJY7WUYPLAWC9", 104 | "bio": "Nullam ac sodales dolor, eu facilisis dui. Maecenas non arcu nulla. Ut viverra quis eros eu tincidunt. Curabitur quis commodo quam.", 105 | "version": 8.34 106 | }, 107 | { 108 | "name": "George Mifsud", 109 | "language": "Maltese", 110 | "id": "N1AS6UFULO6WGTLB", 111 | "bio": "Phasellus tincidunt sollicitudin posuere. Ut accumsan, est vel fringilla varius, purus augue blandit nisl, eu rhoncus ligula purus vel dolor. Donec congue sapien vel euismod interdum. Cras dictum dolor lacinia lectus vehicula rutrum. Phasellus massa ligula, hendrerit eget efficitur eget, tincidunt in ligula.", 112 | "version": 7.47 113 | }, 114 | { 115 | "name": "Aytürk Qasim", 116 | "language": "Uyghur", 117 | "id": "70RODUVRD95CLOJL", 118 | "bio": "Curabitur ultricies id urna nec ultrices. Aliquam scelerisque pretium tellus, sed accumsan est ultrices id. Duis commodo orci ut dolor iaculis facilisis.", 119 | "version": 1.32 120 | }, 121 | { 122 | "name": "Dialè Meso", 123 | "language": "Sesotho sa Leboa", 124 | "id": "VBLI24FKF7VV6BWE", 125 | "bio": "Maecenas non arcu nulla. Vivamus id faucibus velit, id posuere leo. Nullam sodales convallis mauris, sit amet lobortis magna auctor sit amet.", 126 | "version": 6.29 127 | }, 128 | { 129 | "name": "Breixo Galáns", 130 | "language": "Galician", 131 | "id": "4VRLON0GPEZYFCVL", 132 | "bio": "Integer vehicula, arcu sit amet egestas efficitur, orci justo interdum massa, eget ullamcorper risus ligula tristique libero. Morbi ac tellus erat. In id elit malesuada, pulvinar mi eu, imperdiet nulla. Vestibulum pharetra libero et velit gravida euismod. Cras dictum dolor lacinia lectus vehicula rutrum.", 133 | "version": 1.62 134 | }, 135 | { 136 | "name": "Bieito Lorme", 137 | "language": "Galician", 138 | "id": "5DRDI1QLRGLP29RC", 139 | "bio": "Ut viverra quis eros eu tincidunt. Morbi vitae nisi lacinia, laoreet lorem nec, egestas orci. Curabitur quis commodo quam. Morbi ac tellus erat.", 140 | "version": 4.45 141 | }, 142 | { 143 | "name": "Azrugul Osman", 144 | "language": "Uyghur", 145 | "id": "5RCTVD3C5QGVAKTQ", 146 | "bio": "Maecenas tempus neque ut porttitor malesuada. Donec lobortis eleifend condimentum.", 147 | "version": 3.18 148 | }, 149 | { 150 | "name": "Brais Verdiñas", 151 | "language": "Galician", 152 | "id": "BT407GHCC0IHXCD3", 153 | "bio": "Quisque maximus sodales mauris ut elementum. Ut viverra quis eros eu tincidunt. Sed eu libero maximus nunc lacinia lobortis et sit amet nisi. In id elit malesuada, pulvinar mi eu, imperdiet nulla. Curabitur quis commodo quam.", 154 | "version": 5.01 155 | }, 156 | { 157 | "name": "Ekber Sadir", 158 | "language": "Uyghur", 159 | "id": "AGZDAP8D8OVRRLTY", 160 | "bio": "Quisque efficitur vel sapien ut imperdiet. Phasellus massa ligula, hendrerit eget efficitur eget, tincidunt in ligula. In id elit malesuada, pulvinar mi eu, imperdiet nulla. Sed nec suscipit ligula. Integer vehicula, arcu sit amet egestas efficitur, orci justo interdum massa, eget ullamcorper risus ligula tristique libero.", 161 | "version": 2.04 162 | }, 163 | { 164 | "name": "Doreen Bartolo", 165 | "language": "Maltese", 166 | "id": "59QSX02O2XOZGRLH", 167 | "bio": "Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam consequat enim lorem, at tincidunt velit ultricies et. Nam semper gravida nunc, sit amet elementum ipsum. Ut viverra quis eros eu tincidunt. Curabitur sed condimentum felis, ut luctus eros.", 168 | "version": 9.31 169 | }, 170 | { 171 | "name": "Ali Ayaz", 172 | "language": "Sindhi", 173 | "id": "3WNLUZ5LT2F7MYVU", 174 | "bio": "Cras dictum dolor lacinia lectus vehicula rutrum. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam consequat enim lorem, at tincidunt velit ultricies et. Etiam malesuada blandit erat, nec ultricies leo maximus sed.", 175 | "version": 7.8 176 | }, 177 | { 178 | "name": "Guzelnur Polat", 179 | "language": "Uyghur", 180 | "id": "I6QQHAEGV4CYDXLP", 181 | "bio": "Nam laoreet, nunc non suscipit interdum, justo turpis vestibulum massa, non vulputate ex urna at purus. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam consequat enim lorem, at tincidunt velit ultricies et. Nulla finibus massa at viverra facilisis.", 182 | "version": 8.56 183 | }, 184 | { 185 | "name": "John Falzon", 186 | "language": "Maltese", 187 | "id": "U3AWXHDTSU0H82SL", 188 | "bio": "Sed nec suscipit ligula. Nullam sodales convallis mauris, sit amet lobortis magna auctor sit amet.", 189 | "version": 9.96 190 | }, 191 | { 192 | "name": "Erkin Qadir", 193 | "language": "Uyghur", 194 | "id": "GV6TA1AATZYBJ3VR", 195 | "bio": "Phasellus massa ligula, hendrerit eget efficitur eget, tincidunt in ligula. .", 196 | "version": 3.53 197 | }, 198 | { 199 | "name": "Anita Rajput", 200 | "language": "Sindhi", 201 | "id": "XLLVD0NO2ZFEP4AK", 202 | "bio": "Nam semper gravida nunc, sit amet elementum ipsum. Etiam congue dignissim volutpat.", 203 | "version": 5.16 204 | }, 205 | { 206 | "name": "Ayesha Khalique", 207 | "language": "Sindhi", 208 | "id": "Q9A5QNGA0OSU8P6Y", 209 | "bio": "Morbi vitae nisi lacinia, laoreet lorem nec, egestas orci. Etiam mauris magna, fermentum vitae aliquet eu, cursus vitae sapien.", 210 | "version": 3.9 211 | }, 212 | { 213 | "name": "Pheladi Rammala", 214 | "language": "Sesotho sa Leboa", 215 | "id": "EELSIRT2T4Q0M3M4", 216 | "bio": "Quisque efficitur vel sapien ut imperdiet. Morbi ac tellus erat. Aliquam scelerisque pretium tellus, sed accumsan est ultrices id. Ut maximus, libero nec facilisis fringilla, ex sem sollicitudin leo, non congue tortor ligula in eros.", 217 | "version": 1.88 218 | }, 219 | { 220 | "name": "Antón Caneiro", 221 | "language": "Galician", 222 | "id": "ENTAPNU3MMFUGM1W", 223 | "bio": "Integer vehicula, arcu sit amet egestas efficitur, orci justo interdum massa, eget ullamcorper risus ligula tristique libero. Vestibulum pharetra libero et velit gravida euismod.", 224 | "version": 4.84 225 | }, 226 | { 227 | "name": "Qahar Abdulla", 228 | "language": "Uyghur", 229 | "id": "OGLODUPEHKEW0K83", 230 | "bio": "Duis commodo orci ut dolor iaculis facilisis. Aliquam sollicitudin ante ligula, eget malesuada nibh efficitur et. Fusce congue aliquam elit ut luctus. Integer vehicula, arcu sit amet egestas efficitur, orci justo interdum massa, eget ullamcorper risus ligula tristique libero. Quisque maximus sodales mauris ut elementum.", 231 | "version": 3.65 232 | }, 233 | { 234 | "name": "Reyhan Murat", 235 | "language": "Uyghur", 236 | "id": "Y91F4D54794E9ANT", 237 | "bio": "Suspendisse sit amet ullamcorper sem. Curabitur sed condimentum felis, ut luctus eros.", 238 | "version": 2.69 239 | }, 240 | { 241 | "name": "Tatapi Phogole", 242 | "language": "Sesotho sa Leboa", 243 | "id": "7JA42P5CMCWDVPNR", 244 | "bio": "Duis luctus, lacus eu aliquet convallis, purus elit malesuada ex, vitae rutrum ipsum dui ut magna. Nullam ac sodales dolor, eu facilisis dui. Ut viverra quis eros eu tincidunt.", 245 | "version": 3.78 246 | }, 247 | { 248 | "name": "Marcos Amboade", 249 | "language": "Galician", 250 | "id": "WPX7H97C7D70CZJR", 251 | "bio": "Nulla finibus massa at viverra facilisis. Pellentesque massa sem, scelerisque sit amet odio id, cursus tempor urna. Curabitur ultricies id urna nec ultrices. Ut maximus, libero nec facilisis fringilla, ex sem sollicitudin leo, non congue tortor ligula in eros. Nunc aliquet sodales nunc a pulvinar.", 252 | "version": 7.37 253 | }, 254 | { 255 | "name": "Grace Tabone", 256 | "language": "Maltese", 257 | "id": "K4XO8G8DMRNSHF2B", 258 | "bio": "Curabitur sed condimentum felis, ut luctus eros. Duis luctus, lacus eu aliquet convallis, purus elit malesuada ex, vitae rutrum ipsum dui ut magna.", 259 | "version": 5.36 260 | }, 261 | { 262 | "name": "Shafqat Memon", 263 | "language": "Sindhi", 264 | "id": "D8VFLVRXBXMVBRVI", 265 | "bio": "Aliquam scelerisque pretium tellus, sed accumsan est ultrices id. . Curabitur quis commodo quam. Quisque maximus sodales mauris ut elementum. Quisque mauris ligula, efficitur porttitor sodales ac, lacinia non ex.", 266 | "version": 8.95 267 | }, 268 | { 269 | "name": "Zeynep Semet", 270 | "language": "Uyghur", 271 | "id": "Z324TZV8S0FGDSAO", 272 | "bio": "Quisque mauris ligula, efficitur porttitor sodales ac, lacinia non ex. Fusce eu ultrices elit, vel posuere neque. Nulla finibus massa at viverra facilisis.", 273 | "version": 1.03 274 | }, 275 | { 276 | "name": "Meladi Papo", 277 | "language": "Sesotho sa Leboa", 278 | "id": "RJAZQ6BBLRT72CD9", 279 | "bio": "Quisque efficitur vel sapien ut imperdiet. Pellentesque massa sem, scelerisque sit amet odio id, cursus tempor urna. Ut accumsan, est vel fringilla varius, purus augue blandit nisl, eu rhoncus ligula purus vel dolor. Etiam congue dignissim volutpat. Donec congue sapien vel euismod interdum.", 280 | "version": 7.22 281 | }, 282 | { 283 | "name": "Semet Alim", 284 | "language": "Uyghur", 285 | "id": "HI7L2SR4RCS8C8CS", 286 | "bio": "Duis commodo orci ut dolor iaculis facilisis. Ut viverra quis eros eu tincidunt. Lorem ipsum dolor sit amet, consectetur adipiscing elit.", 287 | "version": 1.01 288 | }, 289 | { 290 | "name": "Sabela Veloso", 291 | "language": "Galician", 292 | "id": "QA55WXDLC7SRH97X", 293 | "bio": "Duis commodo orci ut dolor iaculis facilisis. Suspendisse potenti. Cras dictum dolor lacinia lectus vehicula rutrum.", 294 | "version": 7.32 295 | }, 296 | { 297 | "name": "Madule Ledimo", 298 | "language": "Sesotho sa Leboa", 299 | "id": "IHJN2DGJB5O1Y00D", 300 | "bio": "Maecenas non arcu nulla. Aliquam scelerisque pretium tellus, sed accumsan est ultrices id.", 301 | "version": 7.47 302 | }, 303 | { 304 | "name": "Michelle Caruana", 305 | "language": "Maltese", 306 | "id": "EG1I21R75IV9Q0Q8", 307 | "bio": "Nam tristique feugiat est vitae mollis. Morbi ultricies consequat ligula posuere eleifend. Pellentesque massa sem, scelerisque sit amet odio id, cursus tempor urna.", 308 | "version": 4.95 309 | }, 310 | { 311 | "name": "Philip Camilleri", 312 | "language": "Maltese", 313 | "id": "FCO0URUHARX5FDFW", 314 | "bio": "Quisque efficitur vel sapien ut imperdiet. Suspendisse sit amet ullamcorper sem. Aliquam scelerisque pretium tellus, sed accumsan est ultrices id. . Aenean finibus in tortor vel aliquet.", 315 | "version": 9.97 316 | }, 317 | { 318 | "name": "Olalla Romeu", 319 | "language": "Galician", 320 | "id": "WOCMVO6CYPG01ZHY", 321 | "bio": "Maecenas tempus neque ut porttitor malesuada. Sed nec suscipit ligula. Morbi vitae nisi lacinia, laoreet lorem nec, egestas orci. Nullam sodales convallis mauris, sit amet lobortis magna auctor sit amet.", 322 | "version": 1.98 323 | }, 324 | { 325 | "name": "Gulnur Perhat", 326 | "language": "Uyghur", 327 | "id": "VO3M22TTQMBA2XEM", 328 | "bio": "Nullam ac sodales dolor, eu facilisis dui. Etiam mauris magna, fermentum vitae aliquet eu, cursus vitae sapien. Ut accumsan, est vel fringilla varius, purus augue blandit nisl, eu rhoncus ligula purus vel dolor. Maecenas quis nisi nunc. Duis pellentesque ultrices convallis.", 329 | "version": 5.03 330 | }, 331 | { 332 | "name": "Hunadi Makgatho", 333 | "language": "Sesotho sa Leboa", 334 | "id": "MRJDOV2MU7PTCDXE", 335 | "bio": "Phasellus tincidunt sollicitudin posuere. Maecenas quis nisi nunc. Duis luctus, lacus eu aliquet convallis, purus elit malesuada ex, vitae rutrum ipsum dui ut magna.", 336 | "version": 8.18 337 | }, 338 | { 339 | "name": "Charmaine Abela", 340 | "language": "Maltese", 341 | "id": "F6FJP1QDJL944X4Z", 342 | "bio": "Nam rutrum sollicitudin ante tempus consequat. Suspendisse sit amet ullamcorper sem. Morbi ac tellus erat. Sed nec suscipit ligula.", 343 | "version": 6.95 344 | }, 345 | { 346 | "name": "Tumelò Letamo", 347 | "language": "Sesotho sa Leboa", 348 | "id": "F8BL9NPIKV0OWO1X", 349 | "bio": "Aliquam sollicitudin ante ligula, eget malesuada nibh efficitur et. Etiam congue dignissim volutpat. Sed nec suscipit ligula. Nullam sodales convallis mauris, sit amet lobortis magna auctor sit amet.", 350 | "version": 7.17 351 | }, 352 | { 353 | "name": "Aneela Mohan", 354 | "language": "Sindhi", 355 | "id": "CRYN52CXKNJU0YXU", 356 | "bio": "Sed nec suscipit ligula. Phasellus massa ligula, hendrerit eget efficitur eget, tincidunt in ligula. Maecenas tempus neque ut porttitor malesuada.", 357 | "version": 4.45 358 | }, 359 | { 360 | "name": "Koketšo Montjane", 361 | "language": "Sesotho sa Leboa", 362 | "id": "0TTAMXC9TENQCA2O", 363 | "bio": "Curabitur sed condimentum felis, ut luctus eros. Aliquam sollicitudin ante ligula, eget malesuada nibh efficitur et. Etiam mauris magna, fermentum vitae aliquet eu, cursus vitae sapien. Ut maximus, libero nec facilisis fringilla, ex sem sollicitudin leo, non congue tortor ligula in eros.", 364 | "version": 3.61 365 | }, 366 | { 367 | "name": "Tegra Núnez", 368 | "language": "Galician", 369 | "id": "NC1ZUV6B853BZZCW", 370 | "bio": "Maecenas tempus neque ut porttitor malesuada. Pellentesque massa sem, scelerisque sit amet odio id, cursus tempor urna.", 371 | "version": 6.68 372 | }, 373 | { 374 | "name": "Dilnur Qeyser", 375 | "language": "Uyghur", 376 | "id": "JVQ8RQ4YRPGLFMR8", 377 | "bio": "Maecenas non arcu nulla. Nulla finibus massa at viverra facilisis. Integer vehicula, arcu sit amet egestas efficitur, orci justo interdum massa, eget ullamcorper risus ligula tristique libero. Ut maximus, libero nec facilisis fringilla, ex sem sollicitudin leo, non congue tortor ligula in eros.", 378 | "version": 7.93 379 | }, 380 | { 381 | "name": "Tania Agius", 382 | "language": "Maltese", 383 | "id": "WTDGKLDWJLR1BJKR", 384 | "bio": "Etiam congue dignissim volutpat. Pellentesque massa sem, scelerisque sit amet odio id, cursus tempor urna.", 385 | "version": 4.78 386 | }, 387 | { 388 | "name": "Iago Peirallo", 389 | "language": "Galician", 390 | "id": "D51G7XQTX2SPHR52", 391 | "bio": "Aliquam sollicitudin ante ligula, eget malesuada nibh efficitur et. Donec congue sapien vel euismod interdum. Suspendisse potenti. Quisque maximus sodales mauris ut elementum. Quisque maximus sodales mauris ut elementum.", 392 | "version": 6.3 393 | }, 394 | { 395 | "name": "Mpho Lamola", 396 | "language": "Sesotho sa Leboa", 397 | "id": "UGL8EOTXYBW1ILLW", 398 | "bio": "In id elit malesuada, pulvinar mi eu, imperdiet nulla. Curabitur ultricies id urna nec ultrices. Maecenas tempus neque ut porttitor malesuada. In sed ultricies lorem. Nullam sodales convallis mauris, sit amet lobortis magna auctor sit amet.", 399 | "version": 2.05 400 | }, 401 | { 402 | "name": "Josephine Balzan", 403 | "language": "Maltese", 404 | "id": "4OLTG6QD0A2VB432", 405 | "bio": "Maecenas tempus neque ut porttitor malesuada. Sed eu libero maximus nunc lacinia lobortis et sit amet nisi. Maecenas non arcu nulla. Ut accumsan, est vel fringilla varius, purus augue blandit nisl, eu rhoncus ligula purus vel dolor. Curabitur quis commodo quam.", 406 | "version": 7.64 407 | }, 408 | { 409 | "name": "Thabò Motongwane", 410 | "language": "Sesotho sa Leboa", 411 | "id": "NROE4ZZVGKZGDFNO", 412 | "bio": "Donec pellentesque ultrices mi, non consectetur eros luctus non. Suspendisse potenti. Suspendisse potenti.", 413 | "version": 2.07 414 | }, 415 | { 416 | "name": "Mmathabò Mojapelo", 417 | "language": "Sesotho sa Leboa", 418 | "id": "VXJDXYPV5L300IFW", 419 | "bio": "Sed laoreet posuere sapien, ut feugiat nibh gravida at. Duis luctus, lacus eu aliquet convallis, purus elit malesuada ex, vitae rutrum ipsum dui ut magna. Nunc tincidunt laoreet laoreet. .", 420 | "version": 9.36 421 | }, 422 | { 423 | "name": "Kgabo Lerumo", 424 | "language": "Sesotho sa Leboa", 425 | "id": "D63WWKQE2R4TFDIL", 426 | "bio": "Vestibulum pharetra libero et velit gravida euismod. Maecenas tempus neque ut porttitor malesuada. Morbi ultricies consequat ligula posuere eleifend. Quisque efficitur vel sapien ut imperdiet. Nam rutrum sollicitudin ante tempus consequat.", 427 | "version": 6.69 428 | }, 429 | { 430 | "name": "Lawrence Scicluna", 431 | "language": "Maltese", 432 | "id": "0KDA7XKZNNZWL2SR", 433 | "bio": "Donec pellentesque ultrices mi, non consectetur eros luctus non. In id elit malesuada, pulvinar mi eu, imperdiet nulla. Aliquam sollicitudin ante ligula, eget malesuada nibh efficitur et.", 434 | "version": 6.53 435 | }, 436 | { 437 | "name": "Iria Xamardo", 438 | "language": "Galician", 439 | "id": "ULUDKBP9PHBGHX2J", 440 | "bio": "Vivamus id faucibus velit, id posuere leo. Sed eu libero maximus nunc lacinia lobortis et sit amet nisi. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam malesuada blandit erat, nec ultricies leo maximus sed. Ut viverra quis eros eu tincidunt.", 441 | "version": 3.42 442 | }, 443 | { 444 | "name": "Joseph Grech", 445 | "language": "Maltese", 446 | "id": "T4P1164RJBJ8S6XD", 447 | "bio": "Aliquam scelerisque pretium tellus, sed accumsan est ultrices id. Donec lobortis eleifend condimentum.", 448 | "version": 7.68 449 | }, 450 | { 451 | "name": "Napogadi Selepe", 452 | "language": "Sesotho sa Leboa", 453 | "id": "AJK91MKRFIHAQHHG", 454 | "bio": "Quisque maximus sodales mauris ut elementum. Maecenas quis nisi nunc.", 455 | "version": 4.95 456 | }, 457 | { 458 | "name": "Lesetja Theko", 459 | "language": "Sesotho sa Leboa", 460 | "id": "AATM20BURO1DHDAE", 461 | "bio": "Phasellus massa ligula, hendrerit eget efficitur eget, tincidunt in ligula. Etiam mauris magna, fermentum vitae aliquet eu, cursus vitae sapien. Nulla finibus massa at viverra facilisis. Morbi finibus dui sed est fringilla ornare.", 462 | "version": 6.81 463 | }, 464 | { 465 | "name": "Martiño Arxíz", 466 | "language": "Galician", 467 | "id": "CQ56N9MH3WK7H5YQ", 468 | "bio": "Proin tempus eu risus nec mattis. Morbi vitae nisi lacinia, laoreet lorem nec, egestas orci. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam consequat enim lorem, at tincidunt velit ultricies et. Nam rutrum sollicitudin ante tempus consequat. .", 469 | "version": 7.13 470 | }, 471 | { 472 | "name": "Malehumò Ledwaba", 473 | "language": "Sesotho sa Leboa", 474 | "id": "E4F3HGRTKQKCT1SE", 475 | "bio": "Ut accumsan, est vel fringilla varius, purus augue blandit nisl, eu rhoncus ligula purus vel dolor. Curabitur quis commodo quam. Quisque maximus sodales mauris ut elementum. Curabitur sed condimentum felis, ut luctus eros. Curabitur ultricies id urna nec ultrices.", 476 | "version": 6.52 477 | }, 478 | { 479 | "name": "Musa Yasin", 480 | "language": "Uyghur", 481 | "id": "1AF8GIQZ1LF8QW0U", 482 | "bio": "Phasellus tincidunt sollicitudin posuere. Phasellus massa ligula, hendrerit eget efficitur eget, tincidunt in ligula. Ut maximus, libero nec facilisis fringilla, ex sem sollicitudin leo, non congue tortor ligula in eros. Ut accumsan, est vel fringilla varius, purus augue blandit nisl, eu rhoncus ligula purus vel dolor.", 483 | "version": 1.54 484 | }, 485 | { 486 | "name": "Lajwanti Kumari", 487 | "language": "Sindhi", 488 | "id": "INRW3R54RAY7J9IS", 489 | "bio": "In sed ultricies lorem. Sed eu libero maximus nunc lacinia lobortis et sit amet nisi. Quisque mauris ligula, efficitur porttitor sodales ac, lacinia non ex. Lorem ipsum dolor sit amet, consectetur adipiscing elit.", 490 | "version": 9.34 491 | }, 492 | { 493 | "name": "Maria Sammut", 494 | "language": "Maltese", 495 | "id": "BJRF0BWIHJ0Q12A1", 496 | "bio": "Maecenas tempus neque ut porttitor malesuada. Curabitur ultricies id urna nec ultrices.", 497 | "version": 6.83 498 | }, 499 | { 500 | "name": "Rita Busuttil", 501 | "language": "Maltese", 502 | "id": "1QLMU6QZ7EYUNNZV", 503 | "bio": "Phasellus tincidunt sollicitudin posuere. Quisque efficitur vel sapien ut imperdiet. Vestibulum pharetra libero et velit gravida euismod. Maecenas tempus neque ut porttitor malesuada.", 504 | "version": 2.09 505 | }, 506 | { 507 | "name": "Roi Fraguela", 508 | "language": "Galician", 509 | "id": "UAT0M2O42E9M4SFT", 510 | "bio": "Donec congue sapien vel euismod interdum. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce congue aliquam elit ut luctus. Morbi ac tellus erat. Lorem ipsum dolor sit amet, consectetur adipiscing elit.", 511 | "version": 1.08 512 | }, 513 | { 514 | "name": "Matome Molamo", 515 | "language": "Sesotho sa Leboa", 516 | "id": "7HI0UZZLRB9N5CBI", 517 | "bio": "Vestibulum pharetra libero et velit gravida euismod. Fusce eu ultrices elit, vel posuere neque. Duis pellentesque ultrices convallis.", 518 | "version": 9.55 519 | }, 520 | { 521 | "name": "Mapula Selokela", 522 | "language": "Sesotho sa Leboa", 523 | "id": "6ZQTOKQI6K82EE9Q", 524 | "bio": "Duis pellentesque ultrices convallis. Nam laoreet, nunc non suscipit interdum, justo turpis vestibulum massa, non vulputate ex urna at purus. Ut viverra quis eros eu tincidunt. Proin tempus eu risus nec mattis.", 525 | "version": 5.27 526 | }, 527 | { 528 | "name": "Noa Ervello", 529 | "language": "Galician", 530 | "id": "W9FR842CI16V8NU3", 531 | "bio": "Aliquam sollicitudin ante ligula, eget malesuada nibh efficitur et. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam consequat enim lorem, at tincidunt velit ultricies et. Suspendisse sit amet ullamcorper sem. Quisque mauris ligula, efficitur porttitor sodales ac, lacinia non ex.", 532 | "version": 9.33 533 | }, 534 | { 535 | "name": "Naseem Kakepoto", 536 | "language": "Sindhi", 537 | "id": "6C7HZV4WPV9C9KS6", 538 | "bio": "Morbi ultricies consequat ligula posuere eleifend. Fusce congue aliquam elit ut luctus. . Phasellus massa ligula, hendrerit eget efficitur eget, tincidunt in ligula.", 539 | "version": 1.4 540 | }, 541 | { 542 | "name": "sayama Amir", 543 | "language": "Sindhi", 544 | "id": "7K4IJT1X7G0EK9WC", 545 | "bio": "Morbi ac tellus erat. Aliquam scelerisque pretium tellus, sed accumsan est ultrices id. Maecenas quis nisi nunc. Etiam congue dignissim volutpat. Sed nec suscipit ligula.", 546 | "version": 9.48 547 | }, 548 | { 549 | "name": "Mariña Quintá", 550 | "language": "Galician", 551 | "id": "7GXC4OQYXX5JJY9F", 552 | "bio": "Phasellus tincidunt sollicitudin posuere. Morbi ac tellus erat. Nullam ac sodales dolor, eu facilisis dui.", 553 | "version": 8.81 554 | }, 555 | { 556 | "name": "Memet Tursun", 557 | "language": "Uyghur", 558 | "id": "KSFMV2JK2D553083", 559 | "bio": "Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam consequat enim lorem, at tincidunt velit ultricies et. Morbi finibus dui sed est fringilla ornare. Suspendisse sit amet ullamcorper sem.", 560 | "version": 7.56 561 | }, 562 | { 563 | "name": "Carmen Vella", 564 | "language": "Maltese", 565 | "id": "WUALBIMS4E8JS4L2", 566 | "bio": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc aliquet sodales nunc a pulvinar. Morbi vitae nisi lacinia, laoreet lorem nec, egestas orci. Vestibulum pharetra libero et velit gravida euismod.", 567 | "version": 4.55 568 | }, 569 | { 570 | "name": "Sobia Khanam", 571 | "language": "Sindhi", 572 | "id": "YG1ERFWBJ7TIW35D", 573 | "bio": "Phasellus tincidunt sollicitudin posuere. Aliquam scelerisque pretium tellus, sed accumsan est ultrices id. Morbi ultricies consequat ligula posuere eleifend. Curabitur sed condimentum felis, ut luctus eros.", 574 | "version": 4.59 575 | }, 576 | { 577 | "name": "Raheela Ali", 578 | "language": "Sindhi", 579 | "id": "7JGX9SMLD5DE2IMG", 580 | "bio": "Morbi finibus dui sed est fringilla ornare. Maecenas quis nisi nunc. Maecenas tempus neque ut porttitor malesuada. Curabitur ultricies id urna nec ultrices.", 581 | "version": 4.75 582 | }, 583 | { 584 | "name": "Rashid Rajput", 585 | "language": "Sindhi", 586 | "id": "UNBGUGDUATATCLS4", 587 | "bio": "Donec congue sapien vel euismod interdum. Maecenas quis nisi nunc.", 588 | "version": 8.51 589 | }, 590 | { 591 | "name": "Uxía Feal", 592 | "language": "Galician", 593 | "id": "35ZPXUNH1M6W3ZJP", 594 | "bio": "Vestibulum pharetra libero et velit gravida euismod. Vivamus id faucibus velit, id posuere leo.", 595 | "version": 1.31 596 | }, 597 | { 598 | "name": "Andrew Fenech", 599 | "language": "Maltese", 600 | "id": "VEYKDKL8L0R0C7GQ", 601 | "bio": "In sed ultricies lorem. Etiam mauris magna, fermentum vitae aliquet eu, cursus vitae sapien. Sed laoreet posuere sapien, ut feugiat nibh gravida at.", 602 | "version": 2.5 603 | }, 604 | { 605 | "name": "Nicholas Micallef", 606 | "language": "Maltese", 607 | "id": "ZYCAI905154LSICR", 608 | "bio": "Nam tristique feugiat est vitae mollis. Curabitur ultricies id urna nec ultrices. Morbi finibus dui sed est fringilla ornare.", 609 | "version": 6.47 610 | }, 611 | { 612 | "name": "Paul Borg", 613 | "language": "Maltese", 614 | "id": "8AD5MMJ0TD0NJ6H2", 615 | "bio": "Phasellus massa ligula, hendrerit eget efficitur eget, tincidunt in ligula. Etiam mauris magna, fermentum vitae aliquet eu, cursus vitae sapien.", 616 | "version": 3.77 617 | }, 618 | { 619 | "name": "Sara Saleem", 620 | "language": "Sindhi", 621 | "id": "5LPKMTZI7OPSJRBA", 622 | "bio": "Maecenas tempus neque ut porttitor malesuada. Etiam congue dignissim volutpat. Proin tempus eu risus nec mattis. Morbi vitae nisi lacinia, laoreet lorem nec, egestas orci. Duis commodo orci ut dolor iaculis facilisis.", 623 | "version": 5.31 624 | }, 625 | { 626 | "name": "Xurxo Golán", 627 | "language": "Galician", 628 | "id": "526ZUSGXEETODHJK", 629 | "bio": "Ut viverra quis eros eu tincidunt. Morbi finibus dui sed est fringilla ornare. Sed laoreet posuere sapien, ut feugiat nibh gravida at. Duis commodo orci ut dolor iaculis facilisis. In sed ultricies lorem.", 630 | "version": 1.75 631 | }, 632 | { 633 | "name": "Peter Zammit", 634 | "language": "Maltese", 635 | "id": "NNRT5QWNWO2WLS5V", 636 | "bio": "Duis commodo orci ut dolor iaculis facilisis. Maecenas quis nisi nunc.", 637 | "version": 8.23 638 | }, 639 | { 640 | "name": "Maname Mohlare", 641 | "language": "Sesotho sa Leboa", 642 | "id": "KZJZ9SD0DIWTIBUC", 643 | "bio": "Quisque mauris ligula, efficitur porttitor sodales ac, lacinia non ex. Vestibulum pharetra libero et velit gravida euismod. Ut accumsan, est vel fringilla varius, purus augue blandit nisl, eu rhoncus ligula purus vel dolor. Sed eu libero maximus nunc lacinia lobortis et sit amet nisi.", 644 | "version": 8.95 645 | }, 646 | { 647 | "name": "Tshepè Mobu", 648 | "language": "Sesotho sa Leboa", 649 | "id": "8CH586LQR7ZCP73P", 650 | "bio": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla finibus massa at viverra facilisis.", 651 | "version": 7.82 652 | }, 653 | { 654 | "name": "Monica Lohana", 655 | "language": "Sindhi", 656 | "id": "KP1C2WN3DN1R3Y52", 657 | "bio": "Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam consequat enim lorem, at tincidunt velit ultricies et. Aenean finibus in tortor vel aliquet. Nam laoreet, nunc non suscipit interdum, justo turpis vestibulum massa, non vulputate ex urna at purus. Morbi vitae nisi lacinia, laoreet lorem nec, egestas orci.", 658 | "version": 7.95 659 | }, 660 | { 661 | "name": "Patigul Rahman", 662 | "language": "Uyghur", 663 | "id": "NXMNLB0SOYET1VMN", 664 | "bio": "In sed ultricies lorem. Proin tempus eu risus nec mattis. Nam rutrum sollicitudin ante tempus consequat. Aliquam scelerisque pretium tellus, sed accumsan est ultrices id.", 665 | "version": 2.98 666 | } 667 | ] -------------------------------------------------------------------------------- /rand/index.no-alloc.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Deep partial of `T` 3 | * 4 | * When `S` is defined, reading `[S]: true` from an object anywhere in the tree 5 | * coerces it and all its children back into a non-partial (`Complete<>`) mode. 6 | */ 7 | export type Incomplete = 8 | | (T extends (infer V)[] 9 | ? Exclude, undefined>[] 10 | : T extends object 11 | ? S extends string 12 | ? 13 | | Complete 14 | | ({ [K in S]?: never } & { 15 | [P in keyof T]?: Incomplete 16 | }) 17 | : { 18 | [P in keyof T]?: Incomplete 19 | } 20 | : T) 21 | | undefined 22 | 23 | /** 24 | * Deeply adds the entry `[S]: true` to every object in `T` 25 | */ 26 | export type Complete = T extends (infer V)[] 27 | ? Complete[] 28 | : T extends object 29 | ? { [K in S]: true } & { 30 | [P in keyof T]: Complete 31 | } 32 | : T 33 | 34 | type IncompleteJsonOptions = { 35 | /** 36 | * When enabled, strings will only be emitted when their full value has been read. 37 | */ 38 | prohibitPartialStrings?: boolean 39 | 40 | /** 41 | * When enabled, numbers will only be emitted when their full value has been read. 42 | * 43 | * *Note*: If the JSON consists of only a single numeric value and this option is enabled, 44 | * the value will *never* be emitted, as it is impossible to tell whether the number is complete. 45 | */ 46 | prohibitPartialNumbers?: boolean 47 | 48 | /** 49 | * Optional property to be added at the end of each object to signify it has been fully streamed. 50 | * This will be set `true` for complete objects and will be undefined otherwise. 51 | * 52 | * Example: 53 | * ```ts 54 | * > IncompleteJson.parse('[{"foo": "bar"}, {"foo": "b', 55 | * { completeObjectSentinel: '__done' }) 56 | * // [{foo: "bar", __done: true}, {foo: "b"}] 57 | * ``` 58 | */ 59 | completeObjectSentinel?: string 60 | } 61 | 62 | export class IncompleteJson { 63 | /** 64 | * Parse a prefix of a JSON serialized string into as much data as possible 65 | * Options available to ensure atomic string/number parsing and mark completed objects 66 | */ 67 | static parse< 68 | T, 69 | O extends IncompleteJsonOptions, 70 | S extends string | undefined = O["completeObjectSentinel"], 71 | >(string: string, options?: O): Incomplete { 72 | if (!string) return undefined 73 | 74 | const parser = new IncompleteJson(options) 75 | parser.addChunk(string) 76 | if (!options?.prohibitPartialNumbers || !string.match(/(\d|\.)$/)) { 77 | parser.done() 78 | } 79 | 80 | return parser.readValue() 81 | } 82 | 83 | /** 84 | * Parse a ReadableStream of JSON data into a ReadableStream of 85 | * as much data as can be extracted from the stream at each moment in time. 86 | */ 87 | static fromReadable< 88 | T, 89 | O extends IncompleteJsonOptions, 90 | S extends string | undefined = O["completeObjectSentinel"], 91 | >( 92 | readable: ReadableStream, 93 | options?: IncompleteJsonOptions, 94 | ): ReadableStream> { 95 | const parser = new IncompleteJson(options) 96 | let prior: Incomplete 97 | 98 | const transformer = new TransformStream>({ 99 | start() {}, 100 | transform(chunk, controller) { 101 | parser.addChunk(chunk) 102 | const next = parser.readValue() 103 | if (next !== prior) { 104 | controller.enqueue(next) 105 | prior = next 106 | } 107 | }, 108 | flush(controller) { 109 | parser.done() 110 | const next = parser.readValue() 111 | if (next !== prior) { 112 | controller.enqueue(next) 113 | prior = next 114 | } 115 | }, 116 | }) 117 | return readable.pipeThrough(transformer) 118 | } 119 | 120 | // private consumed = "" 121 | private consumedNoAlloc = "" 122 | private unconsumed = "" 123 | private inString = false 124 | private charsNeededToClose: string[] = [] 125 | private context: ("key" | "val" | "arr")[] = [] 126 | 127 | private isDone = false 128 | 129 | private truncationInfo: 130 | | { index: number; append: string; result?: Incomplete } 131 | | undefined = undefined 132 | 133 | constructor(private options?: IncompleteJsonOptions) {} 134 | 135 | /** 136 | * Add a chunk of data to the stream. 137 | * 138 | * This runs in time linear to the size of the chunk. 139 | */ 140 | addChunk(chunk: string) { 141 | if (this.isDone) throw Error("Cannot add chunk to parser marked done") 142 | 143 | // called to save the current state as a "safe" spot to truncate and provide a result 144 | const markTruncateSpot = (delta: number) => 145 | (this.truncationInfo = { 146 | index: this.consumedNoAlloc.length + delta + 1, 147 | append: this.charsNeededToClose.join(""), 148 | }) 149 | 150 | // consume everything we didn't consume last time, then the new chunk 151 | const toConsume = this.unconsumed + chunk 152 | 153 | this.unconsumed = "" 154 | 155 | for (let i = 0; i < toConsume.length; i++) { 156 | const c = toConsume[i] 157 | 158 | // atomically consume escape sequences 159 | if (this.inString && c === "\\") { 160 | // we have seen the `\` 161 | i++ 162 | const escaped = toConsume[i] 163 | // unicode escapes are of the form \uXXXX 164 | if (escaped === "u") { 165 | // we have seen the `u` 166 | i++ 167 | if (toConsume[i + 3] !== undefined) { 168 | // if we can grab 4 chars forward, do so 169 | // this.consumed += c + escaped + toConsume.slice(i, i + 4) 170 | } else { 171 | // otherwise, save the rest of the string for later 172 | this.unconsumed = c + escaped + toConsume.slice(i, i + 4) 173 | } 174 | // we have seen either 4 chars or until the end of the string 175 | // (if this goes over the end the loop exists normally) 176 | i += 4 177 | } else if (escaped !== undefined) { 178 | // standard two char escape (\n, \t, etc.) 179 | // this.consumed += c + escaped 180 | } else { 181 | // end of chunk. save the \ to tack onto front of next chunk 182 | this.unconsumed = c 183 | } 184 | 185 | // restart from after the sequence 186 | continue 187 | } 188 | 189 | // inject completed object sentinels as required 190 | // if (!this.inString && this.options?.completeObjectSentinel && c === "}") { 191 | // if (!this.consumed.trim().endsWith("{")) { 192 | // this.consumed += "," 193 | // } 194 | // this.consumed += `"${this.options.completeObjectSentinel}": true` 195 | // } 196 | 197 | // consume the char itself 198 | // this.consumed += c 199 | 200 | // when in string... 201 | if (this.inString && c !== '"') { 202 | // if partial strings allowed, every location in a string is a potential truncate spot 203 | // EXCEPT in key strings - the following cannot be completed: { "ab 204 | if ( 205 | this.context[0] !== "key" && 206 | !this.options?.prohibitPartialStrings 207 | ) { 208 | markTruncateSpot(i) 209 | } 210 | 211 | // skip over the special char handling 212 | continue 213 | } 214 | 215 | // consuming a matching closing char - pop it from the stack 216 | if (c === this.charsNeededToClose[0]) { 217 | this.charsNeededToClose.shift() 218 | 219 | // good place to truncate, unless we're in a key 220 | if (this.context[0] !== "key") { 221 | markTruncateSpot(i) 222 | } 223 | } 224 | 225 | if (!this.inString && !this.options?.prohibitPartialNumbers) { 226 | // symbols found in numbers 227 | if (c === "e" || c === "." || c === "E") { 228 | // unparsable as suffixes, trim them if partials allowed 229 | markTruncateSpot(i - 1) 230 | } 231 | } 232 | 233 | if (c === '"') { 234 | // if we aren't prohibiting partial strings and we are starting a new string, 235 | // note how to close this string 236 | if (!this.options?.prohibitPartialStrings && !this.inString) { 237 | this.charsNeededToClose.unshift('"') 238 | } 239 | 240 | // toggle string mode 241 | this.inString = !this.inString 242 | } 243 | 244 | if (c === ",") { 245 | // truncate right before the `,` 246 | markTruncateSpot(i - 1) 247 | 248 | // when parsing object, comma switches from val context to key 249 | if (this.context[0] === "val") { 250 | this.context[0] = "key" 251 | } 252 | } 253 | 254 | // colon switches from key context to val 255 | if (c === ":") { 256 | if (this.context[0] === "key") { 257 | this.context[0] = "val" 258 | } 259 | } 260 | 261 | // in array: strings can always be truncated 262 | if (c === "[") { 263 | this.context.unshift("arr") 264 | this.charsNeededToClose.unshift("]") // + this.charsNeededToClose 265 | markTruncateSpot(i) 266 | } 267 | 268 | // in object: strings can be truncated in values, but not keys! 269 | if (c === "{") { 270 | this.context.unshift("key") 271 | this.charsNeededToClose.unshift("}") // + this.charsNeededToClose 272 | markTruncateSpot(i) 273 | } 274 | 275 | // exiting our context, pop! 276 | if (c === "}" || c === "]") { 277 | this.context.shift() 278 | } 279 | } 280 | 281 | this.consumedNoAlloc += chunk.slice( 282 | 0, 283 | chunk.length - this.unconsumed.length, 284 | ) 285 | } 286 | 287 | /** 288 | * Mark the stream complete. 289 | * This will force emit values that were not emitted previously due to potentially being incomplete. 290 | * 291 | * Example: a chunk that ends in a number will not be emitted because the next chunk may continue with a number, 292 | * which would be appended to the existing value (significantly changing the meaning of the represented data) 293 | * ```ts 294 | * > const ij = new IncompleteJSON() 295 | * > ij.addChunk('1234') 296 | * > ij.readValue() 297 | * undefined 298 | * > ij.addChunk('5') 299 | * > ij.readValue() 300 | * undefined 301 | * > ij.done() 302 | * > ij.readValue() 303 | * 12345 304 | * ``` 305 | */ 306 | done() { 307 | if (this.isDone) return 308 | this.isDone = true 309 | 310 | const rawData = this.consumedNoAlloc + this.charsNeededToClose.join("") 311 | try { 312 | const result = this.cachedJSONParse(rawData) 313 | this.truncationInfo = { 314 | index: this.consumedNoAlloc.length, 315 | append: this.charsNeededToClose.join(""), 316 | result, 317 | } 318 | } catch { 319 | // pass: the JSON parse is expected to fail in some cases. 320 | // the prior truncationInfo will still be good. 321 | } 322 | } 323 | 324 | /** 325 | * Attempt to parse the consumed chunks into as much data as is available 326 | * 327 | * This operation runs in time linear to the length of the stream 328 | * 329 | * While modern JSON parsers are significantly faster than modern LLM's, care 330 | * should be taken on very large inputs to not call readValue more then needed 331 | */ 332 | readValue(): Incomplete { 333 | if (!this.consumedNoAlloc || !this.truncationInfo) { 334 | return undefined 335 | } 336 | 337 | if (!("result" in this.truncationInfo)) { 338 | this.truncationInfo.result = this.cachedJSONParse( 339 | this.consumedNoAlloc.slice(0, this.truncationInfo.index) + 340 | this.truncationInfo.append, 341 | ) 342 | } 343 | 344 | return this.truncationInfo.result 345 | } 346 | 347 | private rawParseCache = { key: "", value: undefined as Incomplete } 348 | private cachedJSONParse(str: string): Incomplete { 349 | if (str !== this.rawParseCache.key) { 350 | this.rawParseCache = { 351 | key: str, 352 | value: JSON.parse(str, (k, v) => v), 353 | } 354 | } 355 | return this.rawParseCache.value 356 | } 357 | } 358 | -------------------------------------------------------------------------------- /rand/queryOpenAI.ts: -------------------------------------------------------------------------------- 1 | import { IncompleteJson, ReadableFromOpenAIResponse } from "../index.ts" 2 | 3 | const run = async () => { 4 | const messages = [ 5 | { 6 | content: 7 | "You are a serialized data emitter that always responds to the users request in proper JSON.", 8 | role: "system", 9 | }, 10 | { 11 | content: 12 | "Please give a big example data object with lots of objects, fields, arrays, booleans, strings, numbers, escape sequences, and nulls to stress test my JSON parser! Try to use obsure JSON features like unicode escape sequences and exponential number formats to really throw a wrench in the parser! Respond only with the JSON, do not add any comments.", 13 | role: "user", 14 | }, 15 | ] 16 | 17 | const response = await fetch("https://api.openai.com/v1/chat/completions", { 18 | method: "POST", 19 | headers: { 20 | "Content-Type": "application/json", 21 | Authorization: "Bearer " + process.env.OPEN_AI_KEY, 22 | }, 23 | body: JSON.stringify({ 24 | model: "gpt-3.5-turbo", 25 | stream: true, 26 | messages, 27 | }), 28 | }) 29 | const plaintext = ReadableFromOpenAIResponse(response) 30 | if (plaintext.error) { 31 | console.log("Error!", await plaintext.error) 32 | } else { 33 | const parsed = IncompleteJson.fromReadable(plaintext) 34 | for await (const b of parsed) { 35 | console.log(b) 36 | } 37 | } 38 | } 39 | 40 | run() 41 | -------------------------------------------------------------------------------- /rand/simplisticVanillaJSONParser.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Parse json in vanilla JS ( { 5 | test("defaults", () => { 6 | const tests = [ 7 | { input: "", output: undefined }, 8 | { input: '"', output: "" }, 9 | 10 | { input: "n", output: undefined }, 11 | { input: "nu", output: undefined }, 12 | { input: "nul", output: undefined }, 13 | { input: "null", output: null }, 14 | 15 | { input: "tru", output: undefined }, 16 | { input: "true", output: true }, 17 | { input: "fals", output: undefined }, 18 | { input: "false", output: false }, 19 | 20 | { input: "0", output: 0 }, 21 | { input: "-", output: undefined }, 22 | { input: "1", output: 1 }, 23 | { input: "-1", output: -1 }, 24 | { input: "-1.", output: -1 }, 25 | { input: "-1.3", output: -1.3 }, 26 | { input: "-1.3e", output: -1.3 }, 27 | { input: "-1.3e2", output: -130 }, 28 | { input: "-1.3E", output: -1.3 }, 29 | { input: "-1.3E2", output: -130 }, 30 | { input: "-1.3E+", output: -1.3 }, 31 | { input: "-1.3E+2", output: -130 }, 32 | { input: "-1.3E-", output: -1.3 }, 33 | { input: "-1.3E-2", output: -0.013 }, 34 | 35 | { input: "[", output: [] }, 36 | { input: "[0", output: [0] }, 37 | { input: "[1", output: [1] }, 38 | { input: "[12", output: [12] }, 39 | { input: "[12.", output: [12] }, 40 | { input: "[12.3", output: [12.3] }, 41 | { input: "[0]", output: [0] }, 42 | { input: '["', output: [""] }, 43 | { input: "[0,", output: [0] }, 44 | 45 | { input: "{", output: {} }, 46 | { input: '{"a', output: {} }, 47 | { input: '{"a"', output: {} }, 48 | { input: '{"a": ', output: {} }, 49 | { input: '{"a": 0', output: { a: 0 } }, 50 | { input: '{"a": 0,', output: { a: 0 } }, 51 | { input: '{"a": "', output: { a: "" } }, 52 | { input: '{"a": "b', output: { a: "b" } }, 53 | { input: '{"a": "b"', output: { a: "b" } }, 54 | { input: '{"a": "b",', output: { a: "b" } }, 55 | 56 | { input: `{"key": true`, output: { key: true } }, 57 | { input: `{"key": false`, output: { key: false } }, 58 | { input: `{"key": null`, output: { key: null } }, 59 | 60 | { input: "[{", output: [{}] }, 61 | { input: '[{"', output: [{}] }, 62 | { input: '[{"a', output: [{}] }, 63 | { input: '[{"a"', output: [{}] }, 64 | { input: '[{"a": ', output: [{}] }, 65 | { input: '[{"a": 0', output: [{ a: 0 }] }, 66 | { input: '[{"a": 0, ', output: [{ a: 0 }] }, 67 | { input: '[{"a": 0, "', output: [{ a: 0 }] }, 68 | { input: '[{"a": 0, "b', output: [{ a: 0 }] }, 69 | { input: '[{"a": 0, "b"', output: [{ a: 0 }] }, 70 | { input: '[{"a": 0, "b":', output: [{ a: 0 }] }, 71 | { input: '[{"a": 0, "b": 1', output: [{ a: 0, b: 1 }] }, 72 | 73 | { input: "[{},", output: [{}] }, 74 | { input: "[{},1", output: [{}, 1] }, 75 | { input: '[{},"', output: [{}, ""] }, 76 | { input: '[{},"abc', output: [{}, "abc"] }, 77 | ] 78 | 79 | for (const test of tests) { 80 | const actual = JSON.stringify(IncompleteJson.parse(test.input)) 81 | const expected = JSON.stringify(test.output) 82 | 83 | expect(actual).toEqual(expected) 84 | } 85 | }) 86 | 87 | test("prohibitPartialStrings", () => { 88 | const tests = [ 89 | { input: "", output: undefined }, 90 | { input: '"', output: undefined }, 91 | 92 | { input: "n", output: undefined }, 93 | { input: "nu", output: undefined }, 94 | { input: "nul", output: undefined }, 95 | { input: "null", output: null }, 96 | 97 | { input: "tru", output: undefined }, 98 | { input: "true", output: true }, 99 | { input: "fals", output: undefined }, 100 | { input: "false", output: false }, 101 | 102 | { input: "[", output: [] }, 103 | { input: "[0", output: [0] }, 104 | { input: "[0]", output: [0] }, 105 | { input: '["', output: [] }, 106 | { input: "[0,", output: [0] }, 107 | 108 | { input: "[0,", output: [0] }, 109 | { input: "[1,", output: [1] }, 110 | { input: "[-1,", output: [-1] }, 111 | { input: "[-1.3,", output: [-1.3] }, 112 | { input: "[-1.3e2,", output: [-130] }, 113 | { input: "[-1.3E2,", output: [-130] }, 114 | { input: "[-1.3E+2,", output: [-130] }, 115 | { input: "[-1.3E-2,", output: [-0.013] }, 116 | 117 | { input: "{", output: {} }, 118 | { input: '{"a', output: {} }, 119 | { input: '{"a"', output: {} }, 120 | { input: '{"a": ', output: {} }, 121 | { input: '{"a": 0', output: { a: 0 } }, 122 | { input: '{"a": 0,', output: { a: 0 } }, 123 | { input: '{"a": "', output: {} }, 124 | { input: '{"a": "b', output: {} }, 125 | { input: '{"a": "b"', output: { a: "b" } }, 126 | { input: '{"a": "b",', output: { a: "b" } }, 127 | 128 | { input: "[{", output: [{}] }, 129 | { input: '[{"', output: [{}] }, 130 | { input: '[{"a', output: [{}] }, 131 | { input: '[{"a"', output: [{}] }, 132 | { input: '[{"a": ', output: [{}] }, 133 | { input: '[{"a": 0', output: [{ a: 0 }] }, 134 | { input: '[{"a": 0, ', output: [{ a: 0 }] }, 135 | { input: '[{"a": 0, "', output: [{ a: 0 }] }, 136 | { input: '[{"a": 0, "b', output: [{ a: 0 }] }, 137 | { input: '[{"a": 0, "b"', output: [{ a: 0 }] }, 138 | { input: '[{"a": 0, "b":', output: [{ a: 0 }] }, 139 | { input: '[{"a": 0, "b": 1', output: [{ a: 0, b: 1 }] }, 140 | 141 | { input: "[{},", output: [{}] }, 142 | { input: "[{},1", output: [{}, 1] }, 143 | { input: '[{},"', output: [{}] }, 144 | { input: '[{},"abc', output: [{}] }, 145 | { input: '[{},"abc"', output: [{}, "abc"] }, 146 | ] 147 | 148 | for (const test of tests) { 149 | const actual = JSON.stringify( 150 | IncompleteJson.parse(test.input, { 151 | prohibitPartialStrings: true, 152 | }), 153 | ) 154 | const expected = JSON.stringify(test.output) 155 | expect(actual).toEqual(expected) 156 | } 157 | }) 158 | 159 | test("prohibitPartialNumbers", () => { 160 | const tests = [ 161 | { input: "", output: undefined }, 162 | { input: '"', output: "" }, 163 | 164 | { input: "n", output: undefined }, 165 | { input: "nu", output: undefined }, 166 | { input: "nul", output: undefined }, 167 | { input: "null", output: null }, 168 | 169 | { input: "tru", output: undefined }, 170 | { input: "true", output: true }, 171 | { input: "fals", output: undefined }, 172 | { input: "false", output: false }, 173 | 174 | { input: "0", output: undefined }, 175 | { input: "-", output: undefined }, 176 | { input: "1", output: undefined }, 177 | { input: "-1", output: undefined }, 178 | { input: "-1.", output: undefined }, 179 | { input: "-1.3", output: undefined }, 180 | { input: "-1.3e", output: undefined }, 181 | { input: "-1.3e2", output: undefined }, 182 | { input: "-1.3E", output: undefined }, 183 | { input: "-1.3E2", output: undefined }, 184 | { input: "-1.3E+", output: undefined }, 185 | { input: "-1.3E+2", output: undefined }, 186 | { input: "-1.3E-", output: undefined }, 187 | { input: "-1.3E-2", output: undefined }, 188 | 189 | { input: "[0,", output: [0] }, 190 | { input: "[1,", output: [1] }, 191 | { input: "[-1,", output: [-1] }, 192 | { input: "[-1.3,", output: [-1.3] }, 193 | { input: "[-1.3e2,", output: [-130] }, 194 | { input: "[-1.3E2,", output: [-130] }, 195 | { input: "[-1.3E+2,", output: [-130] }, 196 | { input: "[-1.3E-2,", output: [-0.013] }, 197 | 198 | { input: "[", output: [] }, 199 | { input: "[0", output: [] }, 200 | { input: "[1", output: [] }, 201 | { input: "[12", output: [] }, 202 | { input: "[12.", output: [] }, 203 | { input: "[12.3", output: [] }, 204 | { input: "[1,", output: [1] }, 205 | { input: "[12,", output: [12] }, 206 | { input: "[12.3,", output: [12.3] }, 207 | { input: "[0]", output: [0] }, 208 | { input: '["', output: [""] }, 209 | { input: "[0,", output: [0] }, 210 | 211 | { input: "{", output: {} }, 212 | { input: '{"a', output: {} }, 213 | { input: '{"a"', output: {} }, 214 | { input: '{"a": ', output: {} }, 215 | { input: '{"a": 0', output: {} }, 216 | { input: '{"a": 0,', output: { a: 0 } }, 217 | { input: '{"a": "', output: { a: "" } }, 218 | { input: '{"a": "b', output: { a: "b" } }, 219 | { input: '{"a": "b"', output: { a: "b" } }, 220 | { input: '{"a": "b",', output: { a: "b" } }, 221 | 222 | { input: "[{", output: [{}] }, 223 | { input: '[{"', output: [{}] }, 224 | { input: '[{"a', output: [{}] }, 225 | { input: '[{"a"', output: [{}] }, 226 | { input: '[{"a": ', output: [{}] }, 227 | { input: '[{"a": 0', output: [{}] }, 228 | { input: '[{"a": 0, ', output: [{ a: 0 }] }, 229 | { input: '[{"a": 0, "', output: [{ a: 0 }] }, 230 | { input: '[{"a": 0, "b', output: [{ a: 0 }] }, 231 | { input: '[{"a": 0, "b"', output: [{ a: 0 }] }, 232 | { input: '[{"a": 0, "b":', output: [{ a: 0 }] }, 233 | { input: '[{"a": 0, "b": 1', output: [{ a: 0 }] }, 234 | { input: '[{"a": 0, "b": 1,', output: [{ a: 0, b: 1 }] }, 235 | 236 | { input: "[{},", output: [{}] }, 237 | { input: "[{},1", output: [{}] }, 238 | { input: "[{},1,", output: [{}, 1] }, 239 | { input: '[{},"', output: [{}, ""] }, 240 | { input: '[{},"abc', output: [{}, "abc"] }, 241 | ] 242 | 243 | for (const testCase of tests) { 244 | const actual = JSON.stringify( 245 | IncompleteJson.parse(testCase.input, { 246 | prohibitPartialNumbers: true, 247 | }), 248 | ) 249 | 250 | const expected = JSON.stringify(testCase.output) 251 | 252 | expect(actual).toEqual(expected) 253 | } 254 | }) 255 | 256 | test("completeObjectSentinel", () => { 257 | const tests = [ 258 | { input: "[", output: [] }, 259 | { input: "[0", output: [0] }, 260 | { input: "[0]", output: [0, "__array_is_done"] }, 261 | { input: '["', output: [""] }, 262 | { input: "[0,", output: [0] }, 263 | 264 | { input: "{", output: {} }, 265 | { input: '{"a', output: {} }, 266 | { input: '{"a"', output: {} }, 267 | { input: '{"a": ', output: {} }, 268 | { input: '{"a": 0', output: { a: 0 } }, 269 | { input: '{"a": 0,', output: { a: 0 } }, 270 | { input: '{"a": "', output: { a: "" } }, 271 | { input: '{"a": "b', output: { a: "b" } }, 272 | { input: '{"a": "b"', output: { a: "b" } }, 273 | { input: '{"a": "b",', output: { a: "b" } }, 274 | { input: '{"a": "b"}', output: { a: "b", __done: true } }, 275 | 276 | { input: "[{", output: [{}] }, 277 | { input: "[{}", output: [{ __done: true }] }, 278 | { input: '[{"', output: [{}] }, 279 | { input: '[{"a', output: [{}] }, 280 | { input: '[{"a"', output: [{}] }, 281 | { input: '[{"a": ', output: [{}] }, 282 | { input: '[{"a": 0', output: [{ a: 0 }] }, 283 | { input: '[{"a": 0}', output: [{ a: 0, __done: true }] }, 284 | { input: '[{"a": 0, ', output: [{ a: 0 }] }, 285 | { input: '[{"a": 0, "', output: [{ a: 0 }] }, 286 | { input: '[{"a": 0, "b', output: [{ a: 0 }] }, 287 | { input: '[{"a": 0, "b"', output: [{ a: 0 }] }, 288 | { input: '[{"a": 0, "b":', output: [{ a: 0 }] }, 289 | { input: '[{"a": 0, "b": 1', output: [{ a: 0, b: 1 }] }, 290 | { input: '[{"a": 0, "b": 1}', output: [{ a: 0, b: 1, __done: true }] }, 291 | { 292 | input: '[{"a": 0, "b": 1}, {', 293 | output: [{ a: 0, b: 1, __done: true }, {}], 294 | }, 295 | { 296 | input: '[{"a": 0, "b": 1}, {}', 297 | output: [{ a: 0, b: 1, __done: true }, { __done: true }], 298 | }, 299 | 300 | { input: "[{},", output: [{ __done: true }] }, 301 | { input: "[{},1", output: [{ __done: true }, 1] }, 302 | { input: '[{},"', output: [{ __done: true }, ""] }, 303 | { input: '[{},"abc', output: [{ __done: true }, "abc"] }, 304 | ] 305 | 306 | for (const test of tests) { 307 | const parse = IncompleteJson.parse(test.input) 308 | const actual = JSON.stringify(parse, (k, v) => { 309 | if (v && v[ItemDoneStreaming]) { 310 | v["__done"] = true 311 | if (Array.isArray(v)) { 312 | v.push("__array_is_done") 313 | } 314 | } 315 | return v 316 | }) 317 | const expected = JSON.stringify(test.output) 318 | expect(actual).toEqual(expected) 319 | } 320 | }) 321 | 322 | test("escape sequences", () => { 323 | const tests = [ 324 | { input: '"\\', output: "" }, 325 | { input: '"\\\\', output: "\\" }, 326 | { input: '"\\n"', output: "\n" }, 327 | { input: '"\\"', output: '"' }, 328 | { input: '"\\u"', output: "" }, 329 | { input: '"\\u1"', output: "" }, 330 | { input: '"\\u12"', output: "" }, 331 | { input: '"\\u123"', output: "" }, 332 | { input: '"\\u1234"', output: "ሴ" }, 333 | { input: '"\\u1234hello"', output: "ሴhello" }, 334 | { input: `{"key": "v\\`, output: { key: "v" } }, 335 | { input: `{"key": "v\\n`, output: { key: "v\n" } }, 336 | ] 337 | 338 | for (const testCase of tests) { 339 | const actual = JSON.stringify(IncompleteJson.parse(testCase.input)) 340 | const expected = JSON.stringify(testCase.output) 341 | expect(actual).toEqual(expected) 342 | } 343 | }) 344 | }) 345 | -------------------------------------------------------------------------------- /test/spec.test.ts: -------------------------------------------------------------------------------- 1 | import { IncompleteJson, IncompleteJsonOptions } from "../index.ts" 2 | import { e1, e2, verifyIsValidPartial } from "./utils.test.ts" 3 | 4 | import { describe, expect, test } from "bun:test" 5 | 6 | describe("spec-based parse tests", () => { 7 | test("simple examples", () => { 8 | testParseObjectAllOptions("hello world") 9 | testParseObjectAllOptions(true) 10 | testParseObjectAllOptions(false) 11 | testParseObjectAllOptions(null) 12 | testParseObjectAllOptions([]) 13 | testParseObjectAllOptions([null, true, false, 1, "hello"]) 14 | testParseObjectAllOptions({ abc: "123" }) 15 | 16 | testParseAllPrefixes("1989", {}) 17 | testParseAllPrefixes("1989", { prohibitPartialStrings: true }) 18 | }) 19 | 20 | test("tricky examples", () => { 21 | testParseObjectAllOptions({ 22 | '"hello"world"!': 'Some " \tvalue! \n []]]] }}{{{}}}', 23 | "boop a \u1234!": { "\u2312": null }, 24 | "\u1234!": 'Some " \tvalue! \n []]]] }}{{{}}}', 25 | }) 26 | testParseObjectAllOptions("\u1234") 27 | testParseObjectAllOptions("\\u1234") 28 | testParseStringAllOptions('"hello\\u1234dfd"') 29 | }) 30 | test("example object 1", () => { 31 | testParseObjectAllOptions(e1) 32 | }) 33 | test("example object 2", () => { 34 | testParseObjectAllOptions(e2) 35 | testParseObjectAllOptions({ e2 }) 36 | testParseObjectAllOptions([[e2]]) 37 | }) 38 | }) 39 | 40 | const testParseObjectAllOptions = (object: any) => { 41 | const strs = [JSON.stringify(object), JSON.stringify(object, null, 2)] 42 | const opts: IncompleteJsonOptions[] = [ 43 | {}, 44 | { prohibitPartialNumbers: true }, 45 | { prohibitPartialStrings: true }, 46 | { prohibitPartialNumbers: true, prohibitPartialStrings: true }, 47 | ] 48 | for (const str of strs) { 49 | for (const opt of opts) { 50 | testParseAllPrefixes(str, opt) 51 | } 52 | } 53 | } 54 | 55 | const testParseStringAllOptions = (str: string) => { 56 | const opts: IncompleteJsonOptions[] = [ 57 | {}, 58 | { prohibitPartialNumbers: true }, 59 | { prohibitPartialStrings: true }, 60 | { prohibitPartialNumbers: true, prohibitPartialStrings: true }, 61 | ] 62 | 63 | for (const opt of opts) { 64 | testParseAllPrefixes(str, opt) 65 | } 66 | } 67 | 68 | const testParseAllPrefixes = ( 69 | fullString: string, 70 | options: IncompleteJsonOptions, 71 | ) => { 72 | const fullValue = JSON.parse(fullString) 73 | 74 | // verify partial data always satisfies spec 75 | for (let i = 0; i <= fullString.length; i++) { 76 | const prefix = fullString.slice(0, i) 77 | const partialValue = IncompleteJson.parse(prefix, options) 78 | if (partialValue) { 79 | verifyIsValidPartial(partialValue, fullValue, options) 80 | } 81 | } 82 | 83 | // verify amount of data is monotonically increasing 84 | for (let i = 0; i < fullString.length; i++) { 85 | const a = JSON.stringify( 86 | IncompleteJson.parse(fullString.slice(0, i), options), 87 | ) 88 | const b = JSON.stringify( 89 | IncompleteJson.parse(fullString.slice(0, i + 1), options), 90 | ) 91 | if (a !== undefined) { 92 | expect(b).toBeDefined() 93 | expect(a.length).toBeLessThanOrEqual(b.length) 94 | } 95 | } 96 | 97 | // verify final data is equal to desired 98 | expect(JSON.stringify(IncompleteJson.parse(fullString, options))).toEqual( 99 | JSON.stringify(JSON.parse(fullString)), 100 | ) 101 | } 102 | export { verifyIsValidPartial } 103 | -------------------------------------------------------------------------------- /test/stream.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, test } from "bun:test" 2 | import { IncompleteJsonOptions, IncompleteJson } from "../index.ts" 3 | import { verifyIsValidPartial } from "./spec.test" 4 | import { e1 } from "./utils.test.ts" 5 | 6 | describe("spec-based stream tests", () => { 7 | test("plain literals", async () => { 8 | await testAllOptions(JSON.stringify(1989)) 9 | await testAllOptions(JSON.stringify(true)) 10 | await testAllOptions(JSON.stringify(false)) 11 | await testAllOptions(JSON.stringify(null)) 12 | await testAllOptions(JSON.stringify("Hello world! 🤷‍♀️")) 13 | }) 14 | 15 | test("example object", async () => { 16 | await testAllOptions(JSON.stringify(e1)) // randomized chunking 17 | await testAllOptions(JSON.stringify(e1)) 18 | await testAllOptions(JSON.stringify(e1)) 19 | }) 20 | 21 | test("tricky objects", async () => { 22 | await testAllOptions( 23 | JSON.stringify({ 24 | 'dfd"dfasd': "hello\\u1234567", 25 | 'dfd\\"df\\u1234asd': "hello\\u1234567", 26 | }), 27 | ) 28 | }) 29 | 30 | test("internal consistency", () => { 31 | const ij = new IncompleteJson() 32 | ij.addChunk(`{ "name": "Bob Johnson", "age": 35, "isEmployee": true`) 33 | expect(() => ij.readValue()).not.toThrow() 34 | }) 35 | }) 36 | 37 | const testAllOptions = async (string: string) => { 38 | await testParseAllReadables(JSON.stringify(string), {}) 39 | await testParseAllReadables(JSON.stringify(string), { 40 | prohibitPartialNumbers: true, 41 | }) 42 | await testParseAllReadables(JSON.stringify(string), { 43 | prohibitPartialStrings: true, 44 | }) 45 | await testParseAllReadables(JSON.stringify(string), { 46 | prohibitPartialStrings: true, 47 | prohibitPartialNumbers: true, 48 | }) 49 | } 50 | 51 | const testParseAllReadables = async ( 52 | fullString: string, 53 | options: IncompleteJsonOptions, 54 | ) => { 55 | const fullValue = JSON.parse(fullString) 56 | 57 | const chunks = [""] 58 | for (const char of fullString) { 59 | if (Math.random() < 0.1) { 60 | chunks.push("") 61 | } 62 | chunks[chunks.length - 1] += char 63 | } 64 | 65 | const stream = new TransformStream() 66 | 67 | void (async () => { 68 | const writer = stream.writable.getWriter() 69 | for (const chunk of chunks) { 70 | await writer.write(chunk) 71 | } 72 | await writer.close() 73 | })() 74 | 75 | const readable = IncompleteJson.fromReadable(stream.readable, options) 76 | 77 | let lastPartial 78 | for await (const partialValue of readable.values()) { 79 | if (lastPartial) { 80 | expect(partialValue).toBeDefined 81 | } 82 | 83 | if (partialValue) { 84 | verifyIsValidPartial(partialValue, fullValue, options) 85 | if (lastPartial) { 86 | expect(JSON.stringify(lastPartial).length).toBeLessThanOrEqual( 87 | JSON.stringify(partialValue).length, 88 | ) 89 | } 90 | lastPartial = partialValue 91 | } 92 | } 93 | 94 | // verify final data is equal to desired 95 | expect(JSON.stringify(lastPartial)).toEqual(JSON.stringify(fullValue)) 96 | } 97 | -------------------------------------------------------------------------------- /test/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["ESNext"], 4 | "module": "esnext", 5 | "target": "esnext", 6 | "moduleResolution": "bundler", 7 | "moduleDetection": "force", 8 | "allowImportingTsExtensions": true, 9 | "noEmit": true, 10 | "strict": true, 11 | "downlevelIteration": true, 12 | "skipLibCheck": true, 13 | "jsx": "react-jsx", 14 | "allowSyntheticDefaultImports": true, 15 | "forceConsistentCasingInFileNames": true, 16 | "allowJs": true, 17 | "types": [ 18 | "bun-types" // add Bun global 19 | ], 20 | }, 21 | "files": ["../node_modules/jest-expect-message/types/index.d.ts"], 22 | "include": ["*.test.ts"] 23 | } -------------------------------------------------------------------------------- /test/utils.test.ts: -------------------------------------------------------------------------------- 1 | import { ItemDoneStreaming, IncompleteJsonOptions } from "../index.ts" 2 | import { expect } from "bun:test" 3 | 4 | // https://developer.mozilla.org/en-US/docs/Learn/JavaScript/Objects/JSON 5 | export const e1 = { 6 | squadName: "Super hero squad", 7 | homeTown: "Metro City", 8 | formed: 2016, 9 | secretBase: "Super tower", 10 | active: true, 11 | members: [ 12 | { 13 | name: "Molecule Man", 14 | age: 29, 15 | secretIdentity: "Dan Jukes", 16 | powers: ["Radiation resistance", "Turning tiny", "Radiation blast"], 17 | }, 18 | { 19 | name: "Madame Uppercut", 20 | age: 39, 21 | secretIdentity: "Jane Wilson", 22 | powers: [ 23 | "Million tonne punch", 24 | "Damage resistance", 25 | "Superhuman reflexes", 26 | ], 27 | isAdmin: true, 28 | }, 29 | { 30 | name: "Eternal Flame", 31 | age: 1000000, 32 | secretIdentity: "Unknown", 33 | powers: [ 34 | "Immortality", 35 | "Heat Immunity", 36 | "Inferno", 37 | "Teleportation", 38 | "Interdimensional travel", 39 | ], 40 | }, 41 | ], 42 | } 43 | 44 | export const e2 = [ 45 | { 46 | name: "Molecule Man", 47 | age: 29, 48 | secretIdentity: "Dan Jukes", 49 | powers: ["Radiation resistance", "Turning tiny", "Radiation blast"], 50 | }, 51 | { 52 | name: "Madame Uppercut", 53 | age: 39, 54 | secretIdentity: "Jane Wilson", 55 | powers: ["Million tonne punch", "Damage resistance", "Superhuman reflexes"], 56 | }, 57 | ] 58 | 59 | export const verifyIsValidPartial = ( 60 | partialValue: any, 61 | fullValue: any, 62 | options: IncompleteJsonOptions, 63 | ) => { 64 | const partialValuePaths = getPathsForJSONValue(partialValue) 65 | 66 | for (const partialValuePath of partialValuePaths) { 67 | const partialValueAtPath = getValueAtPath(partialValue, partialValuePath) 68 | const fullValueAtPath = getValueAtPath(fullValue, partialValuePath) 69 | 70 | try { 71 | expect(fullValueAtPath).toBeDefined() 72 | expect(partialValueAtPath).toBeDefined() 73 | 74 | if (typeof fullValueAtPath === "string") { 75 | if (options.prohibitPartialStrings) { 76 | expect(partialValueAtPath).toEqual(fullValueAtPath) 77 | } else { 78 | expect(partialValueAtPath).toBeString() 79 | expect(fullValueAtPath).toStartWith(partialValueAtPath as string) 80 | } 81 | } else if (typeof fullValueAtPath === "number") { 82 | if (options.prohibitPartialNumbers) { 83 | expect(partialValueAtPath).toEqual(fullValueAtPath) 84 | } else { 85 | expect(partialValueAtPath).toBeNumber() 86 | expect(String(fullValueAtPath)).toStartWith( 87 | String(partialValueAtPath), 88 | ) 89 | } 90 | } else if (typeof fullValueAtPath === "boolean") { 91 | expect(partialValueAtPath).toEqual(fullValueAtPath) 92 | } else if (fullValueAtPath === null) { 93 | expect(partialValueAtPath).toBeNull() 94 | } else if (Array.isArray(fullValueAtPath)) { 95 | expect(partialValueAtPath).toBeArray() 96 | if ((partialValueAtPath as any)[ItemDoneStreaming]) { 97 | expect(JSON.stringify(partialValueAtPath)).toEqual( 98 | JSON.stringify(fullValueAtPath), 99 | ) 100 | } 101 | } else { 102 | expect(typeof partialValueAtPath).toBe("object") 103 | expect(typeof fullValueAtPath).toBe("object") 104 | if ((partialValueAtPath as any)[ItemDoneStreaming]) { 105 | expect(JSON.stringify(partialValueAtPath)).toEqual( 106 | JSON.stringify(fullValueAtPath), 107 | ) 108 | } 109 | } 110 | } catch (e) { 111 | console.log({ 112 | error: e, 113 | partialValuePath, 114 | partialValueAtPath, 115 | fullValueAtPath, 116 | partialValue, 117 | fullValue, 118 | options, 119 | }) 120 | throw e 121 | } 122 | } 123 | } 124 | 125 | export type JSONValue = 126 | | true 127 | | false 128 | | null 129 | | string 130 | | number 131 | | { [key: string]: JSONValue } 132 | | JSONValue[] 133 | 134 | const getPathsForJSONValue = (value: JSONValue): string[][] => { 135 | if (value === undefined) 136 | throw Error("attempting to get paths for undefined value") 137 | 138 | const paths: string[][] = [] 139 | paths.push([]) 140 | 141 | if (value && typeof value === "object") { 142 | Object.entries(value).forEach(([k, v]) => { 143 | for (const path of getPathsForJSONValue(v)) { 144 | paths.push([k, ...path]) 145 | } 146 | }) 147 | } 148 | return paths 149 | } 150 | 151 | const getValueAtPath = ( 152 | value: JSONValue, 153 | path: string[], 154 | ): JSONValue | undefined => { 155 | if (path.length === 0 || value === undefined) return value 156 | 157 | const [head, ...tail] = path 158 | return getValueAtPath((value as any)?.[head], tail) 159 | } 160 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["ESNext", "DOM"], 4 | "module": "esnext", 5 | "target": "esnext", 6 | "moduleResolution": "bundler", 7 | "moduleDetection": "force", 8 | // "allowImportingTsExtensions": true, 9 | // "emitDeclarationOnly": true, 10 | "declaration": true, 11 | "outDir": "dist", 12 | "strict": true, 13 | "downlevelIteration": true, 14 | "skipLibCheck": true, 15 | "jsx": "react-jsx", 16 | "allowSyntheticDefaultImports": true, 17 | "forceConsistentCasingInFileNames": true, 18 | }, 19 | "exclude": ["test", "rand", "dist"] 20 | } 21 | --------------------------------------------------------------------------------