├── .github └── workflows │ ├── branches.yml │ └── tags.yml ├── .gitignore ├── LICENSE.md ├── README.md ├── index.ts ├── json-fetch.ts ├── json-parse-stream.ts ├── json-parser.js ├── json-path.ts ├── json-stringify-stream.ts ├── json-stringify.ts ├── scripts └── build_npm.ts ├── task-promise.ts └── test ├── index.test.ts ├── json-fetch.test.ts ├── json-parse-stream.test.ts ├── json-path.test.ts ├── json-stringify.test.ts ├── json1.json └── playground.test.ts /.github/workflows/branches.yml: -------------------------------------------------------------------------------- 1 | # This workflow will run tests using node and then publish a package to GitHub Packages when a release is created 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/publishing-nodejs-packages 3 | 4 | name: Node.js Package 5 | 6 | on: 7 | push: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v2 16 | - uses: denoland/setup-deno@v1 17 | with: 18 | deno-version: v1.x 19 | - run: deno test -A ./test 20 | -------------------------------------------------------------------------------- /.github/workflows/tags.yml: -------------------------------------------------------------------------------- 1 | # This workflow will run tests using node and then publish a package to GitHub Packages when a release is created 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/publishing-nodejs-packages 3 | 4 | name: Node.js Package 5 | 6 | on: 7 | push: 8 | tags: 9 | - v* 10 | 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v2 16 | - uses: denoland/setup-deno@v1 17 | with: 18 | deno-version: v1.x 19 | - run: deno test -A ./test 20 | 21 | publish-npm: 22 | needs: test 23 | runs-on: ubuntu-latest 24 | steps: 25 | - uses: actions/checkout@v2 26 | - uses: denoland/setup-deno@v1 27 | with: 28 | deno-version: v1.x 29 | - uses: actions/setup-node@v1 30 | with: 31 | node-version: 14 32 | registry-url: https://registry.npmjs.org/ 33 | - uses: pnpm/action-setup@v2 34 | with: 35 | version: 6 36 | run_install: false 37 | - run: deno run -A ./scripts/build_npm.ts 38 | - run: cd ./npm && npm publish 39 | env: 40 | NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /npm 2 | _* -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # The MIT License 2 | 3 | Copyright (c) 2022 Florian Klampfer (https://qwtel.com/) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JSON Stream 2 | 3 | Utilities for working with streaming JSON in Worker Runtimes such as Cloudflare Workers, Deno Deploy and Service Workers. 4 | 5 | *** 6 | 7 | __Work in Progress__: See TODOs & Deprecations 8 | 9 | *** 10 | 11 | ## Base Case 12 | The most basic use case is turning a stream of objects into stream of strings that can be sent over the wire. 13 | On the other end, it can be turned back into a stream of JSON objects. 14 | For this `JSONStringifyStream` and `JSONParseStream` are all that is required. 15 | They work practically the same as `TextEncoderStream` and `TextDecoderStream`: 16 | 17 | ```js 18 | const items = [ 19 | { a: 1 }, 20 | { b: 2}, 21 | { c: 3 }, 22 | 'foo', 23 | { a: { nested: { object: true }} } 24 | ]; 25 | const stream = toReadableStream(items) 26 | .pipeThrough(new JSONStringifyStream()) 27 | .pipeThrough(new TextEncoderStream()) 28 | 29 | // Usage e.g.: 30 | await fetch('/endpoint.json', { 31 | body: stream, 32 | method: 'POST', 33 | headers: [['content-type', 'application/json']] 34 | }) 35 | 36 | // On the server side: 37 | const collected = []; 38 | await stream 39 | .pipeThrough(new JSONParseStream()) 40 | .pipeTo(new WritableStream({ write(obj) { collected.push(obj) }})) 41 | 42 | assertEquals(items, collected) 43 | ``` 44 | 45 | Note that standard JSON is used as the transport format. Unlike ND-JSON, 46 | neither side needs to opt-in using the streaming parser/stringifier to accept data. 47 | For example this is just as valid: 48 | 49 | ```js 50 | const collected = await new Response(stream).json() 51 | ``` 52 | 53 | ~~If on the other hand ND-JSON is sufficient for your use case, this module also provides `NDJSONStringifyStream` and `NDJSONParseStream` that work the same way as shown above, but lack the following features.~~ (TODO: make separate module?) 54 | 55 | ## Using JSON Path to locate nested data 56 | __JSON Stream__ also supports more complex use cases. Assume JSON of the following structure: 57 | 58 | ```jsonc 59 | // filename: "nested.json" 60 | { 61 | "type": "foo", 62 | "items": [ 63 | { "a": 1 }, 64 | { "b": 2 }, 65 | { "c": 3 }, 66 | // ... 67 | { "zzz": 999 }, 68 | ] 69 | } 70 | ``` 71 | 72 | Here, the example code from above wouldn't work (or at least not as you would expect), 73 | because by default `JSONParseStream` emits the objects that are the immediate children of the root object. 74 | However, the constructor accepts a JSONPath-like string to locate the desired data to parse: 75 | 76 | ```js 77 | const collected = []; 78 | await (await fetch('/nested.json')).body 79 | .pipeThrough(new JSONParseStream('$.items.*')) // <-- new 80 | .pipeTo(new WritableStream({ write(obj) { collected.push(obj) }})) 81 | ``` 82 | 83 | It's important to add the `.*` at the end, otherwise the entire items array will arrive in a singe call once it is fully parsed. 84 | 85 | `JSONParseStream` only supports a subset of JSONPath, specifically eval (`@`) expressions and negative slices are omitted. 86 | Below is a table showing some examples: 87 | 88 | | JSONPath | Description | 89 | |:--------------------------|:------------------------------------------------------------| 90 | | `$.*` | All direct children of the root. Default. | 91 | | `$.store.book[*].author` | The authors of all books in the store | 92 | | `$..author` | All authors | 93 | | `$.store.*` | All things in store, which are some books and a red bicycle | 94 | | `$.store..price` | The price of everything in the store | 95 | | `$..book[2]` | The third book | 96 | | `$..book[0,1]` | The first two books via subscript union | 97 | | `$..book[:2]` | The first two books via subscript array slice | 98 | | `$..*` | All members of JSON structure | 99 | 100 | ## Streaming Complex Data 101 | You might also be interested in how to stream complex data such as the one above from memory. 102 | In that case `JSONStringifyStream` isn't too helpful, as it only supports JSON arrays (i.e. the root element is an array `[]`). 103 | 104 | For that case __JSON Stream__ provides the `jsonStringifyStream` method (TODO: better name to indicate that it is a readableStream? Change to ReadableStream subclass? Export `JSONStream` object with `stringify` method?) which accepts any JSON-ifiable data as argument. It is mostly compatible with `JSON.stringify` (TODO: replacer & spaces), but with the important exception that it "inlines" any `Promise`, `ReadableStream` and `AsyncIterable` it encounters. Again, an example: 105 | 106 | ```js 107 | const stream = jsonStringifyStream({ 108 | type: Promise.resolve('foo'), 109 | items: (async function* () { 110 | yield { a: 1 } 111 | yield { b: 2 } 112 | yield { c: 3 } 113 | // Can also have nested async values: 114 | yield Promise.resolve({ zzz: 999 }) 115 | })(), 116 | }) 117 | 118 | new Response(stream.pipeThrough(new TextEncoderStream()), { 119 | headers: [['content-type', 'application/json']] 120 | }) 121 | ``` 122 | 123 | Inspecting this on the network would show the following (where every newline is a chunk): 124 | ```json 125 | { 126 | "type": 127 | "foo" 128 | , 129 | "items": 130 | [ 131 | { 132 | "a": 133 | 1 134 | } 135 | , 136 | { 137 | "b": 138 | 2 139 | } 140 | , 141 | { 142 | "c": 143 | 3 144 | } 145 | , 146 | { 147 | "zzz": 148 | 999 149 | } 150 | ] 151 | } 152 | ``` 153 | 154 | ## Retrieving Complex Structures 155 | By providing a JSON Path to `JSONParseStream` we can stream the values of a single, nested array. 156 | For scenarios where the JSON structure is more complex, there is the `JSONParseNexus` (TODO: better name) class. 157 | It provides promise and and stream-based methods that accept JSON paths to retrieve one or multiple values respectively. 158 | While it is much more powerful and can restore arbitrary complex structures, it is also more difficult to use. 159 | 160 | It's best to explain by example. Assuming the data structure from above, we have: 161 | 162 | ```js 163 | const parser = new JSONParseNexus(); 164 | const data = { 165 | type: parser.promise('$.type'), 166 | items: parser.stream('$.items.*'), 167 | } 168 | (await fetch('/nested.json').body) 169 | .pipeThrough(parser) // <-- new 170 | 171 | assertEquals(await data.type, 'foo') 172 | 173 | // We can collect the values as before: 174 | const collected = []; 175 | await data.items 176 | .pipeTo(new WritableStream({ write(obj) { collected.push(obj) }})) 177 | ``` 178 | 179 | While this works just fine, it becomes more complicated when there are multiple streams and values involved. 180 | 181 | ### Managing Internal Queues 182 | It's important to understand that `JSONParseNexus` provides mostly pull-based APIs. 183 | In the cause of `.stream()` and `.iterable()` no work is being done until a consumer requests a value by calling `.read()` or `.next()` respectively. 184 | However, once a value is requested, `JSONParseNexus` will parse values until the requested JSON path is found. 185 | Along the way it will fill up queues for any other requested JSON paths it encounters. 186 | This means that memory usage can grow arbitrarily large unless the data is processed in the order it was stringified: 187 | Take for example the following structure: 188 | 189 | ```js 190 | const parser = new JSONParseNexus(); 191 | 192 | jsonStringifyStream({ 193 | xs: new Array(10_000).fill({ x: 'x' }), 194 | ys: new Array(10_000).fill({ y: 'y' }), 195 | }).pipeThrough(parser) 196 | 197 | for await (const y of parser.iterable('$.ys.*')) console.log(y) 198 | for await (const x of parser.iterable('$.xs.*')) console.log(x) 199 | ``` 200 | 201 | In this examples Ys are being processed before Xs, but were stringified in the opposite order. 202 | This means the internal queue of Xs grows to 10.000 before it is being processed by the second loop. 203 | This can be avoided by changing the order to match the stringification order. 204 | 205 | ### Single Values and Lazy Promises 206 | Special attention has to be given single values, as Promises in JS are eager by default and have no concept of "pulling" data. 207 | `JSONParseNexus` introduces a lazy promise type that has a different behavior. 208 | As with async iterables and streams provided by `.iterable` and `.stream`, it does not pull values form the underlying readable until requested. This happens when `await`ing the promise, i.e. is calling the `.then` instance method, otherwise it stays idle. 209 | 210 | ```js 211 | const parser = new JSONParseNexus(); 212 | 213 | jsonStringifyStream({ 214 | type: 'items', 215 | items: new Array(10_000).fill({ x: 'x' }), 216 | trailer: 'trail', 217 | }).pipeThrough(parser) 218 | 219 | const data = { 220 | type: await parser.promise('$.type') // ok 221 | items: parser.iterable('$.items.*') 222 | trailer: parser.promise('$.trailer') // do not await! 223 | } 224 | 225 | console.log(data.type) //=> 'items' 226 | 227 | // Now async iteration is in control of parser: 228 | for await (const x of data.items) { 229 | console.log(x) 230 | } 231 | // Now we can await the trailer: 232 | console.log(await data.trailer) 233 | ``` 234 | 235 | In the above example, without lazy promises `ctrl.promise('$.trailer')` would immediately parse the entire JSON structure, which involves filling a queue of 10.000 elements. 236 | 237 | In order to transform value without triggering executions, 238 | the class provides a `.map` function that works similar to JS arrays: 239 | 240 | ```js 241 | const trailer = ctrl.promise('$.trailer').map(x => x.toUpperCase()) 242 | ``` 243 | 244 | ## Limitations 245 | **JSON Stream** largely consists of old Node libraries that have been modified to work in Worker Runtimes and the browser. 246 | Currently they are not "integrated", for example specifying a specific JSON Path does not limit the amount of parsing the parser does. 247 | 248 | The stringification implementation, which is original, relies heavily on async generators, which are "slow" but they made the implementation easy and quick to implement. 249 | 250 | **JSON Stream** heavily relies on [`TransformStream`](https://developer.mozilla.org/en-US/docs/Web/API/TransformStream), which has only recently shipped in Chrome & Safari and is still behind a flag in Firefox. However, the latest version of Deno and Cloudflare Workers support it (might require compatibility flags in CF Workers). 251 | 252 | 253 | ## Appendix 254 | ### To ReadableStream Function 255 | An example above uses a `toReadableStream` function, which can be implemented as follows: 256 | ```ts 257 | function toReadableStream(iter: Iterable) { 258 | const xs = [...iter]; 259 | let x: T | undefined; 260 | return new ReadableStream({ 261 | pull(ctrl) { 262 | if (x = xs.shift()) ctrl.enqueue(x); else ctrl.close(); 263 | }, 264 | }); 265 | } 266 | ``` 267 | 268 | -------- 269 | 270 |

271 |

This module is part of the Worker Tools collection
⁕ 272 | 273 | [Worker Tools](https://workers.tools) are a collection of TypeScript libraries for writing web servers in [Worker Runtimes](https://workers.js.org) such as Cloudflare Workers, Deno Deploy and Service Workers in the browser. 274 | 275 | If you liked this module, you might also like: 276 | 277 | - 🧭 [__Worker Router__][router] --- Complete routing solution that works across CF Workers, Deno and Service Workers 278 | - 🔋 [__Worker Middleware__][middleware] --- A suite of standalone HTTP server-side middleware with TypeScript support 279 | - 📄 [__Worker HTML__][html] --- HTML templating and streaming response library 280 | - 📦 [__Storage Area__][kv-storage] --- Key-value store abstraction across [Cloudflare KV][cloudflare-kv-storage], [Deno][deno-kv-storage] and browsers. 281 | - 🆗 [__Response Creators__][response-creators] --- Factory functions for responses with pre-filled status and status text 282 | - 🎏 [__Stream Response__][stream-response] --- Use async generators to build streaming responses for SSE, etc... 283 | - 🥏 [__JSON Fetch__][json-fetch] --- Drop-in replacements for Fetch API classes with first class support for JSON. 284 | - 🦑 [__JSON Stream__][json-stream] --- Streaming JSON parser/stingifier with first class support for web streams. 285 | 286 | Worker Tools also includes a number of polyfills that help bridge the gap between Worker Runtimes: 287 | - ✏️ [__HTML Rewriter__][html-rewriter] --- Cloudflare's HTML Rewriter for use in Deno, browsers, etc... 288 | - 📍 [__Location Polyfill__][location-polyfill] --- A `Location` polyfill for Cloudflare Workers. 289 | - 🦕 [__Deno Fetch Event Adapter__][deno-fetch-event-adapter] --- Dispatches global `fetch` events using Deno’s native HTTP server. 290 | 291 | [router]: https://workers.tools/router 292 | [middleware]: https://workers.tools/middleware 293 | [html]: https://workers.tools/html 294 | [kv-storage]: https://workers.tools/kv-storage 295 | [cloudflare-kv-storage]: https://workers.tools/cloudflare-kv-storage 296 | [deno-kv-storage]: https://workers.tools/deno-kv-storage 297 | [kv-storage-polyfill]: https://workers.tools/kv-storage-polyfill 298 | [response-creators]: https://workers.tools/response-creators 299 | [stream-response]: https://workers.tools/stream-response 300 | [json-fetch]: https://workers.tools/json-fetch 301 | [json-stream]: https://workers.tools/json-stream 302 | [request-cookie-store]: https://workers.tools/request-cookie-store 303 | [extendable-promise]: https://workers.tools/extendable-promise 304 | [html-rewriter]: https://workers.tools/html-rewriter 305 | [location-polyfill]: https://workers.tools/location-polyfill 306 | [deno-fetch-event-adapter]: https://workers.tools/deno-fetch-event-adapter 307 | 308 | Fore more visit [workers.tools](https://workers.tools). 309 | -------------------------------------------------------------------------------- /index.ts: -------------------------------------------------------------------------------- 1 | export * from './json-stringify.ts' 2 | export * from './json-parse-stream.ts' 3 | export * from './json-stringify-stream.ts' 4 | export * from './json-fetch.ts' -------------------------------------------------------------------------------- /json-fetch.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-explicit-any 2 | import { StreamResponse, StreamRequest } from "https://ghuc.cc/worker-tools/stream-response/index.ts"; 3 | import { asyncIterToStream } from 'https://ghuc.cc/qwtel/whatwg-stream-to-async-iter/index.ts'; 4 | import { JSONStringifyReadable, isAsyncIterable } from './json-stringify.ts'; 5 | 6 | export type JSONStreamBodyInit = ReadableStream | AsyncIterable | any; 7 | export type JSONStreamRequestInit = Omit & { body?: JSONStreamBodyInit } 8 | 9 | const toBody = (x: any) => x instanceof ReadableStream 10 | ? x 11 | : isAsyncIterable(x) 12 | ? asyncIterToStream(x) 13 | : new JSONStringifyReadable(x) 14 | 15 | export class JSONStreamRequest extends StreamRequest { 16 | static contentType = 'application/json;charset=UTF-8'; 17 | static accept = 'application/json, text/plain, */*'; 18 | 19 | constructor( 20 | input: RequestInfo | URL, 21 | init?: JSONStreamRequestInit, 22 | // replacer?: Parameters[1], 23 | // space?: Parameters[2], 24 | ) { 25 | const { headers: _headers, body: _body, ...rest } = init || {}; 26 | 27 | const body = toBody(_body); 28 | 29 | const headers = new Headers(_headers); 30 | if (!headers.has('Content-Type') && _body != null) 31 | headers.set('Content-Type', JSONStreamRequest.contentType); 32 | 33 | if (!headers.has('Accept')) 34 | headers.set('Accept', JSONStreamRequest.accept); 35 | 36 | super(input instanceof URL ? input.href : input, { headers, body, ...rest }); 37 | } 38 | } 39 | 40 | export class JSONStreamResponse extends StreamResponse { 41 | static contentType = 'application/json;charset=UTF-8'; 42 | 43 | constructor( 44 | body?: JSONStreamBodyInit | null, 45 | init?: ResponseInit, 46 | // replacer?: Parameters[1], 47 | // space?: Parameters[2], 48 | ) { 49 | const { headers: _headers, ...rest } = init || {}; 50 | 51 | const _body = toBody(body) 52 | 53 | const headers = new Headers(_headers); 54 | 55 | if (!headers.has('Content-Type') && body != null) 56 | headers.set('Content-Type', JSONStreamResponse.contentType); 57 | 58 | super(_body, { headers, ...rest }); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /json-parse-stream.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-explicit-any no-cond-assign ban-unused-ignore no-unused-vars 2 | import { streamToAsyncIter } from 'https://ghuc.cc/qwtel/whatwg-stream-to-async-iter/index.ts' 3 | import { JSONParser } from './json-parser.js'; 4 | import { TaskPromise } from './task-promise.ts'; 5 | import { normalize, match } from './json-path.ts' 6 | 7 | // FIXME: avoid string concatenation/joining 8 | const mkPath = (parser: any) => { 9 | const path = [...parser.stack.map((_: any) => _.key), parser.key]; // TODO: modify parser to provide key efficiently 10 | path[0] = path[0] || '$'; 11 | return normalize(path.join('.')); // FIXME: avoid string concatenation/joining 12 | } 13 | 14 | export class JSONParseStream extends TransformStream { 15 | #jsonPath; 16 | 17 | constructor(jsonPath = '$.*') { 18 | let parser!: JSONParser; 19 | const expr = normalize(jsonPath) 20 | super({ 21 | start: (controller) => { 22 | parser = new JSONParser(); 23 | parser.onValue = (value: T) => { 24 | const path = mkPath(parser) 25 | 26 | if (match(expr, path)) { 27 | controller.enqueue(value as any); 28 | } else if (expr.startsWith(path + ';')) { 29 | controller.terminate() 30 | } 31 | }; 32 | }, 33 | transform: (chunk) => { 34 | parser.write(chunk); 35 | }, 36 | }); 37 | this.#jsonPath = expr; 38 | } 39 | 40 | get path() { return this.#jsonPath } 41 | } 42 | 43 | const remove = (m: Map, k: K) => { const v = m.get(k); m.delete(k); return v; } 44 | 45 | 46 | /** @deprecated Rename!!! */ 47 | export class JSONParseNexus extends TransformStream { 48 | #queues = new Map>(); 49 | #reader: ReadableStreamDefaultReader<[string, T]> 50 | 51 | constructor() { 52 | let parser: JSONParser; 53 | super({ 54 | start: (controller) => { 55 | parser = new JSONParser(); 56 | parser.onValue = (value: T) => { 57 | const path = mkPath(parser) 58 | 59 | for (const expr of this.#queues.keys()) { 60 | if (match(expr, path)) { 61 | this.#queues.get(expr)!.enqueue(value) 62 | } // no else if => can both be true 63 | if (expr.startsWith(path + ';')) { 64 | remove(this.#queues, expr)!.close() 65 | } 66 | } 67 | 68 | controller.enqueue([path, value]); 69 | }; 70 | }, 71 | transform(buffer) { 72 | // console.log('transform', buffer, controller.desiredSize) 73 | parser.write(buffer) 74 | }, 75 | }); 76 | this.#reader = this.readable.getReader(); 77 | } 78 | 79 | promise(jsonPath: string): TaskPromise { 80 | const reader = this.stream(jsonPath).getReader(); 81 | return TaskPromise.from(async () => { 82 | const x = await reader.read(); 83 | return x.done ? undefined : x.value; 84 | }) 85 | } 86 | 87 | stream(jsonPath: string): ReadableStream { 88 | const path = normalize(jsonPath); 89 | return new ReadableStream({ 90 | start: (queue) => { 91 | this.#queues.set(path, queue) 92 | }, 93 | pull: async () => { 94 | while (true) { 95 | const { done, value } = await this.#reader.read(); 96 | // FIXME: avoid duplicate match 97 | if (done || match(value[0], path)) break; 98 | } 99 | }, 100 | cancel: (err) => { 101 | // If one of the child streams errors, error the whole pipeline. 102 | // TODO: Or should it? 103 | this.#reader.cancel(err) 104 | }, 105 | }, { highWaterMark: 0 }) // does not pull on its own 106 | } 107 | 108 | iterable(jsonPath: string): AsyncIterableIterator { 109 | return streamToAsyncIter(this.stream(jsonPath)) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /json-parser.js: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-unused-vars no-case-declarations 2 | 3 | // Modernized/de-nodified version of creationix/jsonparse 4 | // Copyright (c) 2012 Tim Caswell 5 | // Licensed under the MIT (licenses/MIT.md) license. 6 | 7 | // TODO: TypeScript conversion? 8 | // TODO: Integrate with other modules for better performance 9 | 10 | // Named constants with unique integer values 11 | const C = {}; 12 | // Tokens 13 | const LEFT_BRACE = C.LEFT_BRACE = 0x1; 14 | const RIGHT_BRACE = C.RIGHT_BRACE = 0x2; 15 | const LEFT_BRACKET = C.LEFT_BRACKET = 0x3; 16 | const RIGHT_BRACKET = C.RIGHT_BRACKET = 0x4; 17 | const COLON = C.COLON = 0x5; 18 | const COMMA = C.COMMA = 0x6; 19 | const TRUE = C.TRUE = 0x7; 20 | const FALSE = C.FALSE = 0x8; 21 | const NULL = C.NULL = 0x9; 22 | const STRING = C.STRING = 0xa; 23 | const NUMBER = C.NUMBER = 0xb; 24 | // Tokenizer States 25 | const START = C.START = 0x11; 26 | const STOP = C.STOP = 0x12; 27 | const TRUE1 = C.TRUE1 = 0x21; 28 | const TRUE2 = C.TRUE2 = 0x22; 29 | const TRUE3 = C.TRUE3 = 0x23; 30 | const FALSE1 = C.FALSE1 = 0x31; 31 | const FALSE2 = C.FALSE2 = 0x32; 32 | const FALSE3 = C.FALSE3 = 0x33; 33 | const FALSE4 = C.FALSE4 = 0x34; 34 | const NULL1 = C.NULL1 = 0x41; 35 | const NULL2 = C.NULL2 = 0x42; 36 | const NULL3 = C.NULL3 = 0x43; 37 | const NUMBER1 = C.NUMBER1 = 0x51; 38 | const NUMBER3 = C.NUMBER3 = 0x53; 39 | const STRING1 = C.STRING1 = 0x61; 40 | const STRING2 = C.STRING2 = 0x62; 41 | const STRING3 = C.STRING3 = 0x63; 42 | const STRING4 = C.STRING4 = 0x64; 43 | const STRING5 = C.STRING5 = 0x65; 44 | const STRING6 = C.STRING6 = 0x66; 45 | // Parser States 46 | const VALUE = C.VALUE = 0x71; 47 | const KEY = C.KEY = 0x72; 48 | // Parser Modes 49 | const OBJECT = C.OBJECT = 0x81; 50 | const ARRAY = C.ARRAY = 0x82; 51 | // Character constants 52 | const BACK_SLASH = "\\".charCodeAt(0); 53 | const FORWARD_SLASH = "\/".charCodeAt(0); 54 | const BACKSPACE = "\b".charCodeAt(0); 55 | const FORM_FEED = "\f".charCodeAt(0); 56 | const NEWLINE = "\n".charCodeAt(0); 57 | const CARRIAGE_RETURN = "\r".charCodeAt(0); 58 | const TAB = "\t".charCodeAt(0); 59 | 60 | const STRING_BUFFER_SIZE = 64 * 1024; 61 | 62 | function alloc(size) { 63 | return new Uint8Array(size); 64 | } 65 | 66 | class Parser { 67 | constructor() { 68 | this.tState = START; 69 | this.value = undefined; 70 | 71 | this.string = undefined; // string data 72 | this.stringBuffer = alloc(STRING_BUFFER_SIZE); 73 | this.stringBufferOffset = 0; 74 | this.unicode = undefined; // unicode escapes 75 | this.highSurrogate = undefined; 76 | 77 | this.key = undefined; 78 | this.mode = undefined; 79 | this.stack = []; 80 | this.state = VALUE; 81 | this.bytes_remaining = 0; // number of bytes remaining in multi byte utf8 char to read after split boundary 82 | this.bytes_in_sequence = 0; // bytes in multi byte utf8 char to read 83 | this.temp_buffs = { "2": alloc(2), "3": alloc(3), "4": alloc(4) }; // for rebuilding chars split before boundary is reached 84 | 85 | this.encoder = new TextEncoder(); 86 | this.decoder = new TextDecoder(); 87 | 88 | // Stream offset 89 | this.offset = -1; 90 | } 91 | 92 | // Slow code to string converter (only used when throwing syntax errors) 93 | static toknam(code) { 94 | const keys = Object.keys(C); 95 | for (let i = 0, l = keys.length; i < l; i++) { 96 | const key = keys[i]; 97 | if (C[key] === code) { return key; } 98 | } 99 | return code && ("0x" + code.toString(16)); 100 | } 101 | 102 | encode(string) { return this.encoder.encode(string) } 103 | decode(buffer) { return this.decoder.decode(buffer) } 104 | 105 | onError(err) { throw err; } 106 | charError(buffer, i) { 107 | this.tState = STOP; 108 | this.onError(new Error("Unexpected " + JSON.stringify(String.fromCharCode(buffer[i])) + " at position " + i + " in state " + Parser.toknam(this.tState))); 109 | } 110 | appendStringChar(char) { 111 | if (this.stringBufferOffset >= STRING_BUFFER_SIZE) { 112 | this.string += this.decode(this.stringBuffer); 113 | this.stringBufferOffset = 0; 114 | } 115 | 116 | this.stringBuffer[this.stringBufferOffset++] = char; 117 | } 118 | appendStringBuf(buf, start, end) { 119 | let size = buf.length; 120 | if (typeof start === 'number') { 121 | if (typeof end === 'number') { 122 | if (end < 0) { 123 | // adding a negative end decreeses the size 124 | size = buf.length - start + end; 125 | } else { 126 | size = end - start; 127 | } 128 | } else { 129 | size = buf.length - start; 130 | } 131 | } 132 | 133 | if (size < 0) { 134 | size = 0; 135 | } 136 | 137 | if (this.stringBufferOffset + size > STRING_BUFFER_SIZE) { 138 | this.string += this.decode(this.stringBuffer.subarray(0, this.stringBufferOffset)); 139 | this.stringBufferOffset = 0; 140 | } 141 | 142 | this.stringBuffer.set(buf.subarray(start, end), this.stringBufferOffset); 143 | this.stringBufferOffset += size; 144 | } 145 | write(buffer) { 146 | if (typeof buffer === "string") buffer = this.encode(buffer); 147 | let n; 148 | for (let i = 0, l = buffer.length; i < l; i++) { 149 | if (this.tState === START) { 150 | n = buffer[i]; 151 | this.offset++; 152 | if (n === 0x7b) { 153 | this.onToken(LEFT_BRACE, "{"); // { 154 | } else if (n === 0x7d) { 155 | this.onToken(RIGHT_BRACE, "}"); // } 156 | } else if (n === 0x5b) { 157 | this.onToken(LEFT_BRACKET, "["); // [ 158 | } else if (n === 0x5d) { 159 | this.onToken(RIGHT_BRACKET, "]"); // ] 160 | } else if (n === 0x3a) { 161 | this.onToken(COLON, ":"); // : 162 | } else if (n === 0x2c) { 163 | this.onToken(COMMA, ","); // , 164 | } else if (n === 0x74) { 165 | this.tState = TRUE1; // t 166 | } else if (n === 0x66) { 167 | this.tState = FALSE1; // f 168 | } else if (n === 0x6e) { 169 | this.tState = NULL1; // n 170 | } else if (n === 0x22) { // " 171 | this.string = ""; 172 | this.stringBufferOffset = 0; 173 | this.tState = STRING1; 174 | } else if (n === 0x2d) { 175 | this.string = "-"; this.tState = NUMBER1; // - 176 | } else { 177 | if (n >= 0x30 && n < 0x40) { // 1-9 178 | this.string = String.fromCharCode(n); this.tState = NUMBER3; 179 | } else if (n === 0x20 || n === 0x09 || n === 0x0a || n === 0x0d) { 180 | // whitespace 181 | } else { 182 | return this.charError(buffer, i); 183 | } 184 | } 185 | } else if (this.tState === STRING1) { // After open quote 186 | n = buffer[i]; // get current byte from buffer 187 | // check for carry over of a multi byte char split between data chunks 188 | // & fill temp buffer it with start of this data chunk up to the boundary limit set in the last iteration 189 | if (this.bytes_remaining > 0) { 190 | let j; 191 | for (j = 0; j < this.bytes_remaining; j++) { 192 | this.temp_buffs[this.bytes_in_sequence][this.bytes_in_sequence - this.bytes_remaining + j] = buffer[j]; 193 | } 194 | 195 | this.appendStringBuf(this.temp_buffs[this.bytes_in_sequence]); 196 | this.bytes_in_sequence = this.bytes_remaining = 0; 197 | i = i + j - 1; 198 | } else if (this.bytes_remaining === 0 && n >= 128) { // else if no remainder bytes carried over, parse multi byte (>=128) chars one at a time 199 | if (n <= 193 || n > 244) { 200 | return this.onError(new Error("Invalid UTF-8 character at position " + i + " in state " + Parser.toknam(this.tState))); 201 | } 202 | if ((n >= 194) && (n <= 223)) this.bytes_in_sequence = 2; 203 | if ((n >= 224) && (n <= 239)) this.bytes_in_sequence = 3; 204 | if ((n >= 240) && (n <= 244)) this.bytes_in_sequence = 4; 205 | if ((this.bytes_in_sequence + i) > buffer.length) { // if bytes needed to complete char fall outside buffer length, we have a boundary split 206 | for (let k = 0; k <= (buffer.length - 1 - i); k++) { 207 | this.temp_buffs[this.bytes_in_sequence][k] = buffer[i + k]; // fill temp buffer of correct size with bytes available in this chunk 208 | } 209 | this.bytes_remaining = (i + this.bytes_in_sequence) - buffer.length; 210 | i = buffer.length - 1; 211 | } else { 212 | this.appendStringBuf(buffer, i, i + this.bytes_in_sequence); 213 | i = i + this.bytes_in_sequence - 1; 214 | } 215 | } else if (n === 0x22) { 216 | this.tState = START; 217 | this.string += this.decode(this.stringBuffer.subarray(0, this.stringBufferOffset)); 218 | this.stringBufferOffset = 0; 219 | this.onToken(STRING, this.string); 220 | this.offset += this.encode(this.string).length + 1; 221 | this.string = undefined; 222 | } 223 | else if (n === 0x5c) { 224 | this.tState = STRING2; 225 | } 226 | else if (n >= 0x20) { this.appendStringChar(n); } 227 | else { 228 | return this.charError(buffer, i); 229 | } 230 | } else if (this.tState === STRING2) { // After backslash 231 | n = buffer[i]; 232 | if (n === 0x22) { 233 | this.appendStringChar(n); this.tState = STRING1; 234 | } else if (n === 0x5c) { 235 | this.appendStringChar(BACK_SLASH); this.tState = STRING1; 236 | } else if (n === 0x2f) { 237 | this.appendStringChar(FORWARD_SLASH); this.tState = STRING1; 238 | } else if (n === 0x62) { 239 | this.appendStringChar(BACKSPACE); this.tState = STRING1; 240 | } else if (n === 0x66) { 241 | this.appendStringChar(FORM_FEED); this.tState = STRING1; 242 | } else if (n === 0x6e) { 243 | this.appendStringChar(NEWLINE); this.tState = STRING1; 244 | } else if (n === 0x72) { 245 | this.appendStringChar(CARRIAGE_RETURN); this.tState = STRING1; 246 | } else if (n === 0x74) { 247 | this.appendStringChar(TAB); this.tState = STRING1; 248 | } else if (n === 0x75) { 249 | this.unicode = ""; this.tState = STRING3; 250 | } else { 251 | return this.charError(buffer, i); 252 | } 253 | } else if (this.tState === STRING3 || this.tState === STRING4 || this.tState === STRING5 || this.tState === STRING6) { // unicode hex codes 254 | n = buffer[i]; 255 | // 0-9 A-F a-f 256 | if ((n >= 0x30 && n < 0x40) || (n > 0x40 && n <= 0x46) || (n > 0x60 && n <= 0x66)) { 257 | this.unicode += String.fromCharCode(n); 258 | if (this.tState++ === STRING6) { 259 | const intVal = parseInt(this.unicode, 16); 260 | this.unicode = undefined; 261 | if (this.highSurrogate !== undefined && intVal >= 0xDC00 && intVal < (0xDFFF + 1)) { //<56320,57343> - lowSurrogate 262 | this.appendStringBuf(this.encode(String.fromCharCode(this.highSurrogate, intVal))); 263 | this.highSurrogate = undefined; 264 | } else if (this.highSurrogate === undefined && intVal >= 0xD800 && intVal < (0xDBFF + 1)) { //<55296,56319> - highSurrogate 265 | this.highSurrogate = intVal; 266 | } else { 267 | if (this.highSurrogate !== undefined) { 268 | this.appendStringBuf(this.encode(String.fromCharCode(this.highSurrogate))); 269 | this.highSurrogate = undefined; 270 | } 271 | this.appendStringBuf(this.encode(String.fromCharCode(intVal))); 272 | } 273 | this.tState = STRING1; 274 | } 275 | } else { 276 | return this.charError(buffer, i); 277 | } 278 | } else if (this.tState === NUMBER1 || this.tState === NUMBER3) { 279 | n = buffer[i]; 280 | 281 | switch (n) { 282 | case 0x30: // 0 283 | case 0x31: // 1 284 | case 0x32: // 2 285 | case 0x33: // 3 286 | case 0x34: // 4 287 | case 0x35: // 5 288 | case 0x36: // 6 289 | case 0x37: // 7 290 | case 0x38: // 8 291 | case 0x39: // 9 292 | case 0x2e: // . 293 | case 0x65: // e 294 | case 0x45: // E 295 | case 0x2b: // + 296 | case 0x2d: // - 297 | this.string += String.fromCharCode(n); 298 | this.tState = NUMBER3; 299 | break; 300 | default: 301 | this.tState = START; 302 | const error = this.numberReviver(this.string); 303 | if (error) { 304 | return error; 305 | } 306 | 307 | this.offset += this.string.length - 1; 308 | this.string = undefined; 309 | i--; 310 | break; 311 | } 312 | } else if (this.tState === TRUE1) { // r 313 | if (buffer[i] === 0x72) { this.tState = TRUE2; } 314 | else { return this.charError(buffer, i); } 315 | } else if (this.tState === TRUE2) { // u 316 | if (buffer[i] === 0x75) { this.tState = TRUE3; } 317 | else { return this.charError(buffer, i); } 318 | } else if (this.tState === TRUE3) { // e 319 | if (buffer[i] === 0x65) { this.tState = START; this.onToken(TRUE, true); this.offset += 3; } 320 | else { return this.charError(buffer, i); } 321 | } else if (this.tState === FALSE1) { // a 322 | if (buffer[i] === 0x61) { this.tState = FALSE2; } 323 | else { return this.charError(buffer, i); } 324 | } else if (this.tState === FALSE2) { // l 325 | if (buffer[i] === 0x6c) { this.tState = FALSE3; } 326 | else { return this.charError(buffer, i); } 327 | } else if (this.tState === FALSE3) { // s 328 | if (buffer[i] === 0x73) { this.tState = FALSE4; } 329 | else { return this.charError(buffer, i); } 330 | } else if (this.tState === FALSE4) { // e 331 | if (buffer[i] === 0x65) { this.tState = START; this.onToken(FALSE, false); this.offset += 4; } 332 | else { return this.charError(buffer, i); } 333 | } else if (this.tState === NULL1) { // u 334 | if (buffer[i] === 0x75) { this.tState = NULL2; } 335 | else { return this.charError(buffer, i); } 336 | } else if (this.tState === NULL2) { // l 337 | if (buffer[i] === 0x6c) { this.tState = NULL3; } 338 | else { return this.charError(buffer, i); } 339 | } else if (this.tState === NULL3) { // l 340 | if (buffer[i] === 0x6c) { this.tState = START; this.onToken(NULL, null); this.offset += 3; } 341 | else { return this.charError(buffer, i); } 342 | } 343 | } 344 | } 345 | 346 | parseError(token, value) { 347 | this.tState = STOP; 348 | this.onError(new Error("Unexpected " + Parser.toknam(token) + (value ? ("(" + JSON.stringify(value) + ")") : "") + " in state " + Parser.toknam(this.state))); 349 | } 350 | push() { 351 | this.stack.push({ value: this.value, key: this.key, mode: this.mode }); 352 | } 353 | pop() { 354 | const value = this.value; 355 | const parent = this.stack.pop(); 356 | this.value = parent.value; 357 | this.key = parent.key; 358 | this.mode = parent.mode; 359 | this.emit(value); 360 | if (!this.mode) { this.state = VALUE; } 361 | } 362 | emit(value) { 363 | if (this.mode) { this.state = COMMA; } 364 | this.onValue(value); 365 | } 366 | onValue(value) { 367 | // Override me 368 | } 369 | onToken(token, value) { 370 | if (this.state === VALUE) { 371 | if (token === STRING || token === NUMBER || token === TRUE || token === FALSE || token === NULL) { 372 | if (this.value) { 373 | this.value[this.key] = value; 374 | } 375 | this.emit(value); 376 | } else if (token === LEFT_BRACE) { 377 | this.push(); 378 | if (this.value) { 379 | this.value = this.value[this.key] = {}; 380 | } else { 381 | this.value = {}; 382 | } 383 | this.key = undefined; 384 | this.state = KEY; 385 | this.mode = OBJECT; 386 | } else if (token === LEFT_BRACKET) { 387 | this.push(); 388 | if (this.value) { 389 | this.value = this.value[this.key] = []; 390 | } else { 391 | this.value = []; 392 | } 393 | this.key = 0; 394 | this.mode = ARRAY; 395 | this.state = VALUE; 396 | } else if (token === RIGHT_BRACE) { 397 | if (this.mode === OBJECT) { 398 | this.pop(); 399 | } else { 400 | return this.parseError(token, value); 401 | } 402 | } else if (token === RIGHT_BRACKET) { 403 | if (this.mode === ARRAY) { 404 | this.pop(); 405 | } else { 406 | return this.parseError(token, value); 407 | } 408 | } else { 409 | return this.parseError(token, value); 410 | } 411 | } else if (this.state === KEY) { 412 | if (token === STRING) { 413 | this.key = value; 414 | this.state = COLON; 415 | } else if (token === RIGHT_BRACE) { 416 | this.pop(); 417 | } else { 418 | return this.parseError(token, value); 419 | } 420 | } else if (this.state === COLON) { 421 | if (token === COLON) { this.state = VALUE; } 422 | else { return this.parseError(token, value); } 423 | } else if (this.state === COMMA) { 424 | if (token === COMMA) { 425 | if (this.mode === ARRAY) { this.key++; this.state = VALUE; } 426 | else if (this.mode === OBJECT) { this.state = KEY; } 427 | 428 | } else if (token === RIGHT_BRACKET && this.mode === ARRAY || token === RIGHT_BRACE && this.mode === OBJECT) { 429 | this.pop(); 430 | } else { 431 | return this.parseError(token, value); 432 | } 433 | } else { 434 | return this.parseError(token, value); 435 | } 436 | } 437 | 438 | // Override to implement your own number reviver. 439 | // Any value returned is treated as error and will interrupt parsing. 440 | numberReviver(text) { 441 | const result = Number(text); 442 | 443 | if (isNaN(result)) { 444 | return this.charError(buffer, i); 445 | } 446 | 447 | if ((text.match(/[0-9]+/) == text) && (result.toString() != text)) { 448 | // Long string of digits which is an ID string and not valid and/or safe JavaScript integer Number 449 | this.onToken(STRING, text); 450 | } else { 451 | this.onToken(NUMBER, result); 452 | } 453 | } 454 | } 455 | 456 | Parser.C = C; 457 | 458 | export { Parser as JSONParser }; 459 | export default Parser; 460 | -------------------------------------------------------------------------------- /json-path.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-explicit-any no-prototype-builtins 2 | 3 | // Modernized version of Stefan Goessner's original JSON Path implementation. 4 | // Copyright (c) 2007 Stefan Goessner (goessner.net) 5 | // Licensed under the MIT license. 6 | 7 | // TODO: refactor to avoid string splitting/joining 8 | 9 | export function* trace(expr: string, val: unknown, path: string): IterableIterator<[string, T]> { 10 | if (expr) { 11 | const [loc, ...rest] = expr.split(";"); 12 | const x = rest.join(";"); 13 | 14 | if (val !== null && typeof val === 'object' && loc in val) { 15 | yield* trace(x, (val)[loc], path + ";" + loc); 16 | } 17 | else if (loc === "*") { 18 | for (const [m, _l, v, p] of walk(loc, val, path)) { 19 | yield* trace(m + ";" + x, v, p) 20 | } 21 | } 22 | else if (loc === "..") { 23 | yield* trace(x, val, path); 24 | for (const [m, _l, v, p] of walk(loc, val, path)) { 25 | if (typeof (v)[m] === "object") 26 | yield* trace("..;" + x, (v)[m], p + ";" + m); 27 | } 28 | } 29 | else if (/,/.test(loc)) { // [name1,name2,...] 30 | for (let s = loc.split(/'?,'?/), i = 0, n = s.length; i < n; i++) 31 | yield* trace(s[i] + ";" + x, val, path); 32 | } 33 | else if (/^(-?[0-9]*):(-?[0-9]*):?([0-9]*)$/.test(loc)) { // [start:end:step] slice syntax 34 | yield* slice(loc, x, val, path); 35 | } 36 | } 37 | else yield [path, val as T] 38 | } 39 | 40 | function* slice(loc: string, expr: string, val: unknown, path: string): IterableIterator<[string, T]> { 41 | if (val instanceof Array) { 42 | const len = val.length; 43 | let start = 0, end = len, step = 1; 44 | loc.replace(/^(-?[0-9]*):(-?[0-9]*):?(-?[0-9]*)$/g, (_$0, $1, $2, $3) => { 45 | start = parseInt($1 || start); 46 | end = parseInt($2 || end); 47 | step = parseInt($3 || step); 48 | return '' 49 | }); 50 | start = (start < 0) ? Math.max(0, start + len) : Math.min(len, start); 51 | end = (end < 0) ? Math.max(0, end + len) : Math.min(len, end); 52 | for (let i = start; i < end; i += step) 53 | yield* trace(i + ";" + expr, val, path); 54 | } 55 | } 56 | 57 | function* walk(loc: string, val: unknown, path: string) { 58 | if (val instanceof Array) { 59 | for (let i = 0, n = val.length; i < n; i++) 60 | if (i in val) 61 | yield [i, loc, val, path] as const 62 | } 63 | else if (typeof val === "object") { 64 | for (const m in val) 65 | if (val.hasOwnProperty(m)) 66 | yield [m, loc, val, path] as const 67 | } 68 | } 69 | 70 | export function normalize(expr: string) { 71 | const subX: string[] = []; 72 | if (!expr.startsWith('$')) expr = '$' + expr 73 | return expr 74 | .replace(/[\['](\??\(.*?\))[\]']/g, (_$0, $1) => { return "[#" + (subX.push($1) - 1) + "]"; }) 75 | .replace(/'?\.'?|\['?/g, ";") 76 | .replace(/;;;|;;/g, ";..;") 77 | .replace(/;$|'?\]|'$/g, "") 78 | .replace(/#([0-9]+)/g, (_$0, $1) => { return subX[$1]; }); 79 | } 80 | 81 | // FIXME: avoid repeated split/join/regex.test 82 | export function match(expr: string, path: string): boolean { 83 | if (expr && path) { 84 | const [loc, ...restLoc] = expr.split(";"); 85 | const [val, ...restVal] = path.split(";"); 86 | const exprRest = restLoc.join(";"); 87 | const pathRest = restVal.join(';') 88 | 89 | if (loc === val) { 90 | return match(exprRest, pathRest) 91 | } 92 | else if (loc === "*") { 93 | return match(exprRest, pathRest) 94 | } 95 | else if (loc === "..") { 96 | return match(exprRest, path) || match("..;" + exprRest, pathRest); 97 | } 98 | else if (/,/.test(loc)) { // [name1,name2,...] 99 | if (loc.split(/'?,'?/).some(v => v === val)) return match(exprRest, pathRest) 100 | else return false 101 | } 102 | else if (/^(-?[0-9]*):(-?[0-9]*):?([0-9]*)$/.test(loc)) { // [start:end:step] slice syntax 103 | let start = 0, end = Number.MAX_SAFE_INTEGER, step = 1; 104 | loc.replace(/^(-?[0-9]*):(-?[0-9]*):?(-?[0-9]*)$/g, (_$0, $1, $2, $3) => { 105 | start = parseInt($1 || start); 106 | end = parseInt($2 || end); 107 | step = parseInt($3 || step); 108 | return '' 109 | }); 110 | const idx = Number(val) 111 | if (start < 0 || end < 0 || step < 0) 112 | throw TypeError('Negative numbers not supported. Can\'t know length ahead of time when stream parsing'); 113 | if (idx >= start && idx < end && start + idx % step === 0) return match(exprRest, pathRest) 114 | else return false 115 | } 116 | } 117 | else if (!expr && !path) return true 118 | return false; 119 | } 120 | -------------------------------------------------------------------------------- /json-stringify-stream.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-explicit-any 2 | import { jsonStringifyGenerator } from './json-stringify.ts' 3 | 4 | export class JSONStringifyStream extends TransformStream { 5 | constructor() { 6 | let first: boolean; 7 | super({ 8 | start(controller) { 9 | first = true; 10 | controller.enqueue('[') 11 | }, 12 | async transform(obj, controller) { 13 | if (!first) controller.enqueue(','); else first = false; 14 | for await (const chunk of jsonStringifyGenerator(obj)) { 15 | controller.enqueue(chunk) 16 | } 17 | }, 18 | flush(controller) { 19 | controller.enqueue(']') 20 | }, 21 | }) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /json-stringify.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-explicit-any no-empty 2 | import { asyncIterToStream } from 'https://ghuc.cc/qwtel/whatwg-stream-to-async-iter/index.ts' 3 | 4 | type SeenWeakSet = WeakSet; 5 | 6 | type Primitive = undefined | boolean | number | string | bigint | symbol; 7 | 8 | export type ToJSON = { toJSON: (key?: any) => string } 9 | 10 | export const isIterable = (x: unknown): x is Iterable => 11 | x != null && typeof x === 'object' && Symbol.iterator in x 12 | 13 | export const isAsyncIterable = (x: unknown): x is AsyncIterable => 14 | x != null && typeof x === 'object' && Symbol.asyncIterator in x 15 | 16 | const isPromiseLike = (x: unknown): x is PromiseLike => 17 | x != null && typeof x === 'object' && 'then' in x && typeof (x).then === 'function' 18 | 19 | const isToJSON = (x: unknown): x is J => 20 | x != null && typeof x === 'object' && 'toJSON' in x; 21 | 22 | const safeAdd = (seen: SeenWeakSet, value: any) => { 23 | if (seen.has(value)) throw TypeError('Converting circular structure to JSON') 24 | seen.add(value) 25 | } 26 | 27 | const check = (v: any) => { 28 | if (v === undefined) return false; 29 | const type = typeof v; 30 | return type !== 'function' && type !== 'symbol' 31 | } 32 | 33 | // TODO: Add replacer 34 | // TODO: add formatting/spaces 35 | // TODO: concurrent objects/arrays 36 | /** 37 | * @deprecated Change name to something more descriptive!? 38 | */ 39 | export async function* jsonStringifyGenerator( 40 | value: null | Primitive | ToJSON | any[] | Record | PromiseLike | AsyncIterable | ReadableStream, 41 | seen: SeenWeakSet = new WeakSet(), 42 | ): AsyncIterableIterator { 43 | if (isAsyncIterable(value)) { 44 | yield '[' 45 | safeAdd(seen, value) 46 | let first = true; 47 | for await (const v of value) { 48 | if (!first) yield ','; else first = false; 49 | yield* jsonStringifyGenerator(v, seen) 50 | } 51 | seen.delete(value) 52 | yield ']' 53 | } 54 | else if (isPromiseLike(value)) { 55 | const v = await value 56 | if (check(v)) { 57 | safeAdd(seen, value) 58 | yield* jsonStringifyGenerator(v, seen) 59 | seen.delete(value) 60 | } 61 | } 62 | else if (isToJSON(value)) { 63 | const v = JSON.stringify(value); 64 | if (check(v)) yield v 65 | } 66 | else if (Array.isArray(value)) { 67 | yield '[' 68 | safeAdd(seen, value) 69 | let first = true; 70 | for (const v of value) { 71 | if (!first) yield ','; else first = false; 72 | yield* jsonStringifyGenerator(v, seen); 73 | } 74 | seen.delete(value) 75 | yield ']' 76 | } 77 | else if (value != null && typeof value === 'object') { 78 | yield '{' 79 | safeAdd(seen, value) 80 | let first = true; 81 | for (const [k, v] of Object.entries(value)) { 82 | if (check(v)) { 83 | const generator = jsonStringifyGenerator(v, seen) 84 | const peek = await generator.next() 85 | if (check(peek.value)) { 86 | if (!first) yield ','; else first = false; 87 | yield `${JSON.stringify(k)}:` 88 | yield peek.value 89 | yield* generator; 90 | } 91 | } 92 | } 93 | seen.delete(value) 94 | yield '}' 95 | } 96 | else { 97 | yield check(value) ? JSON.stringify(value) : 'null' 98 | } 99 | } 100 | 101 | /** 102 | * @deprecated Change name to something more descriptive!? 103 | */ 104 | export function jsonStringifyStream( 105 | value: null | Primitive | ToJSON | any[] | Record | PromiseLike | AsyncIterable | ReadableStream, 106 | ): ReadableStream { 107 | return asyncIterToStream(jsonStringifyGenerator(value)) 108 | } 109 | 110 | export class JSONStringifyReadable extends ReadableStream { 111 | constructor(value: any) { 112 | let iterator: AsyncIterator; 113 | super({ 114 | start() { 115 | iterator = jsonStringifyGenerator(value)[Symbol.asyncIterator]() 116 | }, 117 | async pull(controller) { 118 | // console.log('stringify', controller.desiredSize) 119 | const { value, done } = await iterator.next(); 120 | if (!done) controller.enqueue(value); else controller.close(); 121 | }, 122 | async cancel(reason) { 123 | try { await iterator.throw?.(reason) } catch { } 124 | }, 125 | }) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /scripts/build_npm.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env -S deno run --allow-read --allow-write=./,/Users/qwtel/Library/Caches/deno --allow-net --allow-env=HOME,DENO_AUTH_TOKENS,DENO_DIR --allow-run=git,pnpm 2 | 3 | import { basename, extname } from "https://deno.land/std@0.133.0/path/mod.ts"; 4 | import { build, emptyDir } from "https://deno.land/x/dnt/mod.ts"; 5 | 6 | import { 7 | copyMdFiles, mkPackage, 8 | } from 'https://gist.githubusercontent.com/qwtel/ecf0c3ba7069a127b3d144afc06952f5/raw/latest-version.ts' 9 | 10 | await emptyDir("./npm"); 11 | 12 | const name = basename(Deno.cwd()) 13 | 14 | await build({ 15 | entryPoints: ["./index.ts"], 16 | outDir: "./npm", 17 | shims: {}, 18 | test: false, 19 | package: await mkPackage(name), 20 | declaration: true, 21 | packageManager: 'pnpm', 22 | compilerOptions: { 23 | sourceMap: true, 24 | target: 'ES2019', 25 | }, 26 | mappings: { 27 | 'https://cdn.skypack.dev/ts-functional-pipe@3.1.2?dts': { 28 | name: 'ts-functional-pipe', 29 | version: '3.1.2', 30 | }, 31 | 'https://ghuc.cc/worker-tools/resolvable-promise/index.ts': { 32 | name: '@worker-tools/resolvable-promise', 33 | version: 'latest', 34 | }, 35 | 'https://ghuc.cc/worker-tools/stream-response/index.ts': { 36 | name: '@worker-tools/stream-response', 37 | version: 'latest', 38 | }, 39 | 'https://ghuc.cc/qwtel/whatwg-stream-to-async-iter/index.ts': { 40 | name: 'whatwg-stream-to-async-iter', 41 | version: 'latest', 42 | }, 43 | }, 44 | }); 45 | 46 | // post build steps 47 | await copyMdFiles(); 48 | -------------------------------------------------------------------------------- /task-promise.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-explicit-any 2 | import { ResolvablePromise } from 'https://ghuc.cc/worker-tools/resolvable-promise/index.ts'; 3 | import { pipe } from 'https://cdn.skypack.dev/ts-functional-pipe@3.1.2?dts'; 4 | 5 | const id = (_: T) => _; 6 | 7 | type Awaitable = T | PromiseLike; 8 | 9 | export type TaskState = 'idle' | 'pending' | 'fulfilled' | 'rejected'; 10 | 11 | class Task { 12 | #task; 13 | #promise; 14 | #state: TaskState = 'idle' 15 | 16 | constructor(task: () => Awaitable) { 17 | this.#task = task; 18 | this.#promise = new ResolvablePromise(); 19 | } 20 | 21 | execute() { 22 | if (this.#state === 'idle') { 23 | this.#state = 'pending' 24 | this.#promise.resolve(this.#task()) 25 | this.#promise.then( 26 | () => { this.#state = 'fulfilled' }, 27 | () => { this.#state = 'rejected' }, 28 | ); 29 | } 30 | } 31 | get state(): TaskState { return this.#state } 32 | get promise(): Promise { return this.#promise } 33 | } 34 | 35 | const lock = Symbol('key'); 36 | 37 | // TODO: Make own module? 38 | // TODO: Add abort signal? 39 | // TODO: use executor instead of task functions? 40 | // TODO: Remove TT type?? 41 | export class TaskPromise implements Promise { 42 | #task: Task; 43 | #mapFn; 44 | #mappedPromise; 45 | 46 | static from(task: () => Awaitable) { 47 | return new TaskPromise(lock, new Task(task)) 48 | } 49 | 50 | private constructor( 51 | key: symbol, 52 | task: Task, 53 | mapFn?: ((value: TT, i?: 0, p?: TaskPromise) => Awaitable) | undefined | null, 54 | thisArg?: any, 55 | ) { 56 | if (key !== lock) throw Error('Illegal constructor'); 57 | this.#task = task; 58 | this.#mapFn = mapFn; 59 | this.#mappedPromise = this.#task.promise.then(mapFn && (x => mapFn.call(thisArg, x, 0, this))); 60 | } 61 | 62 | get state() { 63 | return this.#task.state; 64 | } 65 | 66 | /** 67 | * Starts the execution of the task associated with this task promise. 68 | * If you don't want to start the task at this moment, use `.map` instead. 69 | */ 70 | then( 71 | onfulfilled?: ((value: T) => Awaitable) | undefined | null, 72 | onrejected?: ((reason: any) => Awaitable) | undefined | null 73 | ): Promise { 74 | this.#task.execute(); 75 | return this.#mappedPromise.then(onfulfilled, onrejected) 76 | } 77 | 78 | /** 79 | * Applies transformations to the resolved value without triggering execution. 80 | * Returns another task promise that triggers execution via `.then` 81 | */ 82 | map( 83 | mapFn?: ((value: T, i?: 0, p?: TaskPromise) => Awaitable) | undefined | null, 84 | thisArg?: any 85 | ): TaskPromise { 86 | // @ts-ignore: types of id function (x => x) not correctly inferred... 87 | return new TaskPromise(lock, this.#task, pipe(this.#mapFn??id, mapFn??id), thisArg); 88 | } 89 | 90 | catch(onrejected?: ((reason: any) => Awaitable) | null): Promise { 91 | // FIXME: should this also trigger execution? 92 | return this.#mappedPromise.catch(onrejected) 93 | } 94 | 95 | finally(onfinally?: (() => void) | null): Promise { 96 | // FIXME: should this also trigger execution? 97 | return this.#mappedPromise.finally(onfinally) 98 | } 99 | 100 | readonly [Symbol.toStringTag] = 'TaskPromise' 101 | } 102 | -------------------------------------------------------------------------------- /test/index.test.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-explicit-any no-unused-vars require-await ban-unused-ignore 2 | import 'https://gist.githubusercontent.com/qwtel/b14f0f81e3a96189f7771f83ee113f64/raw/TestRequest.ts' 3 | import { 4 | assert, 5 | assertExists, 6 | assertEquals, 7 | assertStrictEquals, 8 | assertStringIncludes, 9 | assertThrows, 10 | assertRejects, 11 | assertArrayIncludes, 12 | } from 'https://deno.land/std@0.133.0/testing/asserts.ts' 13 | const { test } = Deno; 14 | 15 | import * as json from '../index.ts'; 16 | 17 | test('transform streams', () => { 18 | assertExists(json.JSONParseStream) 19 | assertExists(json.JSONStringifyStream) 20 | }) 21 | 22 | test('stringify', () => { 23 | assertExists(json.jsonStringifyGenerator) 24 | assertExists(json.jsonStringifyStream) 25 | }) 26 | -------------------------------------------------------------------------------- /test/json-fetch.test.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-unused-vars no-explicit-any 2 | import 'https://gist.githubusercontent.com/qwtel/b14f0f81e3a96189f7771f83ee113f64/raw/TestRequest.ts' 3 | import { 4 | assert, 5 | assertExists, 6 | assertEquals, 7 | assertStrictEquals, 8 | assertStringIncludes, 9 | assertThrows, 10 | assertRejects, 11 | assertArrayIncludes, 12 | } from 'https://deno.land/std@0.133.0/testing/asserts.ts' 13 | const { test } = Deno; 14 | 15 | import { JSONStreamResponse, JSONStreamRequest } from '../json-fetch.ts' 16 | import { JSONParseStream } from '../index.ts' 17 | 18 | test('exists', () =>{ 19 | assertExists(JSONStreamRequest) 20 | assertExists(JSONStreamResponse) 21 | }) 22 | 23 | test('simple response', async () => { 24 | const actual = await new JSONStreamResponse({ a: 3, b: { nested: 4 }, c: [1, 2, 3], __x: undefined }).json() 25 | assertEquals(actual, { a: 3, b: { nested: 4 }, c: [1, 2, 3] }) 26 | }) 27 | 28 | test('simple request', async () => { 29 | const actual = await new JSONStreamRequest('/', { method: 'PUT', body: { a: 3, b: { nested: 4 }, c: [1, 2, 3], __x: undefined } }).json() 30 | assertEquals(actual, { a: 3, b: { nested: 4 }, c: [1, 2, 3] }) 31 | }) 32 | 33 | test('with promise response', async () => { 34 | const actual = await new JSONStreamResponse(({ a: 3, b: Promise.resolve(4) })).json() 35 | assertEquals(actual, { a: 3, b: 4 }) 36 | }) 37 | 38 | test('with promise request', async () => { 39 | const actual = await new JSONStreamRequest('/', { method: 'PUT', body: { a: 3, b: Promise.resolve(4) } }).json() 40 | assertEquals(actual, { a: 3, b: 4 }) 41 | }) 42 | 43 | const timeout = (n?: number) => new Promise(r => setTimeout(r, n)) 44 | async function* asyncGen(xs: T[]) { 45 | for (const x of xs) { await timeout(); yield x } 46 | } 47 | 48 | test('with generator response', async () => { 49 | const actual = await new JSONStreamResponse(({ a: 3, b: Promise.resolve(4), c: asyncGen([1, 2, 3]) })).text() 50 | assertEquals(actual, JSON.stringify({ a: 3, b: 4, c: [1, 2, 3] })) 51 | }) 52 | 53 | test('with generator request', async () => { 54 | const actual = await new JSONStreamRequest('/', { method: 'PUT', body: { a: 3, b: Promise.resolve(4), c: asyncGen([1, 2, 3]) } }).json() 55 | assertEquals(actual, { a: 3, b: 4, c: [1, 2, 3] }) 56 | }) 57 | 58 | test('circular throws', () => { 59 | const a: any = { a: 3, foo: { b: 4 } } 60 | a.foo.a = a; 61 | assertRejects(() => new JSONStreamResponse((a)).json(), TypeError) 62 | }) 63 | 64 | test('GET with body throws', () => { 65 | const a: any = { a: 3, foo: { b: 4 } } 66 | assertRejects(() => new JSONStreamRequest('/', { body: a }).json(), TypeError) 67 | }) 68 | 69 | test('stream', async () => { 70 | const actual = new JSONStreamResponse(({ a: 3, b: Promise.resolve(4), c: asyncGen([1, 2, 3]) })) 71 | const reader = actual.body!.pipeThrough(new JSONParseStream('$.c.*')).getReader() 72 | assertEquals((await reader.read()).value, 1) 73 | assertEquals((await reader.read()).value, 2) 74 | assertEquals((await reader.read()).value, 3) 75 | assertEquals((await reader.read()).done, true) 76 | }) 77 | -------------------------------------------------------------------------------- /test/json-parse-stream.test.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-explicit-any no-unused-vars require-await ban-unused-ignore 2 | import 'https://gist.githubusercontent.com/qwtel/b14f0f81e3a96189f7771f83ee113f64/raw/TestRequest.ts' 3 | import { 4 | assert, 5 | assertExists, 6 | assertEquals, 7 | assertStrictEquals, 8 | assertStringIncludes, 9 | assertThrows, 10 | assertRejects, 11 | assertArrayIncludes, 12 | } from 'https://deno.land/std@0.133.0/testing/asserts.ts' 13 | const { test } = Deno; 14 | 15 | import { jsonStringifyGenerator, jsonStringifyStream, JSONStringifyReadable } from '../json-stringify.ts' 16 | import { JSONParseStream, JSONParseNexus } from '../json-parse-stream.ts' 17 | 18 | async function collect(stream: ReadableStream) { 19 | const chunks = [] 20 | const reader = stream.getReader(); 21 | let result: ReadableStreamReadResult 22 | while (!(result = await reader.read()).done) chunks.push(result.value) 23 | return chunks; 24 | } 25 | 26 | async function aCollect(iter: AsyncIterable) { 27 | const chunks = [] 28 | for await (const x of iter) chunks.push(x) 29 | return chunks; 30 | } 31 | 32 | test('exists', () => { 33 | assertExists(JSONParseStream) 34 | }) 35 | 36 | test('ctor', () => { 37 | const x = new JSONParseStream() 38 | assertExists(x) 39 | assertExists(x.readable) 40 | assertExists(x.writable) 41 | }) 42 | 43 | test('simple', async () => { 44 | const res = await collect(new Response(JSON.stringify([{ a: 1 }, { b: 2 }, { c: 3 }])).body! 45 | .pipeThrough(new JSONParseStream())) 46 | assertEquals(res, [{ a: 1 }, { b: 2 }, { c: 3 }]) 47 | }) 48 | 49 | test('simple reader read', async () => { 50 | const reader = new Response(JSON.stringify([{ a: 1 }, { b: 2 }, { c: 3 }])).body! 51 | .pipeThrough(new JSONParseStream()) 52 | .getReader() 53 | assertEquals((await reader.read()).value, { a: 1 }) 54 | assertEquals((await reader.read()).value, { b: 2 }) 55 | assertEquals((await reader.read()).value, { c: 3 }) 56 | assertEquals((await reader.read()).done, true) 57 | }) 58 | 59 | test('read all', async () => { 60 | const stream = new Response(JSON.stringify([{ a: 1 }, { b: 2 }, { c: 3 }])).body! 61 | .pipeThrough(new JSONParseStream('$..*')) 62 | const reader = stream.getReader() 63 | assertEquals((await reader.read()).value, 1) 64 | assertEquals((await reader.read()).value, { a: 1 }) 65 | assertEquals((await reader.read()).value, 2) 66 | assertEquals((await reader.read()).value, { b: 2 }) 67 | assertEquals((await reader.read()).value, 3) 68 | assertEquals((await reader.read()).value, { c: 3 }) 69 | assertEquals((await reader.read()).done, true) 70 | }) 71 | 72 | const aJoin = async (iter: AsyncIterable, separator = '') => { 73 | const chunks: string[] = [] 74 | for await (const x of iter) chunks.push(x) 75 | return chunks.join(separator) 76 | } 77 | 78 | test('promise value', async () => { 79 | const nexus = new JSONParseNexus() 80 | const actual = { 81 | type: nexus.promise('$.type'), 82 | items: nexus.iterable('$.items.*') 83 | } 84 | const expected = JSON.stringify({ type: 'foo', items: [{ a: 1 }, { a: 2 }, { a: 3 }] }) 85 | new Response(expected).body!.pipeThrough(nexus) 86 | 87 | const actualString = await aJoin(jsonStringifyGenerator(actual)) 88 | // console.log('actualString', actualString) 89 | assertEquals(actualString, expected) 90 | }) 91 | 92 | test('promise value II', async () => { 93 | const nexus = new JSONParseNexus() 94 | const actual = { 95 | type: nexus.promise('$.type'), 96 | items: nexus.stream('$.items.*') 97 | } 98 | const expected = JSON.stringify({ type: 'foo', items: [{ a: 1 }, { a: 2 }, { a: 3 }] }) 99 | new Response(expected).body! 100 | .pipeThrough(nexus) 101 | 102 | const actualString = await aJoin(jsonStringifyGenerator(actual)) 103 | assertEquals(actualString, expected) 104 | }) 105 | 106 | test('promise value III', async () => { 107 | const nexus = new JSONParseNexus() 108 | const actual = { 109 | type: nexus.promise('$.type'), 110 | items: nexus.iterable('$.items.*') 111 | } 112 | const expected = JSON.stringify({ type: 'foo', items: [{ a: 1 }, { a: 2 }, { a: 3 }] }) 113 | new Response(expected).body!.pipeThrough(nexus) 114 | 115 | const actualString = await aJoin(jsonStringifyGenerator(actual)) 116 | assertEquals(actualString, expected) 117 | }) 118 | 119 | test('promise value IV', async () => { 120 | const nexus = new JSONParseNexus() 121 | const actual = { 122 | type: nexus.promise('$.type'), 123 | items: nexus.stream('$.items.*') 124 | } 125 | const expected = JSON.stringify({ type: 'foo', items: [{ a: 1 }, { a: 2 }, { a: 3 }] }) 126 | new Response(expected).body!.pipeThrough(nexus) 127 | 128 | const actualString = await aJoin(jsonStringifyGenerator(actual)) 129 | assertEquals(actualString, expected) 130 | }) 131 | 132 | async function* asyncGen(xs: T[]) { 133 | for (const x of xs) yield x 134 | } 135 | 136 | const filler = new Array(10).fill('___'); 137 | const items = Array.from(new Array(10), (_, a) => ({ a })); 138 | const json1 = () => ({ 139 | filler: asyncGen(filler), 140 | type: 'foo', 141 | items: asyncGen(items), 142 | done: true, 143 | }); 144 | 145 | // console.log(filler, items) 146 | 147 | test('read only until first value eager', async () => { 148 | const nexus = new JSONParseNexus() 149 | const type = nexus.promise('$.type'); 150 | new JSONStringifyReadable(json1()).pipeThrough(nexus) 151 | 152 | assertEquals(await type, 'foo') 153 | }) 154 | 155 | const timeout = (n?: number) => new Promise(r => setTimeout(r, n)) 156 | 157 | test('read only until first value lazy', async () => { 158 | const nexus = new JSONParseNexus() 159 | const type = nexus.promise('$.type'); 160 | 161 | let hasBeenCalled = false 162 | async function* asyncGen(xs: T[]) { 163 | for (const x of xs) { yield x; hasBeenCalled = true } 164 | } 165 | 166 | new JSONStringifyReadable({ 167 | items: asyncGen(items), 168 | type: 'foo', 169 | }).pipeThrough(nexus) 170 | 171 | assertEquals(hasBeenCalled, false) 172 | assertEquals(await type, 'foo') 173 | }) 174 | 175 | test('read only until first value lazy II', async () => { 176 | const nexus = new JSONParseNexus() 177 | const typeP = nexus.promise('$.type'); 178 | const itemsS = nexus.stream('$.items.*') 179 | 180 | let callCount = 0 181 | async function* asyncGen(xs: T[]) { 182 | for (const x of xs) { yield x; callCount++ } 183 | } 184 | 185 | new JSONStringifyReadable({ 186 | type: 'foo', 187 | items: asyncGen(items), 188 | }).pipeThrough(nexus) 189 | 190 | assertEquals(callCount, 0) 191 | assertEquals(await typeP, 'foo') 192 | assertEquals(callCount, 0) 193 | await collect(itemsS) 194 | assert(callCount >= items.length) 195 | }) 196 | 197 | test('lazy promise map', async () => { 198 | const nexus = new JSONParseNexus() 199 | const type = nexus.promise('$.type') 200 | .map(x => x?.toUpperCase()); 201 | 202 | let hasBeenCalled = false 203 | async function* asyncGen(xs: T[]) { 204 | for (const x of xs) { yield x; hasBeenCalled = true } 205 | } 206 | 207 | new JSONStringifyReadable({ 208 | items: asyncGen(items), 209 | type: 'foo', 210 | }).pipeThrough(nexus) 211 | 212 | assertEquals(hasBeenCalled, false) 213 | assertEquals(await type, 'FOO') 214 | assertEquals(hasBeenCalled, true) 215 | }) 216 | 217 | test('lazy promise map x2', async () => { 218 | const nexus = new JSONParseNexus() 219 | const type0 = nexus.promise('$.type') 220 | const type1 = type0.map(x => x?.toUpperCase()) 221 | const type2 = type1.map(x => `${x}!!!`) 222 | 223 | let hasBeenCalled = false 224 | async function* asyncGen(xs: T[]) { 225 | for (const x of xs) { yield x; hasBeenCalled = true } 226 | } 227 | 228 | new JSONStringifyReadable({ 229 | items: asyncGen(items), 230 | type: 'foo', 231 | }).pipeThrough(nexus) 232 | 233 | assertEquals(hasBeenCalled, false) 234 | assertEquals(await type2, 'FOO!!!') 235 | assertEquals(hasBeenCalled, true) 236 | assertEquals(await type1, 'FOO') 237 | assertEquals(await type0, 'foo') 238 | }) 239 | 240 | test('two streams', async () => { 241 | const nexus = new JSONParseNexus() 242 | const fillerS = nexus.stream('$.filler.*'); 243 | const itemsS = nexus.stream('$.items.*'); 244 | new JSONStringifyReadable(json1()).pipeThrough(nexus) 245 | assertEquals(await collect(fillerS), filler) 246 | assertEquals(await collect(itemsS), items) 247 | }) 248 | 249 | test('two generators', async () => { 250 | const nexus = new JSONParseNexus() 251 | const fillerS = nexus.iterable('$.filler.*'); 252 | const itemsS = nexus.iterable('$.items.*'); 253 | new JSONStringifyReadable(json1()).pipeThrough(nexus) 254 | assertEquals(await aCollect(fillerS), filler) 255 | assertEquals(await aCollect(itemsS), items) 256 | }) 257 | 258 | test('boxy selector', async () => { 259 | const nexus = new JSONParseNexus() 260 | const itemsS = nexus.stream('$.items.*[a]'); 261 | new JSONStringifyReadable(json1()).pipeThrough(nexus) 262 | assertEquals(await collect(itemsS), items.map(x => x.a)) 263 | }) 264 | 265 | test('from file/fetch', async () => { 266 | const nexus = new JSONParseNexus(); 267 | const fillerS = nexus.iterable('$.filler.*'); 268 | const itemsS = nexus.iterable('$.items.*'); 269 | (await fetch(new URL('./json1.json', import.meta.url).href)).body!.pipeThrough(nexus) 270 | assertEquals((await aCollect(fillerS)).length, 300) 271 | assertEquals((await aCollect(itemsS)).length, 300) 272 | }) 273 | 274 | test('promise throws without data source', () => { 275 | const nexus = new JSONParseNexus(); 276 | const type = nexus.promise('$.type') 277 | assertRejects(() => type, TypeError) 278 | }) 279 | 280 | test('streams throws without data source', () => { 281 | const nexus = new JSONParseNexus(); 282 | const items = nexus.stream('$.items.*') 283 | assertRejects(() => items.getReader().read(), TypeError) 284 | }) 285 | 286 | test('iterations throws without data source', () => { 287 | const nexus = new JSONParseNexus(); 288 | const items = nexus.iterable('$.items.*') 289 | assertRejects(() => items.next(), TypeError) 290 | }) 291 | 292 | test('stream remains open when only promises left', async () => { 293 | const nexus = new JSONParseNexus(); 294 | 295 | const stream = nexus.stream('$.items.*') 296 | const type = nexus.promise('$.type') 297 | 298 | new JSONStringifyReadable({ 299 | items: asyncGen(items), 300 | type: 'foo', 301 | }).pipeThrough(nexus) 302 | 303 | await collect(stream) 304 | await timeout(10) 305 | assertEquals(await type, 'foo') 306 | }) 307 | -------------------------------------------------------------------------------- /test/json-path.test.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-unused-vars 2 | import 'https://gist.githubusercontent.com/qwtel/b14f0f81e3a96189f7771f83ee113f64/raw/TestRequest.ts' 3 | import { 4 | assert, 5 | assertExists, 6 | assertEquals, 7 | assertStrictEquals, 8 | assertStringIncludes, 9 | assertThrows, 10 | assertRejects, 11 | assertArrayIncludes, 12 | } from 'https://deno.land/std@0.133.0/testing/asserts.ts' 13 | const { test } = Deno; 14 | 15 | import { normalize, match as _match } from '../json-path.ts' 16 | 17 | function match(...args: Parameters): boolean { 18 | return _match(normalize(args[0]), normalize(args[1])) 19 | } 20 | 21 | // console.log([...trace(normalize('$..*').replace(/^\$;/, ""), { a: 3, b: 4, c: 5, d: [1, 2, 3], e: { f: { g: { h: 5 } } } }, '$')]) 22 | // consume(new Response(JSON.stringify({ a: 3, b: 4, c: 5, d: [1, 2, 3], e: { f: { g: { h: 5 } } } })).body!.pipeThrough(new JSONParseStream('$..*'))) 23 | 24 | test('exists', () =>{ 25 | assertExists(_match) 26 | }) 27 | 28 | test('*', () =>{ 29 | assert(match('.*', '.a')) 30 | assert(match('.*', '.b')) 31 | assert(!match('.*', '.a.b')) 32 | }) 33 | 34 | test('..', () =>{ 35 | assert(match('..*', '.store.price')); 36 | assert(match('..*', '.store.a.price')); 37 | assert(match('..*', '.store.a.b.price')); 38 | assert(match('..*', '.store.a.price.b')); 39 | assert(match('..*', '.store.foo')); 40 | assert(match('..*', '.store')); 41 | }) 42 | 43 | 44 | test('.. with follow-up', () =>{ 45 | assert(match('.store..price', '.store.price')); 46 | assert(match('.store..price', '.store.a.price')); 47 | assert(match('.store..price', '.store.a.b.price')); 48 | assert(!match('.store..price', '.store.a.price.b')); 49 | assert(!match('.store..price', '.store.foo')); 50 | assert(!match('.store..price', '.store')); 51 | }) 52 | 53 | test('selection', () => { 54 | assert(match('$..foo[a,b]', '$.x.foo.a')); 55 | assert(match('$..foo[a,b]', '$.x.foo.b')); 56 | assert(!match('$..foo[a,b]', '$.x.foo.c')); 57 | }) 58 | 59 | test('selection num', () => { 60 | assert(match('$..book[0,1]', '$.book[0]')); 61 | assert(match('$..book[0,1]', '$.book[0]')); 62 | assert(!match('$..book[0,1]', '$.book[2]')); 63 | }) 64 | 65 | test('range', () => { 66 | assert(match('$..book[0:2]', '$.book[0]')); 67 | assert(match('$..book[0:2]', '$.book[0]')); 68 | assert(!match('$..book[0:2]', '$.book[2]')); 69 | }) 70 | -------------------------------------------------------------------------------- /test/json-stringify.test.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-unused-vars no-explicit-any 2 | import 'https://gist.githubusercontent.com/qwtel/b14f0f81e3a96189f7771f83ee113f64/raw/TestRequest.ts' 3 | import { 4 | assert, 5 | assertExists, 6 | assertEquals, 7 | assertStrictEquals, 8 | assertStringIncludes, 9 | assertThrows, 10 | assertRejects, 11 | assertArrayIncludes, 12 | } from 'https://deno.land/std@0.133.0/testing/asserts.ts' 13 | const { test } = Deno; 14 | 15 | import { jsonStringifyGenerator } from '../index.ts' 16 | 17 | test('exists', () =>{ 18 | assertExists(jsonStringifyGenerator) 19 | }) 20 | 21 | // const aConcat = async (iter: AsyncIterable) => { 22 | // const chunks: T[] = [] 23 | // for await (const x of iter) chunks.push(x) 24 | // return chunks 25 | // } 26 | 27 | const aJoin = async (iter: AsyncIterable, separator = '') => { 28 | const chunks: string[] = [] 29 | for await (const x of iter) chunks.push(x) 30 | return chunks.join(separator) 31 | } 32 | 33 | test('simple', async () => { 34 | const text = await aJoin(jsonStringifyGenerator({ a: 3, b: { nested: 4 }, c: [1, 2, 3], __x: undefined })) 35 | assertEquals(text, JSON.stringify({ a: 3, b: { nested: 4 }, c: [1, 2, 3] })) 36 | }) 37 | 38 | test('with promise', async () => { 39 | const text = await aJoin(jsonStringifyGenerator({ a: 3, b: Promise.resolve(4) })) 40 | assertEquals(text, JSON.stringify({ a: 3, b: 4 })) 41 | }) 42 | 43 | const timeout = (n?: number) => new Promise(r => setTimeout(r, n)) 44 | async function* asyncGen(xs: T[]) { 45 | for (const x of xs) { await timeout(); yield x } 46 | } 47 | 48 | test('with generator', async () => { 49 | const text = await aJoin(jsonStringifyGenerator({ a: 3, b: Promise.resolve(4), c: asyncGen([1, 2, 3]) })) 50 | assertEquals(text, JSON.stringify({ a: 3, b: 4, c: [1, 2, 3] })) 51 | }) 52 | 53 | test('circular throws', () => { 54 | const a: any = { a: 3, foo: { b: 4 } } 55 | a.foo.a = a; 56 | assertRejects(() => aJoin(jsonStringifyGenerator(a)), TypeError) 57 | }) 58 | 59 | test('duplicates do not throw', async () => { 60 | const foo = { foo: 'bar' } 61 | const a = { a: { x: foo, y: foo }, b: foo } 62 | assertEquals(await aJoin(jsonStringifyGenerator(a)), JSON.stringify(a)) 63 | }) 64 | 65 | test('promises that resolve to undefined are omitted', async () => { 66 | assertEquals(await aJoin(jsonStringifyGenerator({ a: Promise.resolve(undefined) })), JSON.stringify({ a: undefined })) 67 | }) 68 | 69 | test('undefined values in generators become null, same as arrays', async () => { 70 | assertEquals(await aJoin(jsonStringifyGenerator({ a: asyncGen([1, undefined, 3]) })), JSON.stringify({ a: [1, undefined, 3] })) 71 | }) 72 | 73 | test('undefined toJSON result', async () => { 74 | const a = { toJSON() { return undefined } } 75 | assertEquals(await aJoin(jsonStringifyGenerator({ a, b: 4 })), JSON.stringify({ a, b: 4 })) 76 | }) 77 | -------------------------------------------------------------------------------- /test/json1.json: -------------------------------------------------------------------------------- 1 | {"filler":["___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___","___"],"type":"foo","items":[{"a":0},{"a":1},{"a":2},{"a":3},{"a":4},{"a":5},{"a":6},{"a":7},{"a":8},{"a":9},{"a":10},{"a":11},{"a":12},{"a":13},{"a":14},{"a":15},{"a":16},{"a":17},{"a":18},{"a":19},{"a":20},{"a":21},{"a":22},{"a":23},{"a":24},{"a":25},{"a":26},{"a":27},{"a":28},{"a":29},{"a":30},{"a":31},{"a":32},{"a":33},{"a":34},{"a":35},{"a":36},{"a":37},{"a":38},{"a":39},{"a":40},{"a":41},{"a":42},{"a":43},{"a":44},{"a":45},{"a":46},{"a":47},{"a":48},{"a":49},{"a":50},{"a":51},{"a":52},{"a":53},{"a":54},{"a":55},{"a":56},{"a":57},{"a":58},{"a":59},{"a":60},{"a":61},{"a":62},{"a":63},{"a":64},{"a":65},{"a":66},{"a":67},{"a":68},{"a":69},{"a":70},{"a":71},{"a":72},{"a":73},{"a":74},{"a":75},{"a":76},{"a":77},{"a":78},{"a":79},{"a":80},{"a":81},{"a":82},{"a":83},{"a":84},{"a":85},{"a":86},{"a":87},{"a":88},{"a":89},{"a":90},{"a":91},{"a":92},{"a":93},{"a":94},{"a":95},{"a":96},{"a":97},{"a":98},{"a":99},{"a":100},{"a":101},{"a":102},{"a":103},{"a":104},{"a":105},{"a":106},{"a":107},{"a":108},{"a":109},{"a":110},{"a":111},{"a":112},{"a":113},{"a":114},{"a":115},{"a":116},{"a":117},{"a":118},{"a":119},{"a":120},{"a":121},{"a":122},{"a":123},{"a":124},{"a":125},{"a":126},{"a":127},{"a":128},{"a":129},{"a":130},{"a":131},{"a":132},{"a":133},{"a":134},{"a":135},{"a":136},{"a":137},{"a":138},{"a":139},{"a":140},{"a":141},{"a":142},{"a":143},{"a":144},{"a":145},{"a":146},{"a":147},{"a":148},{"a":149},{"a":150},{"a":151},{"a":152},{"a":153},{"a":154},{"a":155},{"a":156},{"a":157},{"a":158},{"a":159},{"a":160},{"a":161},{"a":162},{"a":163},{"a":164},{"a":165},{"a":166},{"a":167},{"a":168},{"a":169},{"a":170},{"a":171},{"a":172},{"a":173},{"a":174},{"a":175},{"a":176},{"a":177},{"a":178},{"a":179},{"a":180},{"a":181},{"a":182},{"a":183},{"a":184},{"a":185},{"a":186},{"a":187},{"a":188},{"a":189},{"a":190},{"a":191},{"a":192},{"a":193},{"a":194},{"a":195},{"a":196},{"a":197},{"a":198},{"a":199},{"a":200},{"a":201},{"a":202},{"a":203},{"a":204},{"a":205},{"a":206},{"a":207},{"a":208},{"a":209},{"a":210},{"a":211},{"a":212},{"a":213},{"a":214},{"a":215},{"a":216},{"a":217},{"a":218},{"a":219},{"a":220},{"a":221},{"a":222},{"a":223},{"a":224},{"a":225},{"a":226},{"a":227},{"a":228},{"a":229},{"a":230},{"a":231},{"a":232},{"a":233},{"a":234},{"a":235},{"a":236},{"a":237},{"a":238},{"a":239},{"a":240},{"a":241},{"a":242},{"a":243},{"a":244},{"a":245},{"a":246},{"a":247},{"a":248},{"a":249},{"a":250},{"a":251},{"a":252},{"a":253},{"a":254},{"a":255},{"a":256},{"a":257},{"a":258},{"a":259},{"a":260},{"a":261},{"a":262},{"a":263},{"a":264},{"a":265},{"a":266},{"a":267},{"a":268},{"a":269},{"a":270},{"a":271},{"a":272},{"a":273},{"a":274},{"a":275},{"a":276},{"a":277},{"a":278},{"a":279},{"a":280},{"a":281},{"a":282},{"a":283},{"a":284},{"a":285},{"a":286},{"a":287},{"a":288},{"a":289},{"a":290},{"a":291},{"a":292},{"a":293},{"a":294},{"a":295},{"a":296},{"a":297},{"a":298},{"a":299}],"done":true} -------------------------------------------------------------------------------- /test/playground.test.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file no-explicit-any no-unused-vars require-await ban-unused-ignore no-cond-assign 2 | import 'https://gist.githubusercontent.com/qwtel/b14f0f81e3a96189f7771f83ee113f64/raw/TestRequest.ts' 3 | import { 4 | assert, 5 | assertExists, 6 | assertEquals, 7 | assertStrictEquals, 8 | assertStringIncludes, 9 | assertThrows, 10 | assertRejects, 11 | assertArrayIncludes, 12 | } from 'https://deno.land/std@0.133.0/testing/asserts.ts' 13 | const { test } = Deno; 14 | 15 | import { JSONStringifyStream } from '../json-stringify-stream.ts' 16 | import { JSONParseStream, JSONParseNexus } from '../json-parse-stream.ts' 17 | 18 | const collect = async (stream: ReadableStream) => { 19 | const collected: T[] = []; 20 | await stream.pipeTo(new WritableStream({ write(obj) { collected.push(obj) }})) 21 | return collected; 22 | } 23 | 24 | function toReadableStream(iter: Iterable) { 25 | const data = [...iter]; 26 | let v: T | undefined; 27 | return new ReadableStream({ 28 | pull(ctrl) { 29 | if (v = data.shift()) ctrl.enqueue(v); else ctrl.close(); 30 | }, 31 | }); 32 | } 33 | 34 | test('stringify stream', async () => { 35 | const expected = [ 36 | { a: 1 }, 37 | { b: 2}, 38 | { c: 3 }, 39 | 'foo', 40 | { a: { nested: { object: true }} } 41 | ]; 42 | const chunks = await collect(toReadableStream(expected) 43 | .pipeThrough(new JSONStringifyStream()) 44 | ); 45 | const actual = JSON.parse(chunks.join('')) 46 | assertEquals(actual, expected) 47 | }) 48 | 49 | test('roundtrip', async () => { 50 | const expected = [ 51 | { a: 1 }, 52 | { b: 2}, 53 | { c: 3 }, 54 | 'foo', 55 | { a: { nested: { object: true }} } 56 | ]; 57 | const body = toReadableStream(expected) 58 | .pipeThrough(new JSONStringifyStream()) 59 | .pipeThrough(new TextEncoderStream()) 60 | 61 | const actual = await collect(body.pipeThrough(new JSONParseStream())) 62 | 63 | assertEquals(expected, actual) 64 | 65 | }) 66 | 67 | test('Retrieving multiple values and collections', async () => { 68 | const jsonStream = new JSONParseNexus(); 69 | const asyncData = { 70 | type: jsonStream.promise('$.type'), 71 | items: jsonStream.stream('$.items.*'), 72 | }; 73 | 74 | const nested = { 75 | type: "foo", 76 | items: [ 77 | { "a": 1 }, 78 | { "b": 2 }, 79 | { "c": 3 }, 80 | { "zzz": 999 }, 81 | ] 82 | }; 83 | 84 | new Response(JSON.stringify(nested)).body!.pipeThrough(jsonStream) 85 | 86 | assertEquals(await asyncData.type, 'foo') 87 | 88 | // We can collect the values as before: 89 | const collected = await collect(asyncData.items) 90 | 91 | assertEquals(collected, nested.items) 92 | }) 93 | --------------------------------------------------------------------------------