├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .github └── workflows │ ├── ci.yml │ ├── npm_publish.yml │ ├── suggest_change.yml │ └── update.yml ├── .gitignore ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── deno.json ├── deno.lock ├── dev ├── build_npm.ts ├── transpile.ts └── transpile_check.ts ├── import-map.json ├── js └── mod.js ├── mod.ts ├── mod_test.ts ├── src ├── parser.ts ├── polyfill.ts ├── stringify.ts └── utils.ts └── testdata ├── concat-json.concat-json ├── json-lines.jsonl ├── json-seq.json-seq ├── nd-json.ndjson └── test.ts /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | # [Choice] Debian OS version: bullseye, buster 2 | ARG VARIANT=bullseye 3 | FROM --platform=linux/amd64 mcr.microsoft.com/devcontainers/base:0-${VARIANT} 4 | 5 | ENV DENO_INSTALL=/deno 6 | RUN mkdir -p /deno \ 7 | && curl -fsSL https://deno.land/x/install/install.sh | sh \ 8 | && chown -R vscode /deno 9 | 10 | ENV PATH=${DENO_INSTALL}/bin:${PATH} \ 11 | DENO_DIR=${DENO_INSTALL}/.cache/deno 12 | 13 | # [Optional] Uncomment this section to install additional OS packages. 14 | # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ 15 | # && apt-get -y install --no-install-recommends 16 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Deno", 3 | "build": { 4 | "dockerfile": "Dockerfile" 5 | }, 6 | 7 | // Configure tool-specific properties. 8 | "customizations": { 9 | // Configure properties specific to VS Code. 10 | "vscode": { 11 | // Set *default* container specific settings.json values on container create. 12 | "settings": { 13 | // Enables the project as a Deno project 14 | "deno.enable": true, 15 | // Enables Deno linting for the project 16 | "deno.lint": true, 17 | // Sets Deno as the default formatter for the project 18 | "editor.defaultFormatter": "denoland.vscode-deno" 19 | }, 20 | 21 | // Add the IDs of extensions you want installed when the container is created. 22 | "extensions": [ 23 | "denoland.vscode-deno" 24 | ] 25 | } 26 | }, 27 | 28 | "remoteUser": "vscode" 29 | } 30 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | Lint: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Git Checkout Module 10 | uses: actions/checkout@v2 11 | - name: Use Deno 12 | uses: denoland/setup-deno@v1 13 | with: 14 | deno-version: v1.x 15 | - name: 😉 Format 16 | run: deno task test:fmt 17 | - name: 😋 Lint 18 | run: deno task test:lint 19 | Type-Check: 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Git Checkout Module 23 | uses: actions/checkout@v2 24 | - name: Use Deno 25 | uses: denoland/setup-deno@v1 26 | with: 27 | deno-version: v1.x 28 | - name: 🧐 Type Check 29 | run: deno task check 30 | Test: 31 | runs-on: ubuntu-latest 32 | steps: 33 | - name: Git Checkout Module 34 | uses: actions/checkout@v2 35 | - name: Use Deno 36 | uses: denoland/setup-deno@v1 37 | with: 38 | deno-version: v1.x 39 | - name: 😎 Test 40 | run: deno task test:coverage 41 | - name: Create coverage report 42 | run: deno coverage ./coverage --lcov > coverage.lcov 43 | - name: Codecov 44 | uses: codecov/codecov-action@v1.5.2 45 | with: 46 | file: ./coverage.lcov 47 | fail_ci_if_error: true 48 | Bundle-Check: 49 | runs-on: ubuntu-latest 50 | steps: 51 | - name: Git Checkout Module 52 | uses: actions/checkout@v2 53 | - name: Use Deno 54 | uses: denoland/setup-deno@v1 55 | with: 56 | deno-version: v1.x 57 | - name: Bundle Check 58 | run: deno task test:bundle 59 | -------------------------------------------------------------------------------- /.github/workflows/npm_publish.yml: -------------------------------------------------------------------------------- 1 | name: publish-npm 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | build: 6 | runs-on: ubuntu-latest 7 | permissions: 8 | contents: read 9 | packages: write 10 | steps: 11 | - uses: actions/checkout@v3 12 | - name: Use Deno 13 | uses: denoland/setup-deno@v1 14 | with: 15 | deno-version: v1.x 16 | - uses: actions/setup-node@v3 17 | with: 18 | node-version: '16.x' 19 | registry-url: 'https://registry.npmjs.org' 20 | - name: npm build 21 | run: deno task build-npm 22 | - name: npm publish 23 | # run only when the tag is pushed 24 | if: startsWith(github.ref, 'refs/tags/') 25 | env: 26 | NODE_AUTH_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }} 27 | run: cd dev/npm && npm publish --access=public 28 | -------------------------------------------------------------------------------- /.github/workflows/suggest_change.yml: -------------------------------------------------------------------------------- 1 | name: Suggest-Change 2 | 3 | on: pull_request 4 | 5 | jobs: 6 | Suggest-Change: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Git Checkout Module 10 | uses: actions/checkout@v2 11 | - name: Use Deno 12 | uses: denoland/setup-deno@v1 13 | with: 14 | deno-version: v1.x 15 | - name: 😉 Format 16 | run: deno fmt 17 | - name: Create Suggestion 18 | uses: reviewdog/action-suggester@v1 19 | with: 20 | tool_name: deno fmt 21 | -------------------------------------------------------------------------------- /.github/workflows/update.yml: -------------------------------------------------------------------------------- 1 | name: deno-udd 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | schedule: 8 | - cron: "0 9 25 * *" 9 | 10 | jobs: 11 | update-deno-dependencies: 12 | permissions: write-all 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@master 16 | - uses: denoland/setup-deno@v1 17 | - name: run deno-udd 18 | id: run-deno-udd 19 | env: 20 | NO_COLOR: true 21 | run: | 22 | UDD_RESULT=`deno run -A https://deno.land/x/udd/main.ts $(find . -type f | grep -E ".*\.([mc]?(ts|js)|(ts|js)x?)$" -)` 23 | UDD_RESULT="${UDD_RESULT//'%'/'%25'}" 24 | UDD_RESULT="${UDD_RESULT//$'\n'/'%0A'}" 25 | UDD_RESULT="${UDD_RESULT//$'\r'/'%0D'}" 26 | echo "::set-output name=UDD_RESULT::$UDD_RESULT" 27 | - name: Create Pull Request 28 | uses: peter-evans/create-pull-request@v3 29 | with: 30 | commit-message: "chore(deps): update deno dependencies" 31 | title: Update Deno Dependencies 32 | body: |- 33 | Some external modules are stale. 34 |
Details
35 | 36 | ``` 37 | ${{ steps.run-deno-udd.outputs.UDD_RESULT }} 38 | ``` 39 |
40 | branch: deno-udd 41 | author: GitHub 42 | delete-branch: true 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dev/npm 2 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "deno.enable": true, 3 | "deno.unstable": true, 4 | "deno.lint": true, 5 | "editor.defaultFormatter": "denoland.vscode-deno" 6 | } 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 ayame113 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # jsonlines-web 2 | 3 | [![deno module](https://shield.deno.dev/x/jsonlines)](https://deno.land/x/jsonlines) 4 | [![deno doc](https://doc.deno.land/badge.svg)](https://doc.deno.land/https://deno.land/x/jsonlines/mod.ts) 5 | [![npm version](https://badge.fury.io/js/jsonlines-web.svg)](https://badge.fury.io/js/jsonlines-web) 6 | [![ci](https://github.com/ayame113/jsonlines/actions/workflows/ci.yml/badge.svg)](https://github.com/ayame113/jsonlines/actions) 7 | [![codecov](https://codecov.io/gh/ayame113/jsonlines/branch/main/graph/badge.svg?token=GsQ5af4QLn)](https://codecov.io/gh/ayame113/jsonlines) 8 | ![GitHub Sponsors](https://img.shields.io/github/sponsors/ayame113) 9 | 10 | Web stream based jsonlines decoder/encoder 11 | 12 | - ✅Deno 13 | - ✅browser 14 | - ✅Node.js 15 | 16 | This library supports JSON in the following formats: 17 | 18 | - Line-delimited JSON (JSONLinesParseStream) 19 | - NDJSON 20 | - JSON lines 21 | - Record separator-delimited JSON (JSONLinesParseStream) 22 | - Concatenated JSON (ConcatenatedJSONParseStream) 23 | 24 | See [wikipedia](https://en.wikipedia.org/wiki/JSON_streaming) for the 25 | specifications of each JSON. 26 | 27 | ## install or import 28 | 29 | ### Deno 30 | 31 | https://deno.land/x/jsonlines/ 32 | https://doc.deno.land/https://deno.land/x/jsonlines/mod.ts 33 | 34 | ```ts 35 | import { 36 | ConcatenatedJSONParseStream, 37 | ConcatenatedJSONStringifyStream, 38 | JSONLinesParseStream, 39 | JSONLinesStringifyStream, 40 | } from "https://deno.land/x/jsonlines@v1.2.1/mod.ts"; 41 | ``` 42 | 43 | ### browser 44 | 45 | ```ts 46 | import { 47 | ConcatenatedJSONParseStream, 48 | ConcatenatedJSONStringifyStream, 49 | JSONLinesParseStream, 50 | JSONLinesStringifyStream, 51 | } from "https://deno.land/x/jsonlines@v1.2.1/js/mod.js"; 52 | ``` 53 | 54 | ### Node.js 55 | 56 | https://www.npmjs.com/package/jsonlines-web 57 | 58 | ```shell 59 | npm install jsonlines-web 60 | ``` 61 | 62 | ```ts, ignore 63 | import { 64 | ConcatenatedJSONParseStream, 65 | ConcatenatedJSONStringifyStream, 66 | JSONLinesParseStream, 67 | JSONLinesStringifyStream, 68 | } from "jsonlines-web"; 69 | // if you need 70 | // import { TextDecoderStream, TextEncoderStream } from "node:stream/web"; 71 | // import { fetch } from "undici"; 72 | ``` 73 | 74 | ## Usage 75 | 76 | A working example can be found at [./testdata/test.ts](./testdata/test.ts). 77 | 78 | ### How to parse JSON Lines 79 | 80 | ./json-lines.jsonl 81 | 82 | ```json 83 | {"some":"thing"} 84 | {"foo":17,"bar":false,"quux":true} 85 | {"may":{"include":"nested","objects":["and","arrays"]}} 86 | ``` 87 | 88 | ```ts 89 | import { JSONLinesParseStream } from "https://deno.land/x/jsonlines@v1.2.1/mod.ts"; 90 | 91 | const { body } = await fetch( 92 | "https://deno.land/x/jsonlines@v1.2.1/testdata/json-lines.jsonl", 93 | ); 94 | 95 | const readable = body! 96 | .pipeThrough(new TextDecoderStream()) 97 | .pipeThrough(new JSONLinesParseStream()); 98 | 99 | for await (const data of readable) { 100 | console.log(data); 101 | } 102 | ``` 103 | 104 | ### How to parse json-seq 105 | 106 | ./json-seq.json-seq 107 | 108 | ```json 109 | {"some":"thing\n"} 110 | { 111 | "may": { 112 | "include": "nested", 113 | "objects": [ 114 | "and", 115 | "arrays" 116 | ] 117 | } 118 | } 119 | ``` 120 | 121 | ```ts 122 | import { JSONLinesParseStream } from "https://deno.land/x/jsonlines@v1.2.1/mod.ts"; 123 | 124 | const { body } = await fetch( 125 | "https://deno.land/x/jsonlines@v1.2.1/testdata/json-seq.json-seq", 126 | ); 127 | 128 | const recordSeparator = "\x1E"; 129 | const readable = body! 130 | .pipeThrough(new TextDecoderStream()) 131 | .pipeThrough(new JSONLinesParseStream({ separator: recordSeparator })); 132 | 133 | for await (const data of readable) { 134 | console.log(data); 135 | } 136 | ``` 137 | 138 | ### How to parse concat-json 139 | 140 | ./concat-json.concat-json 141 | 142 | ```json 143 | {"foo":"bar"}{"qux":"corge"}{"baz":{"waldo":"thud"}} 144 | ``` 145 | 146 | ```ts 147 | import { ConcatenatedJSONParseStream } from "https://deno.land/x/jsonlines@v1.2.1/mod.ts"; 148 | 149 | const { body } = await fetch( 150 | "https://deno.land/x/jsonlines@v1.2.1/testdata/concat-json.concat-json", 151 | ); 152 | 153 | const readable = body! 154 | .pipeThrough(new TextDecoderStream()) 155 | .pipeThrough(new ConcatenatedJSONParseStream()); 156 | 157 | for await (const data of readable) { 158 | console.log(data); 159 | } 160 | ``` 161 | 162 | ### How to stringify JSON Lines 163 | 164 | ```ts 165 | import { readableStreamFromIterable } from "https://deno.land/std@0.138.0/streams/mod.ts"; 166 | import { JSONLinesStringifyStream } from "https://deno.land/x/jsonlines@v1.2.1/mod.ts"; 167 | 168 | const file = await Deno.open(new URL("./tmp.concat-json", import.meta.url), { 169 | create: true, 170 | write: true, 171 | }); 172 | 173 | readableStreamFromIterable([{ foo: "bar" }, { baz: 100 }]) 174 | .pipeThrough(new JSONLinesStringifyStream()) 175 | .pipeThrough(new TextEncoderStream()) 176 | .pipeTo(file.writable) 177 | .then(() => console.log("write success")); 178 | ``` 179 | 180 | ### How to stringify json-seq 181 | 182 | ```ts 183 | import { readableStreamFromIterable } from "https://deno.land/std@0.138.0/streams/mod.ts"; 184 | import { JSONLinesStringifyStream } from "https://deno.land/x/jsonlines@v1.2.1/mod.ts"; 185 | 186 | const recordSeparator = "\x1E"; 187 | const file = await Deno.open(new URL("./tmp.concat-json", import.meta.url), { 188 | create: true, 189 | write: true, 190 | }); 191 | 192 | readableStreamFromIterable([{ foo: "bar" }, { baz: 100 }]) 193 | .pipeThrough(new JSONLinesStringifyStream({ separator: recordSeparator })) 194 | .pipeThrough(new TextEncoderStream()) 195 | .pipeTo(file.writable) 196 | .then(() => console.log("write success")); 197 | ``` 198 | 199 | ### How to stringify concat-json 200 | 201 | ```ts 202 | import { readableStreamFromIterable } from "https://deno.land/std@0.138.0/streams/mod.ts"; 203 | import { ConcatenatedJSONStringifyStream } from "https://deno.land/x/jsonlines@v1.2.1/mod.ts"; 204 | 205 | const file = await Deno.open(new URL("./tmp.concat-json", import.meta.url), { 206 | create: true, 207 | write: true, 208 | }); 209 | 210 | readableStreamFromIterable([{ foo: "bar" }, { baz: 100 }]) 211 | .pipeThrough(new ConcatenatedJSONStringifyStream()) 212 | .pipeThrough(new TextEncoderStream()) 213 | .pipeTo(file.writable) 214 | .then(() => console.log("write success")); 215 | ``` 216 | 217 | ## note 218 | 219 | This library contains 220 | [ReadableStream.prototype[Symbol.asyncIterator]](https://github.com/whatwg/streams/issues/778) 221 | polyfills. Importing this library will automatically enable 222 | ReadableStream.prototype[Symbol.asyncIterator]. 223 | 224 | ## develop 225 | 226 | need to manually `deno task transpile` before release. 227 | -------------------------------------------------------------------------------- /deno.json: -------------------------------------------------------------------------------- 1 | { 2 | "tasks": { 3 | "test": "deno test --allow-read=. --doc --ignore=./js/mod.js,./dev/npm/", 4 | "test:coverage": "deno test --allow-read=. --doc --ignore=./js/mod.js,./dev/npm/ --coverage=coverage", 5 | "test:lint": "deno lint", 6 | "test:fmt": "deno fmt --check", 7 | "fmt": "deno fmt", 8 | "check": "deno check ./mod.ts", 9 | "bundle": "deno run --allow-run --allow-read --unstable ./dev/transpile.ts", 10 | "test:bundle": "deno run --allow-run --allow-read --unstable ./dev/transpile_check.ts", 11 | "build-npm": "deno run --allow-read --allow-write --allow-net --allow-env --allow-run ./dev/build_npm.ts" 12 | }, 13 | "lint": { 14 | "files": { 15 | "exclude": ["./js/"] 16 | } 17 | }, 18 | "fmt": { 19 | "files": { 20 | "exclude": ["./js/"] 21 | } 22 | }, 23 | "importMap": "./import-map.json", 24 | "lock": false 25 | } 26 | -------------------------------------------------------------------------------- /deno.lock: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2", 3 | "remote": { 4 | "https://deno.land/std@0.111.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58", 5 | "https://deno.land/std@0.111.0/_util/os.ts": "dfb186cc4e968c770ab6cc3288bd65f4871be03b93beecae57d657232ecffcac", 6 | "https://deno.land/std@0.111.0/bytes/bytes_list.ts": "3bff6a09c72b2e0b1e92e29bd3b135053894196cca07a2bba842901073efe5cb", 7 | "https://deno.land/std@0.111.0/bytes/equals.ts": "69f55fdbd45c71f920c1a621e6c0865dc780cd8ae34e0f5e55a9497b70c31c1b", 8 | "https://deno.land/std@0.111.0/bytes/mod.ts": "fedb80b8da2e7ad8dd251148e65f92a04c73d6c5a430b7d197dc39588c8dda6f", 9 | "https://deno.land/std@0.111.0/fmt/colors.ts": "8368ddf2d48dfe413ffd04cdbb7ae6a1009cf0dccc9c7ff1d76259d9c61a0621", 10 | "https://deno.land/std@0.111.0/fs/_util.ts": "f2ce811350236ea8c28450ed822a5f42a0892316515b1cd61321dec13569c56b", 11 | "https://deno.land/std@0.111.0/fs/ensure_dir.ts": "b7c103dc41a3d1dbbb522bf183c519c37065fdc234831a4a0f7d671b1ed5fea7", 12 | "https://deno.land/std@0.111.0/hash/sha256.ts": "bd85257c68d1fdd9da8457284c4fbb04efa9f4f2229b5f41a638d5b71a3a8d5c", 13 | "https://deno.land/std@0.111.0/io/buffer.ts": "fdf93ba9e5d20ff3369e2c42443efd89131f8a73066f7f59c033cc588a0e2cfe", 14 | "https://deno.land/std@0.111.0/io/types.d.ts": "89a27569399d380246ca7cdd9e14d5e68459f11fb6110790cc5ecbd4ee7f3215", 15 | "https://deno.land/std@0.111.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853", 16 | "https://deno.land/std@0.111.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4", 17 | "https://deno.land/std@0.111.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b", 18 | "https://deno.land/std@0.111.0/path/common.ts": "f41a38a0719a1e85aa11c6ba3bea5e37c15dd009d705bd8873f94c833568cbc4", 19 | "https://deno.land/std@0.111.0/path/glob.ts": "ea87985765b977cc284b92771003b2070c440e0807c90e1eb0ff3e095911a820", 20 | "https://deno.land/std@0.111.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12", 21 | "https://deno.land/std@0.111.0/path/posix.ts": "34349174b9cd121625a2810837a82dd8b986bbaaad5ade690d1de75bbb4555b2", 22 | "https://deno.land/std@0.111.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c", 23 | "https://deno.land/std@0.111.0/path/win32.ts": "11549e8c6df8307a8efcfa47ad7b2a75da743eac7d4c89c9723a944661c8bd2e", 24 | "https://deno.land/std@0.111.0/streams/conversion.ts": "fe0059ed9d3c53eda4ba44eb71a6a9acb98c5fdb5ba1b6c6ab28004724c7641b", 25 | "https://deno.land/std@0.140.0/_util/assert.ts": "e94f2eb37cebd7f199952e242c77654e43333c1ac4c5c700e929ea3aa5489f74", 26 | "https://deno.land/std@0.140.0/_util/os.ts": "3b4c6e27febd119d36a416d7a97bd3b0251b77c88942c8f16ee5953ea13e2e49", 27 | "https://deno.land/std@0.140.0/fs/_util.ts": "0fb24eb4bfebc2c194fb1afdb42b9c3dda12e368f43e8f2321f84fc77d42cb0f", 28 | "https://deno.land/std@0.140.0/fs/ensure_dir.ts": "9dc109c27df4098b9fc12d949612ae5c9c7169507660dcf9ad90631833209d9d", 29 | "https://deno.land/std@0.140.0/fs/expand_glob.ts": "0c10130d67c9b02164b03df8e43c6d6defbf8e395cb69d09e84a8586e6d72ac3", 30 | "https://deno.land/std@0.140.0/fs/walk.ts": "117403ccd21fd322febe56ba06053b1ad5064c802170f19b1ea43214088fe95f", 31 | "https://deno.land/std@0.140.0/path/_constants.ts": "df1db3ffa6dd6d1252cc9617e5d72165cd2483df90e93833e13580687b6083c3", 32 | "https://deno.land/std@0.140.0/path/_interface.ts": "ee3b431a336b80cf445441109d089b70d87d5e248f4f90ff906820889ecf8d09", 33 | "https://deno.land/std@0.140.0/path/_util.ts": "c1e9686d0164e29f7d880b2158971d805b6e0efc3110d0b3e24e4b8af2190d2b", 34 | "https://deno.land/std@0.140.0/path/common.ts": "bee563630abd2d97f99d83c96c2fa0cca7cee103e8cb4e7699ec4d5db7bd2633", 35 | "https://deno.land/std@0.140.0/path/glob.ts": "cb5255638de1048973c3e69e420c77dc04f75755524cb3b2e160fe9277d939ee", 36 | "https://deno.land/std@0.140.0/path/mod.ts": "d3e68d0abb393fb0bf94a6d07c46ec31dc755b544b13144dee931d8d5f06a52d", 37 | "https://deno.land/std@0.140.0/path/posix.ts": "293cdaec3ecccec0a9cc2b534302dfe308adb6f10861fa183275d6695faace44", 38 | "https://deno.land/std@0.140.0/path/separator.ts": "fe1816cb765a8068afb3e8f13ad272351c85cbc739af56dacfc7d93d710fe0f9", 39 | "https://deno.land/std@0.140.0/path/win32.ts": "31811536855e19ba37a999cd8d1b62078235548d67902ece4aa6b814596dd757", 40 | "https://deno.land/std@0.143.0/_util/assert.ts": "e94f2eb37cebd7f199952e242c77654e43333c1ac4c5c700e929ea3aa5489f74", 41 | "https://deno.land/std@0.143.0/_util/os.ts": "3b4c6e27febd119d36a416d7a97bd3b0251b77c88942c8f16ee5953ea13e2e49", 42 | "https://deno.land/std@0.143.0/path/_constants.ts": "df1db3ffa6dd6d1252cc9617e5d72165cd2483df90e93833e13580687b6083c3", 43 | "https://deno.land/std@0.143.0/path/_interface.ts": "ee3b431a336b80cf445441109d089b70d87d5e248f4f90ff906820889ecf8d09", 44 | "https://deno.land/std@0.143.0/path/_util.ts": "c1e9686d0164e29f7d880b2158971d805b6e0efc3110d0b3e24e4b8af2190d2b", 45 | "https://deno.land/std@0.143.0/path/common.ts": "bee563630abd2d97f99d83c96c2fa0cca7cee103e8cb4e7699ec4d5db7bd2633", 46 | "https://deno.land/std@0.143.0/path/glob.ts": "cb5255638de1048973c3e69e420c77dc04f75755524cb3b2e160fe9277d939ee", 47 | "https://deno.land/std@0.143.0/path/mod.ts": "d3e68d0abb393fb0bf94a6d07c46ec31dc755b544b13144dee931d8d5f06a52d", 48 | "https://deno.land/std@0.143.0/path/posix.ts": "293cdaec3ecccec0a9cc2b534302dfe308adb6f10861fa183275d6695faace44", 49 | "https://deno.land/std@0.143.0/path/separator.ts": "fe1816cb765a8068afb3e8f13ad272351c85cbc739af56dacfc7d93d710fe0f9", 50 | "https://deno.land/std@0.143.0/path/win32.ts": "31811536855e19ba37a999cd8d1b62078235548d67902ece4aa6b814596dd757", 51 | "https://deno.land/std@0.171.0/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", 52 | "https://deno.land/std@0.171.0/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", 53 | "https://deno.land/std@0.171.0/fmt/colors.ts": "938c5d44d889fb82eff6c358bea8baa7e85950a16c9f6dae3ec3a7a729164471", 54 | "https://deno.land/std@0.171.0/fs/_util.ts": "65381f341af1ff7f40198cee15c20f59951ac26e51ddc651c5293e24f9ce6f32", 55 | "https://deno.land/std@0.171.0/fs/empty_dir.ts": "c3d2da4c7352fab1cf144a1ecfef58090769e8af633678e0f3fabaef98594688", 56 | "https://deno.land/std@0.171.0/fs/expand_glob.ts": "536055845aafc32de7e7a46c3b778a741825d5e2ed8580d9851a01ec7a5adf2e", 57 | "https://deno.land/std@0.171.0/fs/walk.ts": "ea95ffa6500c1eda6b365be488c056edc7c883a1db41ef46ec3bf057b1c0fe32", 58 | "https://deno.land/std@0.171.0/path/_constants.ts": "e49961f6f4f48039c0dfed3c3f93e963ca3d92791c9d478ac5b43183413136e0", 59 | "https://deno.land/std@0.171.0/path/_interface.ts": "6471159dfbbc357e03882c2266d21ef9afdb1e4aa771b0545e90db58a0ba314b", 60 | "https://deno.land/std@0.171.0/path/_util.ts": "86c2375a996c1931b2f2ac71fefd5ddf0cf0e579fa4ab12d3e4c552d4223b8d8", 61 | "https://deno.land/std@0.171.0/path/common.ts": "ee7505ab01fd22de3963b64e46cff31f40de34f9f8de1fff6a1bd2fe79380000", 62 | "https://deno.land/std@0.171.0/path/glob.ts": "d479e0a695621c94d3fd7fe7abd4f9499caf32a8de13f25073451c6ef420a4e1", 63 | "https://deno.land/std@0.171.0/path/mod.ts": "4b83694ac500d7d31b0cdafc927080a53dc0c3027eb2895790fb155082b0d232", 64 | "https://deno.land/std@0.171.0/path/posix.ts": "2ecc259e6f34013889b7638ff90339a82d8178f629155761ce6001e41af55a43", 65 | "https://deno.land/std@0.171.0/path/separator.ts": "0fb679739d0d1d7bf45b68dacfb4ec7563597a902edbaf3c59b50d5bcadd93b1", 66 | "https://deno.land/std@0.171.0/path/win32.ts": "99170a0eb0e2b1ce41499c1e86bb55320cb6606299ad947f57ee0a291cdb93d5", 67 | "https://deno.land/std@0.173.0/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", 68 | "https://deno.land/std@0.173.0/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", 69 | "https://deno.land/std@0.173.0/fmt/colors.ts": "938c5d44d889fb82eff6c358bea8baa7e85950a16c9f6dae3ec3a7a729164471", 70 | "https://deno.land/std@0.173.0/path/_constants.ts": "e49961f6f4f48039c0dfed3c3f93e963ca3d92791c9d478ac5b43183413136e0", 71 | "https://deno.land/std@0.173.0/path/_interface.ts": "6471159dfbbc357e03882c2266d21ef9afdb1e4aa771b0545e90db58a0ba314b", 72 | "https://deno.land/std@0.173.0/path/_util.ts": "86c2375a996c1931b2f2ac71fefd5ddf0cf0e579fa4ab12d3e4c552d4223b8d8", 73 | "https://deno.land/std@0.173.0/path/common.ts": "ee7505ab01fd22de3963b64e46cff31f40de34f9f8de1fff6a1bd2fe79380000", 74 | "https://deno.land/std@0.173.0/path/glob.ts": "d479e0a695621c94d3fd7fe7abd4f9499caf32a8de13f25073451c6ef420a4e1", 75 | "https://deno.land/std@0.173.0/path/mod.ts": "4b83694ac500d7d31b0cdafc927080a53dc0c3027eb2895790fb155082b0d232", 76 | "https://deno.land/std@0.173.0/path/posix.ts": "0874b341c2c6968ca38d323338b8b295ea1dae10fa872a768d812e2e7d634789", 77 | "https://deno.land/std@0.173.0/path/separator.ts": "0fb679739d0d1d7bf45b68dacfb4ec7563597a902edbaf3c59b50d5bcadd93b1", 78 | "https://deno.land/std@0.173.0/path/win32.ts": "672942919dd66ce1b8c224e77d3447e2ad8254eaff13fe6946420a9f78fa141e", 79 | "https://deno.land/std@0.173.0/streams/readable_stream_from_iterable.ts": "cae337ddafd2abc5e3df699ef2af888ac04091f12732ae658448fba2c7b187e8", 80 | "https://deno.land/std@0.173.0/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea", 81 | "https://deno.land/std@0.173.0/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7", 82 | "https://deno.land/std@0.173.0/testing/asserts.ts": "984ab0bfb3faeed92ffaa3a6b06536c66811185328c5dd146257c702c41b01ab", 83 | "https://deno.land/x/code_block_writer@11.0.3/mod.ts": "2c3448060e47c9d08604c8f40dee34343f553f33edcdfebbf648442be33205e5", 84 | "https://deno.land/x/code_block_writer@11.0.3/utils/string_utils.ts": "60cb4ec8bd335bf241ef785ccec51e809d576ff8e8d29da43d2273b69ce2a6ff", 85 | "https://deno.land/x/deno_cache@0.2.1/auth_tokens.ts": "01b94d25abd974153a3111653998b9a43c66d84a0e4b362fc5f4bbbf40a6e0f7", 86 | "https://deno.land/x/deno_cache@0.2.1/cache.ts": "67e301c20161546fea45405316314f4c3d85cc7a367b2fb72042903f308f55b7", 87 | "https://deno.land/x/deno_cache@0.2.1/deno_dir.ts": "e4dc68da5641aa337bcc06fb1df28fcb086b366dcbea7d8aaed7ac7c853fedb1", 88 | "https://deno.land/x/deno_cache@0.2.1/deps.ts": "2ebaba0ad86fff8b9027c6afd4c3909a17cd8bf8c9e263151c980c15c56a18ee", 89 | "https://deno.land/x/deno_cache@0.2.1/dirs.ts": "e07003fabed7112375d4a50040297aae768f9d06bb6c2655ca46880653b576b4", 90 | "https://deno.land/x/deno_cache@0.2.1/disk_cache.ts": "d7a361f0683a032bcca28513a7bbedc28c77cfcc6719e6f6cea156c0ff1108df", 91 | "https://deno.land/x/deno_cache@0.2.1/file_fetcher.ts": "352702994c190c45215f3b8086621e117e88bc2174b020faefb5eca653d71d6a", 92 | "https://deno.land/x/deno_cache@0.2.1/http_cache.ts": "af1500149496e2d0acadec24569e2a9c86a3f600cceef045dcf6f5ce8de72b3a", 93 | "https://deno.land/x/deno_cache@0.2.1/mod.ts": "709ab9d1068be5fd77b020b33e7a9394f1e9b453553b1e2336b72c90283cf3c0", 94 | "https://deno.land/x/deno_cache@0.2.1/util.ts": "652479928551259731686686ff2df6f26bc04e8e4d311137b2bf3bc10f779f48", 95 | "https://deno.land/x/deno_graph@0.6.0/lib/deno_graph.generated.js": "3e1cccd6376d4ad0ea789d66aa0f6b19f737fa8da37b5e6185ef5c269c974f54", 96 | "https://deno.land/x/deno_graph@0.6.0/lib/loader.ts": "13a11c1dea0d85e0ad211be77217b8c06138bbb916afef6f50a04cca415084a9", 97 | "https://deno.land/x/deno_graph@0.6.0/lib/media_type.ts": "36be751aa63d6ae36475b90dca5fae8fd7c3a77cf13684c48cf23a85ee607b31", 98 | "https://deno.land/x/deno_graph@0.6.0/lib/snippets/deno_graph-1c138d6136337537/src/deno_apis.js": "f13f2678d875372cf8489ceb7124623a39fa5bf8de8ee1ec722dbb2ec5ec7845", 99 | "https://deno.land/x/deno_graph@0.6.0/lib/types.d.ts": "68cb232e02a984658b40ffaf6cafb979a06fbfdce7f5bd4c7a83ed1a32a07687", 100 | "https://deno.land/x/deno_graph@0.6.0/mod.ts": "8fe3d39bdcb273adfb41a0bafbbaabec4c6fe6c611b47fed8f46f218edb37e8e", 101 | "https://deno.land/x/dnt@0.33.0/lib/compiler.ts": "dd589db479d6d7e69999865003ab83c41544e251ece4f21f2f2ee74557097ba6", 102 | "https://deno.land/x/dnt@0.33.0/lib/compiler_transforms.ts": "cbb1fd5948f5ced1aa5c5aed9e45134e2357ce1e7220924c1d7bded30dcd0dd0", 103 | "https://deno.land/x/dnt@0.33.0/lib/mod.deps.ts": "6648fb17b4a49677cb0c24f60ffb5067a86ad69ff97712d40fe0d62b281b1811", 104 | "https://deno.land/x/dnt@0.33.0/lib/npm_ignore.ts": "ddc1a7a76b288ca471bf1a6298527887a0f9eb7e25008072fd9c9fa9bb28c71a", 105 | "https://deno.land/x/dnt@0.33.0/lib/package_json.ts": "2d629dbaef8004971e38ce3661f04b915a35342b905c3d98ff4a25343c2a8293", 106 | "https://deno.land/x/dnt@0.33.0/lib/pkg/dnt_wasm.generated.js": "00257fc2f03321bb5f2b9bc23cb85e79fe55eb49a325d5ab925b9fc81b4aa963", 107 | "https://deno.land/x/dnt@0.33.0/lib/pkg/snippets/dnt-wasm-a15ef721fa5290c5/helpers.js": "2f623f83602d4fbb30caa63444b10e35b45e9c2b267e49585ec9bb790a4888d8", 108 | "https://deno.land/x/dnt@0.33.0/lib/shims.ts": "7998851b149cb230f5183590d3b66c06f696fefb1c31c24eb5736f1ef12a4de1", 109 | "https://deno.land/x/dnt@0.33.0/lib/test_runner/get_test_runner_code.ts": "2a4e26aa33120f3cc9e03b8538211a5047a4bad4c64e895944b87f2dcd55d904", 110 | "https://deno.land/x/dnt@0.33.0/lib/test_runner/test_runner.ts": "b91d77d9d4b82984cb2ba7431ba6935756ba72f62e7dd4db22cd47a680ebd952", 111 | "https://deno.land/x/dnt@0.33.0/lib/transform.deps.ts": "28cee4e09a7c6baf156d363cdeb408088d00e7cb7755946202cb62b47ed223ec", 112 | "https://deno.land/x/dnt@0.33.0/lib/types.ts": "34e45a3136c2f21f797173ea46d9ea5d1639eb7b834a5bd565aad4214fa32603", 113 | "https://deno.land/x/dnt@0.33.0/lib/utils.ts": "5b7bba18f4d4d91b79343216c93a7b21d9dd6019e4d3a7b25ba043d44a1fc237", 114 | "https://deno.land/x/dnt@0.33.0/mod.ts": "691ea4b644cc61123b7beed19e66af301f25985483b81d21cfe49a0be2877fd9", 115 | "https://deno.land/x/dnt@0.33.0/transform.ts": "5960cf0b84d5bfae3ca61569a344a467f448d8e96ab1eceee8cb181698fb6c65", 116 | "https://deno.land/x/ts_morph@17.0.1/bootstrap/mod.ts": "b53aad517f106c4079971fcd4a81ab79fadc40b50061a3ab2b741a09119d51e9", 117 | "https://deno.land/x/ts_morph@17.0.1/bootstrap/ts_morph_bootstrap.d.ts": "607e651c5ae5aa57c2ac4090759a6379e809c0cdc42114742ac67353b1a75038", 118 | "https://deno.land/x/ts_morph@17.0.1/bootstrap/ts_morph_bootstrap.js": "91a954daa993c5acb3361aa5279394f81ea6fe18b3854345c86111b336491cfc", 119 | "https://deno.land/x/ts_morph@17.0.1/common/DenoRuntime.ts": "537800e840d0994f9055164e11bf33eadf96419246af0d3c453793c3ae67bdb3", 120 | "https://deno.land/x/ts_morph@17.0.1/common/mod.ts": "01985d2ee7da8d1caee318a9d07664774fbee4e31602bc2bb6bb62c3489555ed", 121 | "https://deno.land/x/ts_morph@17.0.1/common/ts_morph_common.d.ts": "ee7767b0c68b23c65bb607c94b6cb3512e8237fbcb7d1d8383a33235cde2c068", 122 | "https://deno.land/x/ts_morph@17.0.1/common/ts_morph_common.js": "49a79124b941ba2b35d81ac9eb90fc33c957b2640cdb97569c1941bac5a3bbdb", 123 | "https://deno.land/x/ts_morph@17.0.1/common/typescript.d.ts": "57e52a0882af4e835473dda27e4316cc31149866970210f9f79b940e916b7838", 124 | "https://deno.land/x/ts_morph@17.0.1/common/typescript.js": "5dd669eb199ee2a539924c63a92e23d95df43dfe2fbe3a9d68c871648be1ad5e" 125 | } 126 | } -------------------------------------------------------------------------------- /dev/build_npm.ts: -------------------------------------------------------------------------------- 1 | import { fromFileUrl } from "https://deno.land/std@0.185.0/path/mod.ts"; 2 | import { build, emptyDir } from "https://deno.land/x/dnt@0.34.0/mod.ts"; 3 | 4 | const outDir = fromFileUrl(new URL("./npm/", import.meta.url)); 5 | const projectRootDir = fromFileUrl(new URL("../", import.meta.url)); 6 | 7 | await emptyDir(outDir); 8 | 9 | await build({ 10 | entryPoints: [`${projectRootDir}/mod.ts`], 11 | outDir, 12 | shims: { 13 | // see JS docs for overview and more options 14 | deno: true, 15 | custom: [{ 16 | package: { 17 | name: "stream/web", 18 | }, 19 | globalNames: [ 20 | "WritableStream", 21 | "ReadableStream", 22 | "TransformStream", 23 | { 24 | name: "QueuingStrategy", 25 | typeOnly: true, 26 | }, 27 | { 28 | name: "WritableStreamDefaultWriter", 29 | typeOnly: true, 30 | }, 31 | { 32 | name: "ReadableStreamDefaultReader", 33 | typeOnly: true, 34 | }, 35 | { 36 | name: "TransformStreamDefaultController", 37 | typeOnly: true, 38 | }, 39 | ], 40 | }, { 41 | module: "util", 42 | globalNames: [ 43 | "TextEncoder", 44 | "TextDecoder", 45 | ], 46 | }], 47 | }, 48 | package: { 49 | // package.json properties 50 | name: "jsonlines-web", 51 | version: "v1.2.1", 52 | description: "Web stream based jsonlines decoder/encoder.", 53 | license: "MIT", 54 | repository: { 55 | type: "git", 56 | url: "git+https://github.com/ayame113/jsonlines.git", 57 | }, 58 | bugs: { 59 | url: "https://github.com/ayame113/jsonlines/issues", 60 | }, 61 | }, 62 | rootTestDir: projectRootDir, 63 | testPattern: "mod_test.ts", 64 | }); 65 | 66 | // post build steps 67 | Deno.copyFileSync(`${projectRootDir}/LICENSE`, `${outDir}/LICENSE`); 68 | Deno.copyFileSync(`${projectRootDir}/README.md`, `${outDir}/README.md`); 69 | -------------------------------------------------------------------------------- /dev/transpile.ts: -------------------------------------------------------------------------------- 1 | import { fromFileUrl } from "https://deno.land/std@0.185.0/path/mod.ts"; 2 | 3 | const input = fromFileUrl(new URL("../mod.ts", import.meta.url)); 4 | const output = fromFileUrl(new URL("../js/mod.js", import.meta.url)); 5 | const command = new Deno.Command(Deno.execPath(), { 6 | args: ["bundle", input, output], 7 | stdout: "inherit", 8 | stderr: "inherit", 9 | }); 10 | const { success: bundleSuccess } = await command.output(); 11 | if (!bundleSuccess) { 12 | throw new Error("deno bundle: failed"); 13 | } 14 | 15 | const importMap = { 16 | "imports": { 17 | [`${new URL("../mod.ts", import.meta.url)}`]: `${new URL( 18 | "../js/mod.js", 19 | import.meta.url, 20 | )}`, 21 | }, 22 | }; 23 | const testFile = fromFileUrl(new URL("../mod_test.ts", import.meta.url)); 24 | const command2 = new Deno.Command(Deno.execPath(), { 25 | args: [ 26 | "test", 27 | testFile, 28 | `--import-map=data:application/json,${JSON.stringify(importMap)}`, 29 | "--no-check", 30 | ], 31 | stdout: "inherit", 32 | stderr: "inherit", 33 | }); 34 | const { success: testSuccess } = await command2.output(); 35 | if (!testSuccess) { 36 | throw new Error("deno test: failed"); 37 | } 38 | -------------------------------------------------------------------------------- /dev/transpile_check.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "https://deno.land/std@0.185.0/testing/asserts.ts"; 2 | import { fromFileUrl } from "https://deno.land/std@0.185.0/path/mod.ts"; 3 | 4 | const input = fromFileUrl(new URL("../mod.ts", import.meta.url)); 5 | const output = fromFileUrl(new URL("../js/mod.js", import.meta.url)); 6 | const command = new Deno.Command(Deno.execPath(), { 7 | args: ["bundle", input], 8 | stdout: "piped", 9 | stderr: "inherit", 10 | }); 11 | const { success: bundleSuccess, stdout } = await command.output(); 12 | 13 | if (!bundleSuccess) { 14 | throw new Error("deno bundle: failed"); 15 | } 16 | 17 | const actual = (await Deno.readTextFile(output)).trim(); 18 | const expected = new TextDecoder().decode(stdout).trim(); 19 | 20 | assertEquals(actual, expected, "please run `deno task bundle` before commit"); 21 | -------------------------------------------------------------------------------- /import-map.json: -------------------------------------------------------------------------------- 1 | { 2 | "imports": { 3 | "https://deno.land/x/jsonlines@v1.2.1/": "./" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /js/mod.js: -------------------------------------------------------------------------------- 1 | // deno-fmt-ignore-file 2 | // deno-lint-ignore-file 3 | // This code was bundled using `deno bundle` and it's not recommended to edit it manually 4 | 5 | if (typeof ReadableStream.prototype[Symbol.asyncIterator] !== "function") { 6 | Object.defineProperty(ReadableStream.prototype, Symbol.asyncIterator, { 7 | async *value () { 8 | const reader = this.getReader(); 9 | try { 10 | while(true){ 11 | const { done , value } = await reader.read(); 12 | if (done) return; 13 | yield value; 14 | } 15 | } finally{ 16 | reader.releaseLock(); 17 | } 18 | }, 19 | writable: true, 20 | enumerable: false, 21 | configurable: true 22 | }); 23 | } 24 | function transformStreamFromGeneratorFunction(transformer, writableStrategy, readableStrategy) { 25 | const { writable , readable } = new TransformStream(undefined, writableStrategy); 26 | const iterable = transformer(readable); 27 | const iterator = iterable[Symbol.asyncIterator]?.() ?? iterable[Symbol.iterator]?.(); 28 | return { 29 | writable, 30 | readable: new ReadableStream({ 31 | async pull (controller) { 32 | const { done , value } = await iterator.next(); 33 | if (done) { 34 | controller.close(); 35 | return; 36 | } 37 | controller.enqueue(value); 38 | }, 39 | async cancel (...args) { 40 | await readable.cancel(...args); 41 | } 42 | }, readableStrategy) 43 | }; 44 | } 45 | function createLPS(pat) { 46 | const lps = new Uint8Array(pat.length); 47 | lps[0] = 0; 48 | let prefixEnd = 0; 49 | let i = 1; 50 | while(i < lps.length){ 51 | if (pat[i] == pat[prefixEnd]) { 52 | prefixEnd++; 53 | lps[i] = prefixEnd; 54 | i++; 55 | } else if (prefixEnd === 0) { 56 | lps[i] = 0; 57 | i++; 58 | } else { 59 | prefixEnd = lps[prefixEnd - 1]; 60 | } 61 | } 62 | return lps; 63 | } 64 | class TextDelimiterStream extends TransformStream { 65 | #buf = ""; 66 | #delimiter; 67 | #inspectIndex = 0; 68 | #matchIndex = 0; 69 | #delimLPS; 70 | #disp; 71 | constructor(delimiter, options){ 72 | super({ 73 | transform: (chunk, controller)=>{ 74 | this.#handle(chunk, controller); 75 | }, 76 | flush: (controller)=>{ 77 | controller.enqueue(this.#buf); 78 | } 79 | }); 80 | this.#delimiter = delimiter; 81 | this.#delimLPS = createLPS(new TextEncoder().encode(delimiter)); 82 | this.#disp = options?.disposition ?? "discard"; 83 | } 84 | #handle(chunk, controller) { 85 | this.#buf += chunk; 86 | let localIndex = 0; 87 | while(this.#inspectIndex < this.#buf.length){ 88 | if (chunk[localIndex] === this.#delimiter[this.#matchIndex]) { 89 | this.#inspectIndex++; 90 | localIndex++; 91 | this.#matchIndex++; 92 | if (this.#matchIndex === this.#delimiter.length) { 93 | const start = this.#inspectIndex - this.#delimiter.length; 94 | const end = this.#disp === "suffix" ? this.#inspectIndex : start; 95 | const copy = this.#buf.slice(0, end); 96 | controller.enqueue(copy); 97 | const shift = this.#disp == "prefix" ? start : this.#inspectIndex; 98 | this.#buf = this.#buf.slice(shift); 99 | this.#inspectIndex = this.#disp == "prefix" ? this.#delimiter.length : 0; 100 | this.#matchIndex = 0; 101 | } 102 | } else { 103 | if (this.#matchIndex === 0) { 104 | this.#inspectIndex++; 105 | localIndex++; 106 | } else { 107 | this.#matchIndex = this.#delimLPS[this.#matchIndex - 1]; 108 | } 109 | } 110 | } 111 | } 112 | } 113 | class JSONLinesParseStream { 114 | writable; 115 | readable; 116 | constructor({ separator ="\n" , writableStrategy , readableStrategy } = {}){ 117 | const { writable , readable } = new TextDelimiterStream(separator); 118 | this.writable = writable; 119 | this.readable = readable.pipeThrough(new TransformStream({ 120 | transform (chunk, controller) { 121 | if (!isBrankString(chunk)) { 122 | controller.enqueue(parse(chunk)); 123 | } 124 | } 125 | }, writableStrategy, readableStrategy)); 126 | } 127 | } 128 | class ConcatenatedJSONParseStream { 129 | writable; 130 | readable; 131 | constructor(options = {}){ 132 | const { writable , readable } = transformStreamFromGeneratorFunction(this.#concatenatedJSONIterator, options.writableStrategy, options.readableStrategy); 133 | this.writable = writable; 134 | this.readable = readable; 135 | } 136 | async *#concatenatedJSONIterator(src) { 137 | let targetString = ""; 138 | let hasValue = false; 139 | let nestCount = 0; 140 | let readingString = false; 141 | let escapeNext = false; 142 | for await (const string of src){ 143 | let sliceStart = 0; 144 | for(let i = 0; i < string.length; i++){ 145 | const __char = string[i]; 146 | if (readingString) { 147 | if (__char === '"' && !escapeNext) { 148 | readingString = false; 149 | if (nestCount === 0 && hasValue) { 150 | yield parse(targetString + string.slice(sliceStart, i + 1)); 151 | hasValue = false; 152 | targetString = ""; 153 | sliceStart = i + 1; 154 | } 155 | } 156 | escapeNext = !escapeNext && __char === "\\"; 157 | continue; 158 | } 159 | if (hasValue && nestCount === 0 && (__char === "{" || __char === "[" || __char === '"' || __char === " ")) { 160 | yield parse(targetString + string.slice(sliceStart, i)); 161 | hasValue = false; 162 | readingString = false; 163 | targetString = ""; 164 | sliceStart = i; 165 | i--; 166 | continue; 167 | } 168 | switch(__char){ 169 | case '"': 170 | readingString = true; 171 | escapeNext = false; 172 | break; 173 | case "{": 174 | case "[": 175 | nestCount++; 176 | break; 177 | case "}": 178 | case "]": 179 | nestCount--; 180 | break; 181 | } 182 | if (hasValue && nestCount === 0 && (__char === "}" || __char === "]")) { 183 | yield parse(targetString + string.slice(sliceStart, i + 1)); 184 | hasValue = false; 185 | targetString = ""; 186 | sliceStart = i + 1; 187 | continue; 188 | } 189 | if (!hasValue && !isBrankChar(__char)) { 190 | hasValue = true; 191 | } 192 | } 193 | targetString += string.slice(sliceStart); 194 | } 195 | if (hasValue) { 196 | yield parse(targetString); 197 | } 198 | } 199 | } 200 | function parse(text) { 201 | try { 202 | return JSON.parse(text); 203 | } catch (error) { 204 | if (error instanceof Error) { 205 | const truncatedText = 30 < text.length ? `${text.slice(0, 30)}...` : text; 206 | throw new error.constructor(`${error.message} (parsing: '${truncatedText}')`); 207 | } 208 | throw error; 209 | } 210 | } 211 | const blank = new Set(" \t\r\n"); 212 | function isBrankChar(__char) { 213 | return blank.has(__char); 214 | } 215 | const branks = /[^ \t\r\n]/; 216 | function isBrankString(str) { 217 | return !branks.test(str); 218 | } 219 | class JSONLinesStringifyStream extends TransformStream { 220 | constructor(options = {}){ 221 | const { separator ="\n" , writableStrategy , readableStrategy } = options; 222 | const [prefix, suffix] = separator.includes("\n") ? [ 223 | "", 224 | separator 225 | ] : [ 226 | separator, 227 | "\n" 228 | ]; 229 | super({ 230 | transform (chunk, controller) { 231 | controller.enqueue(`${prefix}${JSON.stringify(chunk)}${suffix}`); 232 | } 233 | }, writableStrategy, readableStrategy); 234 | } 235 | } 236 | class ConcatenatedJSONStringifyStream extends JSONLinesStringifyStream { 237 | constructor(options = {}){ 238 | const { writableStrategy , readableStrategy } = options; 239 | super({ 240 | separator: "\n", 241 | writableStrategy, 242 | readableStrategy 243 | }); 244 | } 245 | } 246 | export { transformStreamFromGeneratorFunction as transformStreamFromGeneratorFunction }; 247 | export { ConcatenatedJSONParseStream as ConcatenatedJSONParseStream, JSONLinesParseStream as JSONLinesParseStream }; 248 | export { ConcatenatedJSONStringifyStream as ConcatenatedJSONStringifyStream, JSONLinesStringifyStream as JSONLinesStringifyStream }; 249 | -------------------------------------------------------------------------------- /mod.ts: -------------------------------------------------------------------------------- 1 | import "./src/polyfill.ts"; 2 | export { 3 | type JSONValue, 4 | transformStreamFromGeneratorFunction, 5 | } from "./src/utils.ts"; 6 | export { 7 | ConcatenatedJSONParseStream, 8 | JSONLinesParseStream, 9 | type ParseStreamOptions, 10 | } from "./src/parser.ts"; 11 | export { 12 | ConcatenatedJSONStringifyStream, 13 | JSONLinesStringifyStream, 14 | type StringifyStreamOptions, 15 | } from "./src/stringify.ts"; 16 | -------------------------------------------------------------------------------- /mod_test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | assertEquals, 3 | assertRejects, 4 | } from "https://deno.land/std@0.185.0/testing/asserts.ts"; 5 | import { readableStreamFromIterable } from "https://deno.land/std@0.185.0/streams/readable_stream_from_iterable.ts"; 6 | import { 7 | ConcatenatedJSONParseStream, 8 | ConcatenatedJSONStringifyStream, 9 | JSONLinesParseStream, 10 | JSONLinesStringifyStream, 11 | JSONValue, 12 | ParseStreamOptions, 13 | StringifyStreamOptions, 14 | transformStreamFromGeneratorFunction, 15 | } from "./mod.ts"; 16 | 17 | async function assertValidParse( 18 | transform: typeof ConcatenatedJSONParseStream | typeof JSONLinesParseStream, 19 | chunks: string[], 20 | expect: unknown[], 21 | options?: ParseStreamOptions, 22 | ) { 23 | const r = new ReadableStream({ 24 | start(controller) { 25 | for (const chunk of chunks) { 26 | controller.enqueue(chunk); 27 | } 28 | controller.close(); 29 | }, 30 | }); 31 | const res = []; 32 | for await (const data of r.pipeThrough(new transform(options))) { 33 | res.push(data); 34 | } 35 | assertEquals(res, expect); 36 | } 37 | 38 | async function assertInvalidParse( 39 | transform: typeof ConcatenatedJSONParseStream | typeof JSONLinesParseStream, 40 | chunks: string[], 41 | options: ParseStreamOptions, 42 | // deno-lint-ignore no-explicit-any 43 | ErrorClass: new (...args: any[]) => Error, 44 | msgIncludes: string | undefined, 45 | ) { 46 | const r = new ReadableStream({ 47 | start(controller) { 48 | for (const chunk of chunks) { 49 | controller.enqueue(chunk); 50 | } 51 | controller.close(); 52 | }, 53 | }); 54 | await assertRejects( 55 | async () => { 56 | for await (const _ of r.pipeThrough(new transform(options))); 57 | }, 58 | ErrorClass, 59 | msgIncludes, 60 | ); 61 | } 62 | 63 | async function assertValidStringify( 64 | transform: 65 | | typeof ConcatenatedJSONStringifyStream 66 | | typeof JSONLinesStringifyStream, 67 | chunks: JSONValue[], 68 | expect: string[], 69 | options?: StringifyStreamOptions, 70 | ) { 71 | const r = new ReadableStream({ 72 | start(controller) { 73 | for (const chunk of chunks) { 74 | controller.enqueue(chunk); 75 | } 76 | controller.close(); 77 | }, 78 | }); 79 | const res = []; 80 | for await (const data of r.pipeThrough(new transform(options))) { 81 | res.push(data); 82 | } 83 | assertEquals(res, expect); 84 | } 85 | 86 | async function assertInvalidStringify( 87 | transform: 88 | | typeof ConcatenatedJSONStringifyStream 89 | | typeof JSONLinesStringifyStream, 90 | chunks: unknown[], 91 | options: StringifyStreamOptions, 92 | // deno-lint-ignore no-explicit-any 93 | ErrorClass: new (...args: any[]) => Error, 94 | msgIncludes: string | undefined, 95 | ) { 96 | const r = new ReadableStream({ 97 | start(controller) { 98 | for (const chunk of chunks) { 99 | controller.enqueue(chunk); 100 | } 101 | controller.close(); 102 | }, 103 | }); 104 | await assertRejects( 105 | async () => { 106 | for await (const _ of r.pipeThrough(new transform(options))); 107 | }, 108 | ErrorClass, 109 | msgIncludes, 110 | ); 111 | } 112 | 113 | Deno.test({ 114 | name: "parse(concatenated)", 115 | async fn() { 116 | await assertValidParse( 117 | ConcatenatedJSONParseStream, 118 | ['{"foo": "bar"}'], 119 | [{ foo: "bar" }], 120 | ); 121 | await assertValidParse( 122 | ConcatenatedJSONParseStream, 123 | ['{"foo": "bar"} '], 124 | [{ foo: "bar" }], 125 | ); 126 | await assertValidParse( 127 | ConcatenatedJSONParseStream, 128 | [' {"foo": "bar"}'], 129 | [{ foo: "bar" }], 130 | ); 131 | await assertValidParse( 132 | ConcatenatedJSONParseStream, 133 | ['[{"foo": "bar"}]'], 134 | [[{ foo: "bar" }]], 135 | ); 136 | await assertValidParse( 137 | ConcatenatedJSONParseStream, 138 | ['{"foo": "bar"}{"foo": "bar"}'], 139 | [{ foo: "bar" }, { foo: "bar" }], 140 | ); 141 | await assertValidParse( 142 | ConcatenatedJSONParseStream, 143 | ['{"foo": "bar"} {"foo": "bar"}'], 144 | [{ foo: "bar" }, { foo: "bar" }], 145 | ); 146 | }, 147 | }); 148 | 149 | Deno.test({ 150 | name: "parse(concatenated): primitive", 151 | async fn() { 152 | await assertValidParse( 153 | ConcatenatedJSONParseStream, 154 | ["0"], 155 | [0], 156 | ); 157 | await assertValidParse( 158 | ConcatenatedJSONParseStream, 159 | ["100"], 160 | [100], 161 | ); 162 | await assertValidParse( 163 | ConcatenatedJSONParseStream, 164 | ['100 200"foo"'], 165 | [100, 200, "foo"], 166 | ); 167 | await assertValidParse( 168 | ConcatenatedJSONParseStream, 169 | ['100 200{"foo": "bar"}'], 170 | [100, 200, { foo: "bar" }], 171 | ); 172 | await assertValidParse( 173 | ConcatenatedJSONParseStream, 174 | ['100 200["foo"]'], 175 | [100, 200, ["foo"]], 176 | ); 177 | 178 | await assertValidParse( 179 | ConcatenatedJSONParseStream, 180 | ['"foo"'], 181 | ["foo"], 182 | ); 183 | await assertValidParse( 184 | ConcatenatedJSONParseStream, 185 | ['"foo""bar"{"foo": "bar"}'], 186 | ["foo", "bar", { foo: "bar" }], 187 | ); 188 | await assertValidParse( 189 | ConcatenatedJSONParseStream, 190 | ['"foo""bar"["foo"]'], 191 | ["foo", "bar", ["foo"]], 192 | ); 193 | await assertValidParse( 194 | ConcatenatedJSONParseStream, 195 | ['"foo""bar"0'], 196 | ["foo", "bar", 0], 197 | ); 198 | 199 | await assertValidParse( 200 | ConcatenatedJSONParseStream, 201 | ["null"], 202 | [null], 203 | ); 204 | await assertValidParse( 205 | ConcatenatedJSONParseStream, 206 | ['null null{"foo": "bar"}'], 207 | [null, null, { foo: "bar" }], 208 | ); 209 | await assertValidParse( 210 | ConcatenatedJSONParseStream, 211 | ['null null["foo"]'], 212 | [null, null, ["foo"]], 213 | ); 214 | await assertValidParse( 215 | ConcatenatedJSONParseStream, 216 | ["null null 0"], 217 | [null, null, 0], 218 | ); 219 | await assertValidParse( 220 | ConcatenatedJSONParseStream, 221 | ['null null"foo"'], 222 | [null, null, "foo"], 223 | ); 224 | 225 | await assertValidParse( 226 | ConcatenatedJSONParseStream, 227 | ["true"], 228 | [true], 229 | ); 230 | await assertValidParse( 231 | ConcatenatedJSONParseStream, 232 | ['true true{"foo": "bar"}'], 233 | [true, true, { foo: "bar" }], 234 | ); 235 | await assertValidParse( 236 | ConcatenatedJSONParseStream, 237 | ['true true["foo"]'], 238 | [true, true, ["foo"]], 239 | ); 240 | await assertValidParse( 241 | ConcatenatedJSONParseStream, 242 | ["true true 0"], 243 | [true, true, 0], 244 | ); 245 | await assertValidParse( 246 | ConcatenatedJSONParseStream, 247 | ['true true"foo"'], 248 | [true, true, "foo"], 249 | ); 250 | 251 | await assertValidParse( 252 | ConcatenatedJSONParseStream, 253 | ["false"], 254 | [false], 255 | ); 256 | await assertValidParse( 257 | ConcatenatedJSONParseStream, 258 | ['false false{"foo": "bar"}'], 259 | [false, false, { foo: "bar" }], 260 | ); 261 | await assertValidParse( 262 | ConcatenatedJSONParseStream, 263 | ['false false["foo"]'], 264 | [false, false, ["foo"]], 265 | ); 266 | await assertValidParse( 267 | ConcatenatedJSONParseStream, 268 | ["false false 0"], 269 | [false, false, 0], 270 | ); 271 | await assertValidParse( 272 | ConcatenatedJSONParseStream, 273 | ['false false"foo"'], 274 | [false, false, "foo"], 275 | ); 276 | }, 277 | }); 278 | 279 | Deno.test({ 280 | name: "parse(concatenated): chunk", 281 | async fn() { 282 | await assertValidParse( 283 | ConcatenatedJSONParseStream, 284 | ["", '{"foo": "bar"}'], 285 | [{ foo: "bar" }], 286 | ); 287 | await assertValidParse( 288 | ConcatenatedJSONParseStream, 289 | ["{", '"foo": "bar"}'], 290 | [{ foo: "bar" }], 291 | ); 292 | await assertValidParse( 293 | ConcatenatedJSONParseStream, 294 | ['{"foo": "b', 'ar"}'], 295 | [{ foo: "bar" }], 296 | ); 297 | await assertValidParse( 298 | ConcatenatedJSONParseStream, 299 | ['{"foo": "bar"', "}"], 300 | [{ foo: "bar" }], 301 | ); 302 | await assertValidParse( 303 | ConcatenatedJSONParseStream, 304 | ['{"foo": "bar"}', ""], 305 | [{ foo: "bar" }], 306 | ); 307 | await assertValidParse( 308 | ConcatenatedJSONParseStream, 309 | ['{"foo": "bar"}', '{"foo": "bar"}'], 310 | [{ foo: "bar" }, { foo: "bar" }], 311 | ); 312 | await assertValidParse( 313 | ConcatenatedJSONParseStream, 314 | ['{"foo": "bar"', '}{"foo": "bar"}'], 315 | [{ foo: "bar" }, { foo: "bar" }], 316 | ); 317 | await assertValidParse( 318 | ConcatenatedJSONParseStream, 319 | ['{"foo": "bar"}{', '"foo": "bar"}'], 320 | [{ foo: "bar" }, { foo: "bar" }], 321 | ); 322 | }, 323 | }); 324 | 325 | Deno.test({ 326 | name: "parse(concatenated): surrogate pair", 327 | async fn() { 328 | await assertValidParse( 329 | ConcatenatedJSONParseStream, 330 | ['{"foo": "👪"}{"foo": "👪"}'], 331 | [{ foo: "👪" }, { foo: "👪" }], 332 | ); 333 | }, 334 | }); 335 | 336 | Deno.test({ 337 | name: "parse(concatenated): halfway chunk", 338 | async fn() { 339 | await assertInvalidParse( 340 | ConcatenatedJSONParseStream, 341 | ['{"foo": "bar"} {"foo": '], 342 | {}, 343 | SyntaxError, 344 | `Unexpected end of JSON input (parsing: ' {"foo": ')`, 345 | ); 346 | }, 347 | }); 348 | 349 | Deno.test({ 350 | name: "parse(concatenated): truncate error message", 351 | async fn() { 352 | await assertInvalidParse( 353 | ConcatenatedJSONParseStream, 354 | [`{${"foo".repeat(100)}}`], 355 | {}, 356 | SyntaxError, 357 | `(parsing: '{foofoofoofoofoofoofoofoofoofo...')`, 358 | ); 359 | }, 360 | }); 361 | 362 | Deno.test({ 363 | name: "parse(separator)", 364 | async fn() { 365 | await assertValidParse( 366 | JSONLinesParseStream, 367 | ['{"foo": "bar"}'], 368 | [{ foo: "bar" }], 369 | ); 370 | await assertValidParse( 371 | JSONLinesParseStream, 372 | ['{"foo": "bar"}\n'], 373 | [{ foo: "bar" }], 374 | ); 375 | await assertValidParse( 376 | JSONLinesParseStream, 377 | ['{"foo": "bar"}\r\n'], 378 | [{ foo: "bar" }], 379 | ); 380 | await assertValidParse( 381 | JSONLinesParseStream, 382 | ['\n{"foo": "bar"}\n'], 383 | [{ foo: "bar" }], 384 | ); 385 | await assertValidParse( 386 | JSONLinesParseStream, 387 | ["[0]\n"], 388 | [[0]], 389 | ); 390 | await assertValidParse( 391 | JSONLinesParseStream, 392 | ["0\n"], 393 | [0], 394 | ); 395 | }, 396 | }); 397 | 398 | Deno.test({ 399 | name: "parse(separator): chunk", 400 | async fn() { 401 | await assertValidParse( 402 | JSONLinesParseStream, 403 | ["{", '"foo": "bar"}\n'], 404 | [{ foo: "bar" }], 405 | ); 406 | await assertValidParse( 407 | JSONLinesParseStream, 408 | ['{"foo', '": "bar"}\n'], 409 | [{ foo: "bar" }], 410 | ); 411 | await assertValidParse( 412 | JSONLinesParseStream, 413 | ['{"foo":', ' "bar"}\n'], 414 | [{ foo: "bar" }], 415 | ); 416 | await assertValidParse( 417 | JSONLinesParseStream, 418 | ['{"foo": "bar"', "}\n"], 419 | [{ foo: "bar" }], 420 | ); 421 | await assertValidParse( 422 | JSONLinesParseStream, 423 | ['{"foo": "bar"}', "\n"], 424 | [{ foo: "bar" }], 425 | ); 426 | await assertValidParse( 427 | JSONLinesParseStream, 428 | ['{"foo": "bar"}\n', ""], 429 | [{ foo: "bar" }], 430 | ); 431 | }, 432 | }); 433 | 434 | Deno.test({ 435 | name: "parse(separator): special separator", 436 | async fn() { 437 | const separators = ["\x1E", "xxxxx", "😊", "🦕"]; 438 | for (const separator of separators) { 439 | await assertValidParse( 440 | JSONLinesParseStream, 441 | [`${separator}{"foo": "bar"}${separator}{"foo": "bar"}${separator}`], 442 | [{ foo: "bar" }, { foo: "bar" }], 443 | { separator }, 444 | ); 445 | } 446 | }, 447 | }); 448 | 449 | Deno.test({ 450 | name: "parse(separator): empty line", 451 | async fn() { 452 | await assertValidParse( 453 | JSONLinesParseStream, 454 | ['{"foo": "bar"} \n {"foo": "bar"} \n'], 455 | [{ foo: "bar" }, { foo: "bar" }], 456 | ); 457 | await assertValidParse( 458 | JSONLinesParseStream, 459 | ['{"foo": "bar"} \n\n {"foo": "bar"}'], 460 | [{ foo: "bar" }, { foo: "bar" }], 461 | ); 462 | }, 463 | }); 464 | 465 | Deno.test({ 466 | name: "parse(separator): surrogate pair", 467 | async fn() { 468 | await assertValidParse( 469 | JSONLinesParseStream, 470 | ['{"foo": "👪"}\n{"foo": "👪"}\n'], 471 | [{ foo: "👪" }, { foo: "👪" }], 472 | ); 473 | }, 474 | }); 475 | 476 | Deno.test({ 477 | name: "parse(separator): invalid line break", 478 | async fn() { 479 | await assertInvalidParse( 480 | JSONLinesParseStream, 481 | ['{"foo": \n "bar"} \n {"foo": \n "bar"}'], 482 | {}, 483 | SyntaxError, 484 | `Unexpected end of JSON input (parsing: '{"foo": ')`, 485 | ); 486 | }, 487 | }); 488 | 489 | Deno.test({ 490 | name: "parse(separator): halfway chunk", 491 | async fn() { 492 | await assertInvalidParse( 493 | JSONLinesParseStream, 494 | ['{"foo": "bar"} \n {"foo": '], 495 | {}, 496 | SyntaxError, 497 | `Unexpected end of JSON input (parsing: ' {"foo": ')`, 498 | ); 499 | }, 500 | }); 501 | 502 | Deno.test({ 503 | name: "stringify(concatenated)", 504 | async fn() { 505 | await assertValidStringify( 506 | ConcatenatedJSONStringifyStream, 507 | [{ foo: "bar" }, { foo: "bar" }], 508 | ['{"foo":"bar"}\n', '{"foo":"bar"}\n'], 509 | ); 510 | 511 | const cyclic: Record = {}; 512 | cyclic.cyclic = cyclic; 513 | await assertInvalidStringify( 514 | ConcatenatedJSONStringifyStream, 515 | [cyclic], 516 | {}, 517 | TypeError, 518 | "Converting circular structure to JSON", 519 | ); 520 | }, 521 | }); 522 | 523 | Deno.test({ 524 | name: "stringify(separator)", 525 | async fn() { 526 | await assertValidStringify( 527 | JSONLinesStringifyStream, 528 | [{ foo: "bar" }, { foo: "bar" }], 529 | ['{"foo":"bar"}aaa\n', '{"foo":"bar"}aaa\n'], 530 | { separator: "aaa\n" }, 531 | ); 532 | await assertValidStringify( 533 | JSONLinesStringifyStream, 534 | [{ foo: "bar" }, { foo: "bar" }], 535 | ['aaa{"foo":"bar"}\n', 'aaa{"foo":"bar"}\n'], 536 | { separator: "aaa" }, 537 | ); 538 | }, 539 | }); 540 | 541 | Deno.test({ 542 | name: "transformStreamFromGeneratorFunction", 543 | async fn() { 544 | const reader = readableStreamFromIterable([0, 1, 2]) 545 | .pipeThrough(transformStreamFromGeneratorFunction(async function* (src) { 546 | for await (const i of src) { 547 | yield i * 100; 548 | } 549 | })); 550 | const res = []; 551 | for await (const i of reader) { 552 | res.push(i); 553 | } 554 | assertEquals(res, [0, 100, 200]); 555 | }, 556 | }); 557 | 558 | Deno.test({ 559 | name: "transformStreamFromGeneratorFunction: iterable (not async)", 560 | async fn() { 561 | const reader = readableStreamFromIterable([0, 1, 2]) 562 | .pipeThrough(transformStreamFromGeneratorFunction(function* (_src) { 563 | yield 0; 564 | yield 100; 565 | yield 200; 566 | })); 567 | const res = []; 568 | for await (const i of reader) { 569 | res.push(i); 570 | } 571 | assertEquals(res, [0, 100, 200]); 572 | }, 573 | }); 574 | 575 | Deno.test({ 576 | name: "transformStreamFromGeneratorFunction: cancel", 577 | async fn() { 578 | let callCount = 0; 579 | let cancelReason = ""; 580 | const reader = new ReadableStream({ 581 | cancel(reason) { 582 | callCount++; 583 | cancelReason = reason; 584 | }, 585 | }).pipeThrough(transformStreamFromGeneratorFunction(async function* (_) { 586 | yield 0; 587 | })); 588 | 589 | await reader.cancel("__reason__"); 590 | 591 | assertEquals(callCount, 1); 592 | assertEquals(cancelReason, "__reason__"); 593 | }, 594 | }); 595 | -------------------------------------------------------------------------------- /src/parser.ts: -------------------------------------------------------------------------------- 1 | import { TextDelimiterStream } from "https://deno.land/std@0.185.0/streams/text_delimiter_stream.ts"; 2 | import { JSONValue, transformStreamFromGeneratorFunction } from "./utils.ts"; 3 | 4 | // avoid dnt typecheck error 5 | type _QueuingStrategy = QueuingStrategy; 6 | 7 | export interface ParseStreamOptions { 8 | /**a character to separate JSON. The default is '\n'. */ 9 | readonly separator?: string; 10 | /** Controls the buffer of the TransformStream used internally. Check https://developer.mozilla.org/en-US/docs/Web/API/TransformStream/TransformStream. */ 11 | readonly writableStrategy?: _QueuingStrategy; 12 | /** Controls the buffer of the TransformStream used internally. Check https://developer.mozilla.org/en-US/docs/Web/API/TransformStream/TransformStream. */ 13 | readonly readableStrategy?: _QueuingStrategy; 14 | } 15 | 16 | /** 17 | * stream to parse [JSON lines](https://jsonlines.org/), [NDJSON](http://ndjson.org/) and [JSON Text Sequences](https://datatracker.ietf.org/doc/html/rfc7464). 18 | * 19 | * ```ts 20 | * import { JSONLinesParseStream } from "https://deno.land/x/jsonlines@v1.2.1/mod.ts"; 21 | * 22 | * const url = new URL("./testdata/json-lines.jsonl", import.meta.url); 23 | * const { body } = await fetch(`${url}`); 24 | * 25 | * const readable = body! 26 | * .pipeThrough(new TextDecoderStream()) 27 | * .pipeThrough(new JSONLinesParseStream()); 28 | * 29 | * for await (const data of readable) { 30 | * console.log(data); 31 | * } 32 | * ``` 33 | */ 34 | export class JSONLinesParseStream 35 | implements TransformStream { 36 | readonly writable: WritableStream; 37 | readonly readable: ReadableStream; 38 | /** 39 | * @param options 40 | * @param options.separator a character to separate JSON. The character length must be 1. The default is '\n'. 41 | * @param options.writableStrategy Controls the buffer of the TransformStream used internally. Check https://developer.mozilla.org/en-US/docs/Web/API/TransformStream/TransformStream. 42 | * @param options.readableStrategy Controls the buffer of the TransformStream used internally. Check https://developer.mozilla.org/en-US/docs/Web/API/TransformStream/TransformStream. 43 | */ 44 | constructor({ 45 | separator = "\n", 46 | writableStrategy, 47 | readableStrategy, 48 | }: ParseStreamOptions = {}) { 49 | const { writable, readable } = new TextDelimiterStream(separator); 50 | this.writable = writable; 51 | this.readable = readable.pipeThrough( 52 | new TransformStream( 53 | { 54 | transform( 55 | chunk: string, 56 | controller: TransformStreamDefaultController, 57 | ) { 58 | if (!isBrankString(chunk)) { 59 | controller.enqueue(parse(chunk)); 60 | } 61 | }, 62 | }, 63 | writableStrategy, 64 | readableStrategy, 65 | ), 66 | ); 67 | } 68 | } 69 | 70 | /** 71 | * stream to parse [Concatenated JSON](https://en.wikipedia.org/wiki/JSON_streaming#Concatenated_JSON). 72 | * 73 | * ```ts 74 | * import { ConcatenatedJSONParseStream } from "https://deno.land/x/jsonlines@v1.2.1/mod.ts"; 75 | * 76 | * const url = new URL("./testdata/concat-json.concat-json", import.meta.url); 77 | * const { body } = await fetch(`${url}`); 78 | * 79 | * const readable = body! 80 | * .pipeThrough(new TextDecoderStream()) 81 | * .pipeThrough(new ConcatenatedJSONParseStream()); 82 | * 83 | * for await (const data of readable) { 84 | * console.log(data); 85 | * } 86 | * ``` 87 | */ 88 | export class ConcatenatedJSONParseStream 89 | implements TransformStream { 90 | readonly writable: WritableStream; 91 | readonly readable: ReadableStream; 92 | /** 93 | * @param options 94 | * @param options.separator This parameter will be ignored. 95 | * @param options.writableStrategy Controls the buffer of the TransformStream used internally. Check https://developer.mozilla.org/en-US/docs/Web/API/TransformStream/TransformStream. 96 | * @param options.readableStrategy Controls the buffer of the TransformStream used internally. Check https://developer.mozilla.org/en-US/docs/Web/API/TransformStream/TransformStream. 97 | */ 98 | constructor(options: ParseStreamOptions = {}) { 99 | const { writable, readable } = transformStreamFromGeneratorFunction( 100 | this.#concatenatedJSONIterator, 101 | options.writableStrategy, 102 | options.readableStrategy, 103 | ); 104 | this.writable = writable; 105 | this.readable = readable; 106 | } 107 | 108 | async *#concatenatedJSONIterator(src: AsyncIterable) { 109 | // Counts the number of '{', '}', '[', ']', and when the nesting level reaches 0, concatenates and returns the string. 110 | let targetString = ""; 111 | let hasValue = false; 112 | let nestCount = 0; 113 | let readingString = false; 114 | let escapeNext = false; 115 | for await (const string of src) { 116 | let sliceStart = 0; 117 | for (let i = 0; i < string.length; i++) { 118 | const char = string[i]; 119 | 120 | if (readingString) { 121 | if (char === '"' && !escapeNext) { 122 | readingString = false; 123 | 124 | // When the nesting level is 0, it returns a string when '"' comes. 125 | if (nestCount === 0 && hasValue) { 126 | yield parse(targetString + string.slice(sliceStart, i + 1)); 127 | hasValue = false; 128 | targetString = ""; 129 | sliceStart = i + 1; 130 | } 131 | } 132 | escapeNext = !escapeNext && char === "\\"; 133 | continue; 134 | } 135 | 136 | // Parses number, true, false, null with a nesting level of 0. 137 | // example: 'null["foo"]' => null, ["foo"] 138 | // example: 'false{"foo": "bar"}' => false, {foo: "bar"} 139 | if ( 140 | hasValue && nestCount === 0 && 141 | (char === "{" || char === "[" || char === '"' || char === " ") 142 | ) { 143 | yield parse(targetString + string.slice(sliceStart, i)); 144 | hasValue = false; 145 | readingString = false; 146 | targetString = ""; 147 | sliceStart = i; 148 | i--; 149 | continue; 150 | } 151 | 152 | switch (char) { 153 | case '"': 154 | readingString = true; 155 | escapeNext = false; 156 | break; 157 | case "{": 158 | case "[": 159 | nestCount++; 160 | break; 161 | case "}": 162 | case "]": 163 | nestCount--; 164 | break; 165 | } 166 | 167 | // parse object or array 168 | if ( 169 | hasValue && nestCount === 0 && 170 | (char === "}" || char === "]") 171 | ) { 172 | yield parse(targetString + string.slice(sliceStart, i + 1)); 173 | hasValue = false; 174 | targetString = ""; 175 | sliceStart = i + 1; 176 | continue; 177 | } 178 | 179 | if (!hasValue && !isBrankChar(char)) { 180 | // We want to ignore the character string with only blank, so if there is a character other than blank, record it. 181 | hasValue = true; 182 | } 183 | } 184 | targetString += string.slice(sliceStart); 185 | } 186 | if (hasValue) { 187 | yield parse(targetString); 188 | } 189 | } 190 | } 191 | 192 | /** JSON.parse with detailed error message */ 193 | function parse(text: string) { 194 | try { 195 | return JSON.parse(text) as JSONValue; 196 | } catch (error: unknown) { 197 | if (error instanceof Error) { 198 | // Truncate the string so that it is within 30 lengths. 199 | const truncatedText = 30 < text.length ? `${text.slice(0, 30)}...` : text; 200 | throw new (error.constructor as ErrorConstructor)( 201 | `${error.message} (parsing: '${truncatedText}')`, 202 | ); 203 | } 204 | throw error; 205 | } 206 | } 207 | 208 | const blank = new Set(" \t\r\n"); 209 | function isBrankChar(char: string) { 210 | return blank.has(char); 211 | } 212 | 213 | const branks = /[^ \t\r\n]/; 214 | function isBrankString(str: string) { 215 | return !branks.test(str); 216 | } 217 | -------------------------------------------------------------------------------- /src/polyfill.ts: -------------------------------------------------------------------------------- 1 | declare global { 2 | // deno-lint-ignore no-explicit-any 3 | interface ReadableStream { 4 | [Symbol.asyncIterator](options?: { 5 | preventCancel?: boolean; 6 | }): AsyncIterableIterator; 7 | } 8 | } 9 | 10 | // polyfill for ReadableStream.prototype[Symbol.asyncIterator] 11 | // https://bugs.chromium.org/p/chromium/issues/detail?id=929585#c10 12 | if (typeof ReadableStream.prototype[Symbol.asyncIterator] !== "function") { 13 | Object.defineProperty(ReadableStream.prototype, Symbol.asyncIterator, { 14 | async *value() { 15 | const reader = this.getReader(); 16 | try { 17 | while (true) { 18 | const { done, value } = await reader.read(); 19 | if (done) return; 20 | yield value; 21 | } 22 | } finally { 23 | reader.releaseLock(); 24 | } 25 | }, 26 | writable: true, 27 | enumerable: false, 28 | configurable: true, 29 | }); 30 | } 31 | 32 | export {}; 33 | -------------------------------------------------------------------------------- /src/stringify.ts: -------------------------------------------------------------------------------- 1 | // avoid dnt typecheck error 2 | type _QueuingStrategy = QueuingStrategy; 3 | 4 | export interface StringifyStreamOptions { 5 | /**a character to separate JSON. The default is '\n'. */ 6 | readonly separator?: string; 7 | /** Controls the buffer of the TransformStream used internally. Check https://developer.mozilla.org/en-US/docs/Web/API/TransformStream/TransformStream. */ 8 | readonly writableStrategy?: _QueuingStrategy; 9 | /** Controls the buffer of the TransformStream used internally. Check https://developer.mozilla.org/en-US/docs/Web/API/TransformStream/TransformStream. */ 10 | readonly readableStrategy?: _QueuingStrategy; 11 | } 12 | 13 | /** 14 | * stream to stringify [JSON lines](https://jsonlines.org/), [NDJSON](http://ndjson.org/) and [JSON Text Sequences](https://datatracker.ietf.org/doc/html/rfc7464). 15 | * 16 | * ```ts 17 | * import { readableStreamFromIterable } from "https://deno.land/std@0.185.0/streams/mod.ts"; 18 | * import { JSONLinesStringifyStream } from "https://deno.land/x/jsonlines@v1.2.1/mod.ts"; 19 | * 20 | * const file = await Deno.open(new URL("./tmp.concat-json", import.meta.url), { 21 | * create: true, 22 | * write: true, 23 | * }); 24 | * 25 | * readableStreamFromIterable([{ foo: "bar" }, { baz: 100 }]) 26 | * .pipeThrough(new JSONLinesStringifyStream()) 27 | * .pipeThrough(new TextEncoderStream()) 28 | * .pipeTo(file.writable) 29 | * .then(() => console.log("write success")); 30 | * ``` 31 | */ 32 | export class JSONLinesStringifyStream extends TransformStream { 33 | /** 34 | * @param options 35 | * @param options.separator a character to separate JSON. The default is '\n'. 36 | * @param options.writableStrategy Controls the buffer of the TransformStream used internally. Check https://developer.mozilla.org/en-US/docs/Web/API/TransformStream/TransformStream. 37 | * @param options.readableStrategy Controls the buffer of the TransformStream used internally. Check https://developer.mozilla.org/en-US/docs/Web/API/TransformStream/TransformStream. 38 | */ 39 | constructor(options: StringifyStreamOptions = {}) { 40 | const { separator = "\n", writableStrategy, readableStrategy } = options; 41 | const [prefix, suffix] = separator.includes("\n") 42 | ? ["", separator] 43 | : [separator, "\n"]; 44 | super( 45 | { 46 | transform(chunk, controller) { 47 | controller.enqueue(`${prefix}${JSON.stringify(chunk)}${suffix}`); 48 | }, 49 | }, 50 | writableStrategy, 51 | readableStrategy, 52 | ); 53 | } 54 | } 55 | 56 | /** 57 | * stream to stringify [Concatenated JSON](https://en.wikipedia.org/wiki/JSON_streaming#Concatenated_JSON). 58 | * 59 | * ```ts 60 | * import { readableStreamFromIterable } from "https://deno.land/std@0.185.0/streams/mod.ts"; 61 | * import { ConcatenatedJSONStringifyStream } from "https://deno.land/x/jsonlines@v1.2.1/mod.ts"; 62 | * 63 | * const file = await Deno.open(new URL("./tmp.concat-json", import.meta.url), { 64 | * create: true, 65 | * write: true, 66 | * }); 67 | * 68 | * readableStreamFromIterable([{ foo: "bar" }, { baz: 100 }]) 69 | * .pipeThrough(new ConcatenatedJSONStringifyStream()) 70 | * .pipeThrough(new TextEncoderStream()) 71 | * .pipeTo(file.writable) 72 | * .then(() => console.log("write success")); 73 | * ``` 74 | */ 75 | export class ConcatenatedJSONStringifyStream extends JSONLinesStringifyStream { 76 | /** 77 | * @param options 78 | * @param options.separator This parameter will be ignored. 79 | * @param options.writableStrategy Controls the buffer of the TransformStream used internally. Check https://developer.mozilla.org/en-US/docs/Web/API/TransformStream/TransformStream. 80 | * @param options.readableStrategy Controls the buffer of the TransformStream used internally. Check https://developer.mozilla.org/en-US/docs/Web/API/TransformStream/TransformStream. 81 | */ 82 | constructor(options: StringifyStreamOptions = {}) { 83 | const { writableStrategy, readableStrategy } = options; 84 | super({ separator: "\n", writableStrategy, readableStrategy }); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | export type JSONValue = 2 | | { [key: string]: JSONValue } 3 | | JSONValue[] 4 | | string 5 | | number 6 | | boolean; 7 | 8 | // avoid dnt typecheck error 9 | type _QueuingStrategy = QueuingStrategy; 10 | 11 | /** 12 | * Convert the generator function into a TransformStream. 13 | * 14 | * ```ts 15 | * import { readableStreamFromIterable } from "https://deno.land/std@0.185.0/streams/mod.ts"; 16 | * import { transformStreamFromGeneratorFunction } from "https://deno.land/x/jsonlines@v1.2.1/mod.ts"; 17 | * 18 | * const reader = readableStreamFromIterable([0, 1, 2]) 19 | * .pipeThrough(transformStreamFromGeneratorFunction(async function* (src) { 20 | * for await (const chunk of src) { 21 | * yield chunk * 100; 22 | * } 23 | * })); 24 | * 25 | * for await (const chunk of reader) { 26 | * console.log(chunk); 27 | * } 28 | * // output: 0, 100, 200 29 | * ``` 30 | * 31 | * @param transformer A function to transform. 32 | * @param writableStrategy An object that optionally defines a queuing strategy for the stream. 33 | * @param readableStrategy An object that optionally defines a queuing strategy for the stream. 34 | */ 35 | export function transformStreamFromGeneratorFunction( 36 | transformer: (src: ReadableStream) => Iterable | AsyncIterable, 37 | writableStrategy?: _QueuingStrategy, 38 | readableStrategy?: _QueuingStrategy, 39 | ): TransformStream { 40 | const { 41 | writable, 42 | readable, 43 | } = new TransformStream(undefined, writableStrategy); 44 | 45 | const iterable = transformer(readable); 46 | const iterator: Iterator | AsyncIterator = 47 | (iterable as AsyncIterable)[Symbol.asyncIterator]?.() ?? 48 | (iterable as Iterable)[Symbol.iterator]?.(); 49 | return { 50 | writable, 51 | readable: new ReadableStream({ 52 | async pull(controller) { 53 | const { done, value } = await iterator.next(); 54 | if (done) { 55 | controller.close(); 56 | return; 57 | } 58 | controller.enqueue(value); 59 | }, 60 | async cancel(...args) { 61 | await readable.cancel(...args); 62 | }, 63 | }, readableStrategy), 64 | }; 65 | } 66 | -------------------------------------------------------------------------------- /testdata/concat-json.concat-json: -------------------------------------------------------------------------------- 1 | {"foo":"bar"}{"qux":"corge"}{"baz":{"waldo":"thud"}} 2 | -------------------------------------------------------------------------------- /testdata/json-lines.jsonl: -------------------------------------------------------------------------------- 1 | {"some":"thing"} 2 | {"foo":17,"bar":false,"quux":true} 3 | {"may":{"include":"nested","objects":["and","arrays"]}} 4 | -------------------------------------------------------------------------------- /testdata/json-seq.json-seq: -------------------------------------------------------------------------------- 1 | {"some":"thing\n"} 2 | { 3 | "may": { 4 | "include": "nested", 5 | "objects": [ 6 | "and", 7 | "arrays" 8 | ] 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /testdata/nd-json.ndjson: -------------------------------------------------------------------------------- 1 | {"some":"thing"} 2 | {"foo":17,"bar":false,"quux":true} 3 | {"may":{"include":"nested","objects":["and","arrays"]}} 4 | -------------------------------------------------------------------------------- /testdata/test.ts: -------------------------------------------------------------------------------- 1 | import { ConcatenatedJSONParseStream, JSONLinesParseStream } from "../mod.ts"; 2 | 3 | Deno.test(async function read_jsonlines() { 4 | const url = new URL("./json-lines.jsonl", import.meta.url); 5 | const { body } = await fetch(`${url}`); 6 | 7 | const readable = body! 8 | .pipeThrough(new TextDecoderStream()) 9 | .pipeThrough(new JSONLinesParseStream()); 10 | 11 | for await (const data of readable) { 12 | console.log(data); 13 | } 14 | }); 15 | 16 | Deno.test(async function read_ndjson() { 17 | const url = new URL("./nd-json.ndjson", import.meta.url); 18 | const { body } = await fetch(`${url}`); 19 | 20 | const readable = body! 21 | .pipeThrough(new TextDecoderStream()) 22 | .pipeThrough(new JSONLinesParseStream()); 23 | 24 | for await (const data of readable) { 25 | console.log(data); 26 | } 27 | }); 28 | 29 | Deno.test(async function read_json_seq() { 30 | const url = new URL("./json-seq.json-seq", import.meta.url); 31 | const { body } = await fetch(`${url}`); 32 | 33 | const recordSeparator = "\x1E"; 34 | const readable = body! 35 | .pipeThrough(new TextDecoderStream()) 36 | .pipeThrough(new JSONLinesParseStream({ separator: recordSeparator })); 37 | 38 | for await (const data of readable) { 39 | console.log(data); 40 | } 41 | }); 42 | 43 | Deno.test(async function read_concat_json() { 44 | const url = new URL("./concat-json.concat-json", import.meta.url); 45 | const { body } = await fetch(`${url}`); 46 | 47 | const readable = body! 48 | .pipeThrough(new TextDecoderStream()) 49 | .pipeThrough(new ConcatenatedJSONParseStream()); 50 | 51 | for await (const data of readable) { 52 | console.log(data); 53 | } 54 | }); 55 | --------------------------------------------------------------------------------