├── .editorconfig ├── .gitattributes ├── .github ├── security.md └── workflows │ └── main.yml ├── .gitignore ├── .npmrc ├── benchmarks ├── fixture.js ├── index.js ├── measure.js └── stream.js ├── fixture ├── license ├── package.json ├── readme.md ├── source ├── array-buffer.js ├── array.js ├── buffer.js ├── contents.js ├── exports.js ├── index.d.ts ├── index.js ├── index.test-d.ts ├── stream.js ├── string.js └── utils.js └── test ├── array-buffer.js ├── array.js ├── browser.js ├── buffer.js ├── contents.js ├── fixtures ├── index.js ├── iterable.js ├── node-stream.js └── web-stream.js ├── helpers └── index.js ├── integration.js ├── stream.js ├── string.js ├── web-stream-ponyfill.js └── web-stream.js /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = tab 5 | end_of_line = lf 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | 10 | [*.yml] 11 | indent_style = space 12 | indent_size = 2 13 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | -------------------------------------------------------------------------------- /.github/security.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | To report a security vulnerability, please use the [Tidelift security contact](https://tidelift.com/security). Tidelift will coordinate the fix and disclosure. 4 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | - push 4 | - pull_request 5 | jobs: 6 | test: 7 | name: Node.js ${{ matrix.node-version }} 8 | runs-on: ubuntu-latest 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | node-version: 13 | - 23 14 | - 18 15 | steps: 16 | - uses: actions/checkout@v4 17 | - uses: actions/setup-node@v4 18 | with: 19 | node-version: ${{ matrix.node-version }} 20 | - run: npm install 21 | - run: npm test 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | yarn.lock 3 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | package-lock=false 2 | -------------------------------------------------------------------------------- /benchmarks/fixture.js: -------------------------------------------------------------------------------- 1 | import {writeFile, rm} from 'node:fs/promises'; 2 | 3 | // Create and delete a big fixture file 4 | export const createFixture = async () => { 5 | await writeFile(FIXTURE_FILE, '.'.repeat(FIXTURE_BYTE_SIZE)); 6 | }; 7 | 8 | export const deleteFixture = async () => { 9 | await rm(FIXTURE_FILE); 10 | }; 11 | 12 | export const FIXTURE_FILE = 'benchmark_fixture'; 13 | 14 | const FIXTURE_BYTE_SIZE = 1e8; 15 | export const FIXTURE_HUMAN_SIZE = `${FIXTURE_BYTE_SIZE / 1e6} MB`; 16 | -------------------------------------------------------------------------------- /benchmarks/index.js: -------------------------------------------------------------------------------- 1 | import {text, buffer, arrayBuffer} from 'node:stream/consumers'; 2 | import getStream, {getStreamAsBuffer, getStreamAsArrayBuffer, getStreamAsArray} from '../source/index.js'; 3 | import {createFixture, deleteFixture, FIXTURE_HUMAN_SIZE} from './fixture.js'; 4 | import { 5 | createNodeStreamBinary, 6 | createNodeStreamText, 7 | createWebStreamBinary, 8 | createWebStreamText, 9 | } from './stream.js'; 10 | import {measureTask} from './measure.js'; 11 | 12 | const runBenchmarks = async () => { 13 | await createFixture(); 14 | 15 | try { 16 | await benchmarkNodeStreams(createNodeStreamBinary, `Node.js stream (${FIXTURE_HUMAN_SIZE}, binary)`); 17 | await benchmarkNodeStreams(createNodeStreamText, `Node.js stream (${FIXTURE_HUMAN_SIZE}, text)`); 18 | await benchmarkStreams(createWebStreamBinary, `Web ReadableStream (${FIXTURE_HUMAN_SIZE}, binary)`); 19 | await benchmarkStreams(createWebStreamText, `Web ReadableStream (${FIXTURE_HUMAN_SIZE}, text)`); 20 | } finally { 21 | await deleteFixture(); 22 | } 23 | }; 24 | 25 | const benchmarkNodeStreams = async (createStream, header) => { 26 | await benchmarkStreams(createStream, header); 27 | await logResult('stream.toArray', createStream, stream => stream.toArray()); 28 | }; 29 | 30 | const benchmarkStreams = async (createStream, header) => { 31 | logHeader(header); 32 | await logResult('getStream', createStream, getStream); 33 | await logResult('text', createStream, text); 34 | await logResult('getStreamAsBuffer', createStream, getStreamAsBuffer); 35 | await logResult('buffer', createStream, buffer); 36 | await logResult('getStreamAsArrayBuffer', createStream, getStreamAsArrayBuffer); 37 | await logResult('arrayBuffer', createStream, arrayBuffer); 38 | await logResult('getStreamAsArray', createStream, getStreamAsArray); 39 | }; 40 | 41 | const logHeader = header => { 42 | console.log(`\n### ${header}\n`); 43 | }; 44 | 45 | const logResult = async (name, createStream, task) => { 46 | console.log(`- \`${name}()\`: ${await measureTask(createStream, task)}ms`); 47 | }; 48 | 49 | await runBenchmarks(); 50 | -------------------------------------------------------------------------------- /benchmarks/measure.js: -------------------------------------------------------------------------------- 1 | import now from 'precise-now'; 2 | 3 | // Return how many ms running `task()` takes 4 | export const measureTask = async ({start, stop}, task) => { 5 | const taskInputs = await Promise.all(Array.from({length: MAX_LOOPS + 1}, start)); 6 | 7 | // Pre-warm 8 | await task(taskInputs[0].stream); 9 | 10 | const startTimestamp = now(); 11 | for (let index = 1; index <= MAX_LOOPS; index += 1) { 12 | // eslint-disable-next-line no-await-in-loop 13 | await task(taskInputs[index].stream); 14 | } 15 | 16 | const duration = Math.round((now() - startTimestamp) / (MAX_LOOPS * NANOSECS_TO_MILLESECS)); 17 | 18 | await Promise.all(taskInputs.map(taskInput => stop(taskInput))); 19 | 20 | return duration; 21 | }; 22 | 23 | const MAX_LOOPS = 10; 24 | const NANOSECS_TO_MILLESECS = 1e6; 25 | -------------------------------------------------------------------------------- /benchmarks/stream.js: -------------------------------------------------------------------------------- 1 | import {open} from 'node:fs/promises'; 2 | import {createReadStream} from 'node:fs'; 3 | import {FIXTURE_FILE} from './fixture.js'; 4 | 5 | const createNodeStream = encoding => ({ 6 | start: () => ({stream: createReadStream(FIXTURE_FILE, encoding)}), 7 | stop() {}, 8 | }); 9 | 10 | export const createNodeStreamBinary = createNodeStream(undefined); 11 | export const createNodeStreamText = createNodeStream('utf8'); 12 | 13 | const createWebStream = type => ({ 14 | async start() { 15 | const fileHandle = await open(FIXTURE_FILE); 16 | const stream = fileHandle.readableWebStream({type}); 17 | return {fileHandle, stream}; 18 | }, 19 | async stop({fileHandle}) { 20 | await fileHandle.close(); 21 | }, 22 | }); 23 | 24 | export const createWebStreamBinary = createWebStream('bytes'); 25 | // `Text` is somewhat of a misnomer here: 26 | // - `fs.readableWebStream({ type: 'bytes' })` creates a `ReadableStream` with a "bytes controller" and `Uint8Array` chunks 27 | // - `fs.readableWebStream({ type: undefined })` creates a `ReadableStream` with a "default controller" and `ArrayBuffer` chunks. 28 | // Node.js currently does not allow creating a file-based `ReadableStream` with string chunks. 29 | export const createWebStreamText = createWebStream(undefined); 30 | -------------------------------------------------------------------------------- /fixture: -------------------------------------------------------------------------------- 1 | unicorn 2 | -------------------------------------------------------------------------------- /license: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Sindre Sorhus (https://sindresorhus.com) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "get-stream", 3 | "version": "9.0.1", 4 | "description": "Get a stream as a string, Buffer, ArrayBuffer or array", 5 | "license": "MIT", 6 | "repository": "sindresorhus/get-stream", 7 | "funding": "https://github.com/sponsors/sindresorhus", 8 | "author": { 9 | "name": "Sindre Sorhus", 10 | "email": "sindresorhus@gmail.com", 11 | "url": "https://sindresorhus.com" 12 | }, 13 | "type": "module", 14 | "exports": { 15 | "types": "./source/index.d.ts", 16 | "browser": "./source/exports.js", 17 | "default": "./source/index.js" 18 | }, 19 | "sideEffects": false, 20 | "engines": { 21 | "node": ">=18" 22 | }, 23 | "scripts": { 24 | "benchmark": "node benchmarks/index.js", 25 | "test": "xo && ava && tsd --typings=source/index.d.ts --files=source/index.test-d.ts" 26 | }, 27 | "files": [ 28 | "source", 29 | "!*.test-d.ts" 30 | ], 31 | "keywords": [ 32 | "get", 33 | "stream", 34 | "promise", 35 | "concat", 36 | "string", 37 | "text", 38 | "buffer", 39 | "read", 40 | "data", 41 | "consume", 42 | "readable", 43 | "readablestream", 44 | "object", 45 | "concat" 46 | ], 47 | "dependencies": { 48 | "@sec-ant/readable-stream": "^0.6.0", 49 | "is-stream": "^4.0.1" 50 | }, 51 | "devDependencies": { 52 | "@types/node": "^20.8.9", 53 | "ava": "^6.1.2", 54 | "onetime": "^7.0.0", 55 | "precise-now": "^3.0.0", 56 | "stream-json": "^1.8.0", 57 | "tsd": "^0.31.0", 58 | "xo": "^0.59.0" 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # get-stream 2 | 3 | > Get a stream as a string, Buffer, ArrayBuffer or array 4 | 5 | ## Features 6 | 7 | - Works in any JavaScript environment ([Node.js](#nodejs-streams), [browsers](#browser-support), etc.). 8 | - Supports [text streams](#getstreamstream-options), [binary streams](#getstreamasbufferstream-options) and [object streams](#getstreamasarraystream-options). 9 | - Supports [async iterables](#async-iterables). 10 | - Can set a [maximum stream size](#maxbuffer). 11 | - Returns [partially read data](#errors) when the stream errors. 12 | - [Fast](#benchmarks). 13 | 14 | ## Install 15 | 16 | ```sh 17 | npm install get-stream 18 | ``` 19 | 20 | ## Usage 21 | 22 | ### Node.js streams 23 | 24 | ```js 25 | import fs from 'node:fs'; 26 | import getStream from 'get-stream'; 27 | 28 | const stream = fs.createReadStream('unicorn.txt'); 29 | 30 | console.log(await getStream(stream)); 31 | /* 32 | ,,))))))));, 33 | __)))))))))))))), 34 | \|/ -\(((((''''((((((((. 35 | -*-==//////(('' . `)))))), 36 | /|\ ))| o ;-. '((((( ,(, 37 | ( `| / ) ;))))' ,_))^;(~ 38 | | | | ,))((((_ _____------~~~-. %,;(;(>';'~ 39 | o_); ; )))(((` ~---~ `:: \ %%~~)(v;(`('~ 40 | ; ''''```` `: `:::|\,__,%% );`'; ~ 41 | | _ ) / `:|`----' `-' 42 | ______/\/~ | / / 43 | /~;;.____/;;' / ___--,-( `;;;/ 44 | / // _;______;'------~~~~~ /;;/\ / 45 | // | | / ; \;;,\ 46 | (<_ | ; /',/-----' _> 47 | \_| ||_ //~;~~~~~~~~~ 48 | `\_| (,~~ 49 | \~\ 50 | ~~ 51 | */ 52 | ``` 53 | 54 | ### Web streams 55 | 56 | ```js 57 | import getStream from 'get-stream'; 58 | 59 | const {body: readableStream} = await fetch('https://example.com'); 60 | console.log(await getStream(readableStream)); 61 | ``` 62 | 63 | This works in any browser, even [the ones](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream#browser_compatibility) not supporting `ReadableStream.values()` yet. 64 | 65 | ### Async iterables 66 | 67 | ```js 68 | import {opendir} from 'node:fs/promises'; 69 | import {getStreamAsArray} from 'get-stream'; 70 | 71 | const asyncIterable = await opendir(directory); 72 | console.log(await getStreamAsArray(asyncIterable)); 73 | ``` 74 | 75 | ## API 76 | 77 | The following methods read the stream's contents and return it as a promise. 78 | 79 | ### getStream(stream, options?) 80 | 81 | `stream`: [`stream.Readable`](https://nodejs.org/api/stream.html#class-streamreadable), [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream), or [`AsyncIterable`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols#the_async_iterator_and_async_iterable_protocols)\ 82 | `options`: [`Options`](#options) 83 | 84 | Get the given `stream` as a string. 85 | 86 | ### getStreamAsBuffer(stream, options?) 87 | 88 | Get the given `stream` as a Node.js [`Buffer`](https://nodejs.org/api/buffer.html#class-buffer). 89 | 90 | ```js 91 | import {getStreamAsBuffer} from 'get-stream'; 92 | 93 | const stream = fs.createReadStream('unicorn.png'); 94 | console.log(await getStreamAsBuffer(stream)); 95 | ``` 96 | 97 | ### getStreamAsArrayBuffer(stream, options?) 98 | 99 | Get the given `stream` as an [`ArrayBuffer`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer). 100 | 101 | ```js 102 | import {getStreamAsArrayBuffer} from 'get-stream'; 103 | 104 | const {body: readableStream} = await fetch('https://example.com'); 105 | console.log(await getStreamAsArrayBuffer(readableStream)); 106 | ``` 107 | 108 | ### getStreamAsArray(stream, options?) 109 | 110 | Get the given `stream` as an array. Unlike [other methods](#api), this supports [streams of objects](https://nodejs.org/api/stream.html#object-mode). 111 | 112 | ```js 113 | import {getStreamAsArray} from 'get-stream'; 114 | 115 | const {body: readableStream} = await fetch('https://example.com'); 116 | console.log(await getStreamAsArray(readableStream)); 117 | ``` 118 | 119 | #### options 120 | 121 | Type: `object` 122 | 123 | ##### maxBuffer 124 | 125 | Type: `number`\ 126 | Default: `Infinity` 127 | 128 | Maximum length of the stream. If exceeded, the promise will be rejected with a `MaxBufferError`. 129 | 130 | Depending on the [method](#api), the length is measured with [`string.length`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/length), [`buffer.length`](https://nodejs.org/api/buffer.html#buflength), [`arrayBuffer.byteLength`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer/byteLength) or [`array.length`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/length). 131 | 132 | ## Errors 133 | 134 | If the stream errors, the returned promise will be rejected with the `error`. Any contents already read from the stream will be set to `error.bufferedData`, which is a `string`, a `Buffer`, an `ArrayBuffer` or an array depending on the [method used](#api). 135 | 136 | ```js 137 | import getStream from 'get-stream'; 138 | 139 | try { 140 | await getStream(streamThatErrorsAtTheEnd('unicorn')); 141 | } catch (error) { 142 | console.log(error.bufferedData); 143 | //=> 'unicorn' 144 | } 145 | ``` 146 | 147 | ## Browser support 148 | 149 | For this module to work in browsers, a bundler must be used that either: 150 | - Supports the [`exports.browser`](https://nodejs.org/api/packages.html#community-conditions-definitions) field in `package.json` 151 | - Strips or ignores `node:*` imports 152 | 153 | Most bundlers (such as [Webpack](https://webpack.js.org/guides/package-exports/#target-environment)) support either of these. 154 | 155 | Additionally, browsers support [web streams](#web-streams) and [async iterables](#async-iterables), but not [Node.js streams](#nodejs-streams). 156 | 157 | ## Tips 158 | 159 | ### Alternatives 160 | 161 | If you do not need the [`maxBuffer`](#maxbuffer) option, [`error.bufferedData`](#errors), nor browser support, you can use the following methods instead of this package. 162 | 163 | #### [`streamConsumers.text()`](https://nodejs.org/api/webstreams.html#streamconsumerstextstream) 164 | 165 | ```js 166 | import fs from 'node:fs'; 167 | import {text} from 'node:stream/consumers'; 168 | 169 | const stream = fs.createReadStream('unicorn.txt', {encoding: 'utf8'}); 170 | console.log(await text(stream)) 171 | ``` 172 | 173 | #### [`streamConsumers.buffer()`](https://nodejs.org/api/webstreams.html#streamconsumersbufferstream) 174 | 175 | ```js 176 | import {buffer} from 'node:stream/consumers'; 177 | 178 | console.log(await buffer(stream)) 179 | ``` 180 | 181 | #### [`streamConsumers.arrayBuffer()`](https://nodejs.org/api/webstreams.html#streamconsumersarraybufferstream) 182 | 183 | ```js 184 | import {arrayBuffer} from 'node:stream/consumers'; 185 | 186 | console.log(await arrayBuffer(stream)) 187 | ``` 188 | 189 | #### [`readable.toArray()`](https://nodejs.org/api/stream.html#readabletoarrayoptions) 190 | 191 | ```js 192 | console.log(await stream.toArray()) 193 | ``` 194 | 195 | #### [`Array.fromAsync()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/fromAsync) 196 | 197 | If your [environment supports it](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/fromAsync#browser_compatibility): 198 | 199 | ```js 200 | console.log(await Array.fromAsync(stream)) 201 | ``` 202 | 203 | ### Non-UTF-8 encoding 204 | 205 | When all of the following conditions apply: 206 | - [`getStream()`](#getstreamstream-options) is used (as opposed to [`getStreamAsBuffer()`](#getstreamasbufferstream-options) or [`getStreamAsArrayBuffer()`](#getstreamasarraybufferstream-options)) 207 | - The stream is binary (not text) 208 | - The stream's encoding is not UTF-8 (for example, it is UTF-16, hexadecimal, or Base64) 209 | 210 | Then the stream must be decoded using a transform stream like [`TextDecoderStream`](https://developer.mozilla.org/en-US/docs/Web/API/TextDecoderStream) or [`b64`](https://github.com/hapijs/b64). 211 | 212 | ```js 213 | import getStream from 'get-stream'; 214 | 215 | const textDecoderStream = new TextDecoderStream('utf-16le'); 216 | const {body: readableStream} = await fetch('https://example.com'); 217 | console.log(await getStream(readableStream.pipeThrough(textDecoderStream))); 218 | ``` 219 | 220 | ### Blobs 221 | 222 | [`getStreamAsArrayBuffer()`](#getstreamasarraybufferstream-options) can be used to create [Blobs](https://developer.mozilla.org/en-US/docs/Web/API/Blob). 223 | 224 | ```js 225 | import {getStreamAsArrayBuffer} from 'get-stream'; 226 | 227 | const stream = fs.createReadStream('unicorn.txt'); 228 | console.log(new Blob([await getStreamAsArrayBuffer(stream)])); 229 | ``` 230 | 231 | ### JSON streaming 232 | 233 | [`getStreamAsArray()`](#getstreamasarraystream-options) can be combined with JSON streaming utilities to parse JSON incrementally. 234 | 235 | ```js 236 | import fs from 'node:fs'; 237 | import {compose as composeStreams} from 'node:stream'; 238 | import {getStreamAsArray} from 'get-stream'; 239 | import streamJson from 'stream-json'; 240 | import streamJsonArray from 'stream-json/streamers/StreamArray.js'; 241 | 242 | const stream = fs.createReadStream('big-array-of-objects.json'); 243 | console.log(await getStreamAsArray( 244 | composeStreams(stream, streamJson.parser(), streamJsonArray.streamArray()), 245 | )); 246 | ``` 247 | 248 | ## Benchmarks 249 | 250 | ### Node.js stream (100 MB, binary) 251 | 252 | - `getStream()`: 142ms 253 | - `text()`: 139ms 254 | - `getStreamAsBuffer()`: 106ms 255 | - `buffer()`: 83ms 256 | - `getStreamAsArrayBuffer()`: 105ms 257 | - `arrayBuffer()`: 81ms 258 | - `getStreamAsArray()`: 24ms 259 | - `stream.toArray()`: 21ms 260 | 261 | ### Node.js stream (100 MB, text) 262 | 263 | - `getStream()`: 90ms 264 | - `text()`: 89ms 265 | - `getStreamAsBuffer()`: 127ms 266 | - `buffer()`: 192ms 267 | - `getStreamAsArrayBuffer()`: 129ms 268 | - `arrayBuffer()`: 195ms 269 | - `getStreamAsArray()`: 89ms 270 | - `stream.toArray()`: 90ms 271 | 272 | ### Web ReadableStream (100 MB, binary) 273 | 274 | - `getStream()`: 223ms 275 | - `text()`: 221ms 276 | - `getStreamAsBuffer()`: 182ms 277 | - `buffer()`: 153ms 278 | - `getStreamAsArrayBuffer()`: 171ms 279 | - `arrayBuffer()`: 155ms 280 | - `getStreamAsArray()`: 83ms 281 | 282 | ### Web ReadableStream (100 MB, text) 283 | 284 | - `getStream()`: 141ms 285 | - `text()`: 139ms 286 | - `getStreamAsBuffer()`: 91ms 287 | - `buffer()`: 80ms 288 | - `getStreamAsArrayBuffer()`: 89ms 289 | - `arrayBuffer()`: 81ms 290 | - `getStreamAsArray()`: 21ms 291 | 292 | [Benchmarks' source file](benchmarks/index.js). 293 | 294 | ## FAQ 295 | 296 | ### How is this different from [`concat-stream`](https://github.com/maxogden/concat-stream)? 297 | 298 | This module accepts a stream instead of being one and returns a promise instead of using a callback. The API is simpler and it only supports returning a string, `Buffer`, an `ArrayBuffer` or an array. It doesn't have a fragile type inference. You explicitly choose what you want. And it doesn't depend on the huge `readable-stream` package. 299 | 300 | ## Related 301 | 302 | - [get-stdin](https://github.com/sindresorhus/get-stdin) - Get stdin as a string or buffer 303 | - [into-stream](https://github.com/sindresorhus/into-stream) - The opposite of this package 304 | -------------------------------------------------------------------------------- /source/array-buffer.js: -------------------------------------------------------------------------------- 1 | import {getStreamContents} from './contents.js'; 2 | import {noop, throwObjectStream, getLengthProperty} from './utils.js'; 3 | 4 | export async function getStreamAsArrayBuffer(stream, options) { 5 | return getStreamContents(stream, arrayBufferMethods, options); 6 | } 7 | 8 | const initArrayBuffer = () => ({contents: new ArrayBuffer(0)}); 9 | 10 | const useTextEncoder = chunk => textEncoder.encode(chunk); 11 | const textEncoder = new TextEncoder(); 12 | 13 | const useUint8Array = chunk => new Uint8Array(chunk); 14 | 15 | const useUint8ArrayWithOffset = chunk => new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength); 16 | 17 | const truncateArrayBufferChunk = (convertedChunk, chunkSize) => convertedChunk.slice(0, chunkSize); 18 | 19 | // `contents` is an increasingly growing `Uint8Array`. 20 | const addArrayBufferChunk = (convertedChunk, {contents, length: previousLength}, length) => { 21 | const newContents = hasArrayBufferResize() ? resizeArrayBuffer(contents, length) : resizeArrayBufferSlow(contents, length); 22 | new Uint8Array(newContents).set(convertedChunk, previousLength); 23 | return newContents; 24 | }; 25 | 26 | // Without `ArrayBuffer.resize()`, `contents` size is always a power of 2. 27 | // This means its last bytes are zeroes (not stream data), which need to be 28 | // trimmed at the end with `ArrayBuffer.slice()`. 29 | const resizeArrayBufferSlow = (contents, length) => { 30 | if (length <= contents.byteLength) { 31 | return contents; 32 | } 33 | 34 | const arrayBuffer = new ArrayBuffer(getNewContentsLength(length)); 35 | new Uint8Array(arrayBuffer).set(new Uint8Array(contents), 0); 36 | return arrayBuffer; 37 | }; 38 | 39 | // With `ArrayBuffer.resize()`, `contents` size matches exactly the size of 40 | // the stream data. It does not include extraneous zeroes to trim at the end. 41 | // The underlying `ArrayBuffer` does allocate a number of bytes that is a power 42 | // of 2, but those bytes are only visible after calling `ArrayBuffer.resize()`. 43 | const resizeArrayBuffer = (contents, length) => { 44 | if (length <= contents.maxByteLength) { 45 | contents.resize(length); 46 | return contents; 47 | } 48 | 49 | // eslint-disable-next-line n/no-unsupported-features/es-syntax 50 | const arrayBuffer = new ArrayBuffer(length, {maxByteLength: getNewContentsLength(length)}); 51 | new Uint8Array(arrayBuffer).set(new Uint8Array(contents), 0); 52 | return arrayBuffer; 53 | }; 54 | 55 | // Retrieve the closest `length` that is both >= and a power of 2 56 | const getNewContentsLength = length => SCALE_FACTOR ** Math.ceil(Math.log(length) / Math.log(SCALE_FACTOR)); 57 | 58 | const SCALE_FACTOR = 2; 59 | 60 | const finalizeArrayBuffer = ({contents, length}) => hasArrayBufferResize() ? contents : contents.slice(0, length); 61 | 62 | // `ArrayBuffer.slice()` is slow. When `ArrayBuffer.resize()` is available 63 | // (Node >=20.0.0, Safari >=16.4 and Chrome), we can use it instead. 64 | // eslint-disable-next-line no-warning-comments 65 | // TODO: remove after dropping support for Node 20. 66 | // eslint-disable-next-line no-warning-comments 67 | // TODO: use `ArrayBuffer.transferToFixedLength()` instead once it is available 68 | const hasArrayBufferResize = () => 'resize' in ArrayBuffer.prototype; 69 | 70 | const arrayBufferMethods = { 71 | init: initArrayBuffer, 72 | convertChunk: { 73 | string: useTextEncoder, 74 | buffer: useUint8Array, 75 | arrayBuffer: useUint8Array, 76 | dataView: useUint8ArrayWithOffset, 77 | typedArray: useUint8ArrayWithOffset, 78 | others: throwObjectStream, 79 | }, 80 | getSize: getLengthProperty, 81 | truncateChunk: truncateArrayBufferChunk, 82 | addChunk: addArrayBufferChunk, 83 | getFinalChunk: noop, 84 | finalize: finalizeArrayBuffer, 85 | }; 86 | -------------------------------------------------------------------------------- /source/array.js: -------------------------------------------------------------------------------- 1 | import {getStreamContents} from './contents.js'; 2 | import {identity, noop, getContentsProperty} from './utils.js'; 3 | 4 | export async function getStreamAsArray(stream, options) { 5 | return getStreamContents(stream, arrayMethods, options); 6 | } 7 | 8 | const initArray = () => ({contents: []}); 9 | 10 | const increment = () => 1; 11 | 12 | const addArrayChunk = (convertedChunk, {contents}) => { 13 | contents.push(convertedChunk); 14 | return contents; 15 | }; 16 | 17 | const arrayMethods = { 18 | init: initArray, 19 | convertChunk: { 20 | string: identity, 21 | buffer: identity, 22 | arrayBuffer: identity, 23 | dataView: identity, 24 | typedArray: identity, 25 | others: identity, 26 | }, 27 | getSize: increment, 28 | truncateChunk: noop, 29 | addChunk: addArrayChunk, 30 | getFinalChunk: noop, 31 | finalize: getContentsProperty, 32 | }; 33 | -------------------------------------------------------------------------------- /source/buffer.js: -------------------------------------------------------------------------------- 1 | import {getStreamAsArrayBuffer} from './array-buffer.js'; 2 | 3 | export async function getStreamAsBuffer(stream, options) { 4 | if (!('Buffer' in globalThis)) { 5 | throw new Error('getStreamAsBuffer() is only supported in Node.js'); 6 | } 7 | 8 | try { 9 | return arrayBufferToNodeBuffer(await getStreamAsArrayBuffer(stream, options)); 10 | } catch (error) { 11 | if (error.bufferedData !== undefined) { 12 | error.bufferedData = arrayBufferToNodeBuffer(error.bufferedData); 13 | } 14 | 15 | throw error; 16 | } 17 | } 18 | 19 | const arrayBufferToNodeBuffer = arrayBuffer => globalThis.Buffer.from(arrayBuffer); 20 | -------------------------------------------------------------------------------- /source/contents.js: -------------------------------------------------------------------------------- 1 | import {getAsyncIterable} from './stream.js'; 2 | 3 | export const getStreamContents = async (stream, {init, convertChunk, getSize, truncateChunk, addChunk, getFinalChunk, finalize}, {maxBuffer = Number.POSITIVE_INFINITY} = {}) => { 4 | const asyncIterable = getAsyncIterable(stream); 5 | 6 | const state = init(); 7 | state.length = 0; 8 | 9 | try { 10 | for await (const chunk of asyncIterable) { 11 | const chunkType = getChunkType(chunk); 12 | const convertedChunk = convertChunk[chunkType](chunk, state); 13 | appendChunk({ 14 | convertedChunk, 15 | state, 16 | getSize, 17 | truncateChunk, 18 | addChunk, 19 | maxBuffer, 20 | }); 21 | } 22 | 23 | appendFinalChunk({ 24 | state, 25 | convertChunk, 26 | getSize, 27 | truncateChunk, 28 | addChunk, 29 | getFinalChunk, 30 | maxBuffer, 31 | }); 32 | return finalize(state); 33 | } catch (error) { 34 | const normalizedError = typeof error === 'object' && error !== null ? error : new Error(error); 35 | normalizedError.bufferedData = finalize(state); 36 | throw normalizedError; 37 | } 38 | }; 39 | 40 | const appendFinalChunk = ({state, getSize, truncateChunk, addChunk, getFinalChunk, maxBuffer}) => { 41 | const convertedChunk = getFinalChunk(state); 42 | if (convertedChunk !== undefined) { 43 | appendChunk({ 44 | convertedChunk, 45 | state, 46 | getSize, 47 | truncateChunk, 48 | addChunk, 49 | maxBuffer, 50 | }); 51 | } 52 | }; 53 | 54 | const appendChunk = ({convertedChunk, state, getSize, truncateChunk, addChunk, maxBuffer}) => { 55 | const chunkSize = getSize(convertedChunk); 56 | const newLength = state.length + chunkSize; 57 | 58 | if (newLength <= maxBuffer) { 59 | addNewChunk(convertedChunk, state, addChunk, newLength); 60 | return; 61 | } 62 | 63 | const truncatedChunk = truncateChunk(convertedChunk, maxBuffer - state.length); 64 | 65 | if (truncatedChunk !== undefined) { 66 | addNewChunk(truncatedChunk, state, addChunk, maxBuffer); 67 | } 68 | 69 | throw new MaxBufferError(); 70 | }; 71 | 72 | const addNewChunk = (convertedChunk, state, addChunk, newLength) => { 73 | state.contents = addChunk(convertedChunk, state, newLength); 74 | state.length = newLength; 75 | }; 76 | 77 | const getChunkType = chunk => { 78 | const typeOfChunk = typeof chunk; 79 | 80 | if (typeOfChunk === 'string') { 81 | return 'string'; 82 | } 83 | 84 | if (typeOfChunk !== 'object' || chunk === null) { 85 | return 'others'; 86 | } 87 | 88 | if (globalThis.Buffer?.isBuffer(chunk)) { 89 | return 'buffer'; 90 | } 91 | 92 | const prototypeName = objectToString.call(chunk); 93 | 94 | if (prototypeName === '[object ArrayBuffer]') { 95 | return 'arrayBuffer'; 96 | } 97 | 98 | if (prototypeName === '[object DataView]') { 99 | return 'dataView'; 100 | } 101 | 102 | if ( 103 | Number.isInteger(chunk.byteLength) 104 | && Number.isInteger(chunk.byteOffset) 105 | && objectToString.call(chunk.buffer) === '[object ArrayBuffer]' 106 | ) { 107 | return 'typedArray'; 108 | } 109 | 110 | return 'others'; 111 | }; 112 | 113 | const {toString: objectToString} = Object.prototype; 114 | 115 | export class MaxBufferError extends Error { 116 | name = 'MaxBufferError'; 117 | 118 | constructor() { 119 | super('maxBuffer exceeded'); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /source/exports.js: -------------------------------------------------------------------------------- 1 | export {getStreamAsArray} from './array.js'; 2 | export {getStreamAsArrayBuffer} from './array-buffer.js'; 3 | export {getStreamAsBuffer} from './buffer.js'; 4 | export {getStreamAsString as default} from './string.js'; 5 | export {MaxBufferError} from './contents.js'; 6 | -------------------------------------------------------------------------------- /source/index.d.ts: -------------------------------------------------------------------------------- 1 | import {type Readable} from 'node:stream'; 2 | import {type Buffer} from 'node:buffer'; 3 | 4 | export class MaxBufferError extends Error { 5 | readonly name: 'MaxBufferError'; 6 | constructor(); 7 | } 8 | 9 | // eslint-disable-next-line @typescript-eslint/ban-types 10 | type TextStreamItem = string | Buffer | ArrayBuffer | ArrayBufferView; 11 | 12 | export type AnyStream = Readable | ReadableStream | AsyncIterable; 13 | 14 | export type Options = { 15 | /** 16 | Maximum length of the stream. If exceeded, the promise will be rejected with a `MaxBufferError`. 17 | 18 | Depending on the [method](#api), the length is measured with [`string.length`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/length), [`buffer.length`](https://nodejs.org/api/buffer.html#buflength), [`arrayBuffer.byteLength`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer/byteLength) or [`array.length`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/length). 19 | 20 | @default Infinity 21 | */ 22 | readonly maxBuffer?: number; 23 | }; 24 | 25 | /** 26 | Get the given `stream` as a string. 27 | 28 | @returns The stream's contents as a promise. 29 | 30 | @example 31 | ``` 32 | import fs from 'node:fs'; 33 | import getStream from 'get-stream'; 34 | 35 | const stream = fs.createReadStream('unicorn.txt'); 36 | 37 | console.log(await getStream(stream)); 38 | // ,,))))))));, 39 | // __)))))))))))))), 40 | // \|/ -\(((((''''((((((((. 41 | // -*-==//////(('' . `)))))), 42 | // /|\ ))| o ;-. '((((( ,(, 43 | // ( `| / ) ;))))' ,_))^;(~ 44 | // | | | ,))((((_ _____------~~~-. %,;(;(>';'~ 45 | // o_); ; )))(((` ~---~ `:: \ %%~~)(v;(`('~ 46 | // ; ''''```` `: `:::|\,__,%% );`'; ~ 47 | // | _ ) / `:|`----' `-' 48 | // ______/\/~ | / / 49 | // /~;;.____/;;' / ___--,-( `;;;/ 50 | // / // _;______;'------~~~~~ /;;/\ / 51 | // // | | / ; \;;,\ 52 | // (<_ | ; /',/-----' _> 53 | // \_| ||_ //~;~~~~~~~~~ 54 | // `\_| (,~~ 55 | // \~\ 56 | // ~~ 57 | ``` 58 | 59 | @example 60 | ``` 61 | import getStream from 'get-stream'; 62 | 63 | const {body: readableStream} = await fetch('https://example.com'); 64 | console.log(await getStream(readableStream)); 65 | ``` 66 | 67 | @example 68 | ``` 69 | import {opendir} from 'node:fs/promises'; 70 | import {getStreamAsArray} from 'get-stream'; 71 | 72 | const asyncIterable = await opendir(directory); 73 | console.log(await getStreamAsArray(asyncIterable)); 74 | ``` 75 | */ 76 | export default function getStream(stream: AnyStream, options?: Options): Promise; 77 | 78 | /** 79 | Get the given `stream` as a Node.js [`Buffer`](https://nodejs.org/api/buffer.html#class-buffer). 80 | 81 | @returns The stream's contents as a promise. 82 | 83 | @example 84 | ``` 85 | import {getStreamAsBuffer} from 'get-stream'; 86 | 87 | const stream = fs.createReadStream('unicorn.png'); 88 | console.log(await getStreamAsBuffer(stream)); 89 | ``` 90 | */ 91 | // eslint-disable-next-line @typescript-eslint/ban-types 92 | export function getStreamAsBuffer(stream: AnyStream, options?: Options): Promise; 93 | 94 | /** 95 | Get the given `stream` as an [`ArrayBuffer`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer). 96 | 97 | @returns The stream's contents as a promise. 98 | 99 | @example 100 | ``` 101 | import {getStreamAsArrayBuffer} from 'get-stream'; 102 | 103 | const {body: readableStream} = await fetch('https://example.com'); 104 | console.log(await getStreamAsArrayBuffer(readableStream)); 105 | ``` 106 | */ 107 | export function getStreamAsArrayBuffer(stream: AnyStream, options?: Options): Promise; 108 | 109 | /** 110 | Get the given `stream` as an array. Unlike [other methods](#api), this supports [streams of objects](https://nodejs.org/api/stream.html#object-mode). 111 | 112 | @returns The stream's contents as a promise. 113 | 114 | @example 115 | ``` 116 | import {getStreamAsArray} from 'get-stream'; 117 | 118 | const {body: readableStream} = await fetch('https://example.com'); 119 | console.log(await getStreamAsArray(readableStream)); 120 | ``` 121 | */ 122 | export function getStreamAsArray(stream: AnyStream, options?: Options): Promise; 123 | -------------------------------------------------------------------------------- /source/index.js: -------------------------------------------------------------------------------- 1 | import {on} from 'node:events'; 2 | import {finished} from 'node:stream/promises'; 3 | import {nodeImports} from './stream.js'; 4 | 5 | Object.assign(nodeImports, {on, finished}); 6 | 7 | export { 8 | default, 9 | getStreamAsArray, 10 | getStreamAsArrayBuffer, 11 | getStreamAsBuffer, 12 | MaxBufferError, 13 | } from './exports.js'; 14 | -------------------------------------------------------------------------------- /source/index.test-d.ts: -------------------------------------------------------------------------------- 1 | import {Buffer} from 'node:buffer'; 2 | import {open} from 'node:fs/promises'; 3 | import {type Readable} from 'node:stream'; 4 | import fs from 'node:fs'; 5 | import { 6 | expectType, 7 | expectError, 8 | expectAssignable, 9 | expectNotAssignable, 10 | } from 'tsd'; 11 | import getStream, { 12 | getStreamAsBuffer, 13 | getStreamAsArrayBuffer, 14 | getStreamAsArray, 15 | MaxBufferError, 16 | type Options, 17 | type AnyStream, 18 | } from './index.js'; 19 | 20 | const nodeStream = fs.createReadStream('foo') as Readable; 21 | 22 | const fileHandle = await open('test'); 23 | const readableStream = fileHandle.readableWebStream(); 24 | 25 | const asyncIterable = (value: T): AsyncGenerator => (async function * () { 26 | yield value; 27 | })(); 28 | const stringAsyncIterable = asyncIterable(''); 29 | const bufferAsyncIterable = asyncIterable(Buffer.from('')); 30 | const arrayBufferAsyncIterable = asyncIterable(new ArrayBuffer(0)); 31 | const dataViewAsyncIterable = asyncIterable(new DataView(new ArrayBuffer(0))); 32 | const typedArrayAsyncIterable = asyncIterable(new Uint8Array([])); 33 | const objectItem = {test: true}; 34 | const objectAsyncIterable = asyncIterable(objectItem); 35 | 36 | expectType(await getStream(nodeStream)); 37 | expectType(await getStream(nodeStream, {maxBuffer: 10})); 38 | expectType(await getStream(readableStream)); 39 | expectType(await getStream(stringAsyncIterable)); 40 | expectType(await getStream(bufferAsyncIterable)); 41 | expectType(await getStream(arrayBufferAsyncIterable)); 42 | expectType(await getStream(dataViewAsyncIterable)); 43 | expectType(await getStream(typedArrayAsyncIterable)); 44 | expectError(await getStream(objectAsyncIterable)); 45 | expectError(await getStream({})); 46 | expectError(await getStream(nodeStream, {maxBuffer: '10'})); 47 | expectError(await getStream(nodeStream, {unknownOption: 10})); 48 | expectError(await getStream(nodeStream, {maxBuffer: 10}, {})); 49 | 50 | /* eslint-disable @typescript-eslint/ban-types */ 51 | expectType(await getStreamAsBuffer(nodeStream)); 52 | expectType(await getStreamAsBuffer(nodeStream, {maxBuffer: 10})); 53 | expectType(await getStreamAsBuffer(readableStream)); 54 | expectType(await getStreamAsBuffer(stringAsyncIterable)); 55 | expectType(await getStreamAsBuffer(bufferAsyncIterable)); 56 | expectType(await getStreamAsBuffer(arrayBufferAsyncIterable)); 57 | expectType(await getStreamAsBuffer(dataViewAsyncIterable)); 58 | expectType(await getStreamAsBuffer(typedArrayAsyncIterable)); 59 | /* eslint-enable @typescript-eslint/ban-types */ 60 | expectError(await getStreamAsBuffer(objectAsyncIterable)); 61 | expectError(await getStreamAsBuffer({})); 62 | expectError(await getStreamAsBuffer(nodeStream, {maxBuffer: '10'})); 63 | expectError(await getStreamAsBuffer(nodeStream, {unknownOption: 10})); 64 | expectError(await getStreamAsBuffer(nodeStream, {maxBuffer: 10}, {})); 65 | 66 | expectType(await getStreamAsArrayBuffer(nodeStream)); 67 | expectType(await getStreamAsArrayBuffer(nodeStream, {maxBuffer: 10})); 68 | expectType(await getStreamAsArrayBuffer(readableStream)); 69 | expectType(await getStreamAsArrayBuffer(stringAsyncIterable)); 70 | expectType(await getStreamAsArrayBuffer(bufferAsyncIterable)); 71 | expectType(await getStreamAsArrayBuffer(arrayBufferAsyncIterable)); 72 | expectType(await getStreamAsArrayBuffer(dataViewAsyncIterable)); 73 | expectType(await getStreamAsArrayBuffer(typedArrayAsyncIterable)); 74 | expectError(await getStreamAsArrayBuffer(objectAsyncIterable)); 75 | expectError(await getStreamAsArrayBuffer({})); 76 | expectError(await getStreamAsArrayBuffer(nodeStream, {maxBuffer: '10'})); 77 | expectError(await getStreamAsArrayBuffer(nodeStream, {unknownOption: 10})); 78 | expectError(await getStreamAsArrayBuffer(nodeStream, {maxBuffer: 10}, {})); 79 | 80 | expectType(await getStreamAsArray(nodeStream)); 81 | expectType(await getStreamAsArray(nodeStream, {maxBuffer: 10})); 82 | expectType(await getStreamAsArray(readableStream)); 83 | expectType(await getStreamAsArray(readableStream as ReadableStream)); 84 | expectType(await getStreamAsArray(stringAsyncIterable)); 85 | // eslint-disable-next-line @typescript-eslint/ban-types 86 | expectType(await getStreamAsArray(bufferAsyncIterable)); 87 | expectType(await getStreamAsArray(arrayBufferAsyncIterable)); 88 | expectType(await getStreamAsArray(dataViewAsyncIterable)); 89 | expectType(await getStreamAsArray(typedArrayAsyncIterable)); 90 | expectType>(await getStreamAsArray(objectAsyncIterable)); 91 | expectError(await getStreamAsArray({})); 92 | expectError(await getStreamAsArray(nodeStream, {maxBuffer: '10'})); 93 | expectError(await getStreamAsArray(nodeStream, {unknownOption: 10})); 94 | expectError(await getStreamAsArray(nodeStream, {maxBuffer: 10}, {})); 95 | 96 | expectAssignable(nodeStream); 97 | expectAssignable(readableStream); 98 | expectAssignable(stringAsyncIterable); 99 | expectAssignable(bufferAsyncIterable); 100 | expectAssignable(arrayBufferAsyncIterable); 101 | expectAssignable(dataViewAsyncIterable); 102 | expectAssignable(typedArrayAsyncIterable); 103 | expectAssignable>(objectAsyncIterable); 104 | expectNotAssignable(objectAsyncIterable); 105 | expectAssignable>(stringAsyncIterable); 106 | expectNotAssignable>(bufferAsyncIterable); 107 | expectNotAssignable({}); 108 | 109 | expectAssignable({maxBuffer: 10}); 110 | expectNotAssignable({maxBuffer: '10'}); 111 | expectNotAssignable({unknownOption: 10}); 112 | 113 | expectType(new MaxBufferError()); 114 | -------------------------------------------------------------------------------- /source/stream.js: -------------------------------------------------------------------------------- 1 | import {isReadableStream} from 'is-stream'; 2 | import {asyncIterator} from '@sec-ant/readable-stream/ponyfill'; 3 | 4 | export const getAsyncIterable = stream => { 5 | if (isReadableStream(stream, {checkOpen: false}) && nodeImports.on !== undefined) { 6 | return getStreamIterable(stream); 7 | } 8 | 9 | if (typeof stream?.[Symbol.asyncIterator] === 'function') { 10 | return stream; 11 | } 12 | 13 | // `ReadableStream[Symbol.asyncIterator]` support is missing in multiple browsers, so we ponyfill it 14 | if (toString.call(stream) === '[object ReadableStream]') { 15 | return asyncIterator(stream); 16 | } 17 | 18 | throw new TypeError('The first argument must be a Readable, a ReadableStream, or an async iterable.'); 19 | }; 20 | 21 | const {toString} = Object.prototype; 22 | 23 | // The default iterable for Node.js streams does not allow for multiple readers at once, so we re-implement it 24 | const getStreamIterable = async function * (stream) { 25 | const controller = new AbortController(); 26 | const state = {}; 27 | handleStreamEnd(stream, controller, state); 28 | 29 | try { 30 | for await (const [chunk] of nodeImports.on(stream, 'data', {signal: controller.signal})) { 31 | yield chunk; 32 | } 33 | } catch (error) { 34 | // Stream failure, for example due to `stream.destroy(error)` 35 | if (state.error !== undefined) { 36 | throw state.error; 37 | // `error` event directly emitted on stream 38 | } else if (!controller.signal.aborted) { 39 | throw error; 40 | // Otherwise, stream completed successfully 41 | } 42 | // The `finally` block also runs when the caller throws, for example due to the `maxBuffer` option 43 | } finally { 44 | stream.destroy(); 45 | } 46 | }; 47 | 48 | const handleStreamEnd = async (stream, controller, state) => { 49 | try { 50 | await nodeImports.finished(stream, { 51 | cleanup: true, 52 | readable: true, 53 | writable: false, 54 | error: false, 55 | }); 56 | } catch (error) { 57 | state.error = error; 58 | } finally { 59 | controller.abort(); 60 | } 61 | }; 62 | 63 | // Loaded by the Node entrypoint, but not by the browser one. 64 | // This prevents using dynamic imports. 65 | export const nodeImports = {}; 66 | -------------------------------------------------------------------------------- /source/string.js: -------------------------------------------------------------------------------- 1 | import {getStreamContents} from './contents.js'; 2 | import { 3 | identity, 4 | getContentsProperty, 5 | throwObjectStream, 6 | getLengthProperty, 7 | } from './utils.js'; 8 | 9 | export async function getStreamAsString(stream, options) { 10 | return getStreamContents(stream, stringMethods, options); 11 | } 12 | 13 | const initString = () => ({contents: '', textDecoder: new TextDecoder()}); 14 | 15 | const useTextDecoder = (chunk, {textDecoder}) => textDecoder.decode(chunk, {stream: true}); 16 | 17 | const addStringChunk = (convertedChunk, {contents}) => contents + convertedChunk; 18 | 19 | const truncateStringChunk = (convertedChunk, chunkSize) => convertedChunk.slice(0, chunkSize); 20 | 21 | const getFinalStringChunk = ({textDecoder}) => { 22 | const finalChunk = textDecoder.decode(); 23 | return finalChunk === '' ? undefined : finalChunk; 24 | }; 25 | 26 | const stringMethods = { 27 | init: initString, 28 | convertChunk: { 29 | string: identity, 30 | buffer: useTextDecoder, 31 | arrayBuffer: useTextDecoder, 32 | dataView: useTextDecoder, 33 | typedArray: useTextDecoder, 34 | others: throwObjectStream, 35 | }, 36 | getSize: getLengthProperty, 37 | truncateChunk: truncateStringChunk, 38 | addChunk: addStringChunk, 39 | getFinalChunk: getFinalStringChunk, 40 | finalize: getContentsProperty, 41 | }; 42 | -------------------------------------------------------------------------------- /source/utils.js: -------------------------------------------------------------------------------- 1 | export const identity = value => value; 2 | 3 | export const noop = () => undefined; 4 | 5 | export const getContentsProperty = ({contents}) => contents; 6 | 7 | export const throwObjectStream = chunk => { 8 | throw new Error(`Streams in object mode are not supported: ${String(chunk)}`); 9 | }; 10 | 11 | export const getLengthProperty = convertedChunk => convertedChunk.length; 12 | -------------------------------------------------------------------------------- /test/array-buffer.js: -------------------------------------------------------------------------------- 1 | import {Buffer} from 'node:buffer'; 2 | import {arrayBuffer, blob} from 'node:stream/consumers'; 3 | import test from 'ava'; 4 | import {getStreamAsArrayBuffer, MaxBufferError} from '../source/index.js'; 5 | import {createStream} from './helpers/index.js'; 6 | import { 7 | fixtureString, 8 | fixtureLength, 9 | fixtureBuffer, 10 | fixtureTypedArray, 11 | fixtureArrayBuffer, 12 | fixtureUint16Array, 13 | fixtureDataView, 14 | fixtureMultiString, 15 | fixtureMultiBuffer, 16 | fixtureMultiTypedArray, 17 | fixtureMultiArrayBuffer, 18 | fixtureMultiUint16Array, 19 | fixtureMultiDataView, 20 | fixtureTypedArrayWithOffset, 21 | fixtureUint16ArrayWithOffset, 22 | fixtureDataViewWithOffset, 23 | longString, 24 | bigArray, 25 | } from './fixtures/index.js'; 26 | 27 | const longTypedArray = new TextEncoder().encode(longString); 28 | const longArrayBuffer = longTypedArray.buffer; 29 | const longUint16Array = new Uint16Array(longArrayBuffer); 30 | const longDataView = new DataView(longArrayBuffer); 31 | const fixtureMultibyteUint16Array = new Uint16Array([0, 0]); 32 | const longMultibyteUint16Array = new Uint16Array([0, 0, 0]); 33 | const bigArrayBuffer = new Uint8Array(bigArray).buffer; 34 | 35 | const setupArrayBuffer = (streamDefinition, options) => getStreamAsArrayBuffer(createStream(streamDefinition), options); 36 | 37 | const getStreamToArrayBuffer = async (t, fixtureValue) => { 38 | const result = await setupArrayBuffer(fixtureValue); 39 | t.true(result instanceof ArrayBuffer); 40 | t.true(Buffer.from(result).equals(fixtureBuffer)); 41 | }; 42 | 43 | test('get stream from string to arrayBuffer, with a single chunk', getStreamToArrayBuffer, [fixtureString]); 44 | test('get stream from buffer to arrayBuffer, with a single chunk', getStreamToArrayBuffer, [fixtureBuffer]); 45 | test('get stream from arrayBuffer to arrayBuffer, with a single chunk', getStreamToArrayBuffer, [fixtureArrayBuffer]); 46 | test('get stream from typedArray to arrayBuffer, with a single chunk', getStreamToArrayBuffer, [fixtureTypedArray]); 47 | test('get stream from typedArray with offset to arrayBuffer, with a single chunk', getStreamToArrayBuffer, [fixtureTypedArrayWithOffset]); 48 | test('get stream from uint16Array to arrayBuffer, with a single chunk', getStreamToArrayBuffer, [fixtureUint16Array]); 49 | test('get stream from uint16Array with offset to arrayBuffer, with a single chunk', getStreamToArrayBuffer, [fixtureUint16ArrayWithOffset]); 50 | test('get stream from dataView to arrayBuffer, with a single chunk', getStreamToArrayBuffer, [fixtureDataView]); 51 | test('get stream from dataView with offset to arrayBuffer, with a single chunk', getStreamToArrayBuffer, [fixtureDataViewWithOffset]); 52 | 53 | test('get stream from string to arrayBuffer, with multiple chunks', getStreamToArrayBuffer, fixtureMultiString); 54 | test('get stream from buffer to arrayBuffer, with multiple chunks', getStreamToArrayBuffer, fixtureMultiBuffer); 55 | test('get stream from arrayBuffer to arrayBuffer, with multiple chunks', getStreamToArrayBuffer, fixtureMultiArrayBuffer); 56 | test('get stream from typedArray to arrayBuffer, with multiple chunks', getStreamToArrayBuffer, fixtureMultiTypedArray); 57 | test('get stream from uint16Array to arrayBuffer, with multiple chunks', getStreamToArrayBuffer, fixtureMultiUint16Array); 58 | test('get stream from dataView to arrayBuffer, with multiple chunks', getStreamToArrayBuffer, fixtureMultiDataView); 59 | 60 | const throwOnInvalidChunkType = async (t, fixtureValue) => { 61 | await t.throwsAsync(setupArrayBuffer([fixtureValue]), {message: /not supported/}); 62 | }; 63 | 64 | test('get stream from bigint to arrayBuffer', throwOnInvalidChunkType, 0n); 65 | test('get stream from number to arrayBuffer', throwOnInvalidChunkType, 0); 66 | test('get stream from array to arrayBuffer', throwOnInvalidChunkType, []); 67 | test('get stream from object to arrayBuffer', throwOnInvalidChunkType, {}); 68 | test('get stream from boolean to arrayBuffer', throwOnInvalidChunkType, false); 69 | test('get stream from undefined to arrayBuffer', throwOnInvalidChunkType, undefined); 70 | test('get stream from symbol to arrayBuffer', throwOnInvalidChunkType, Symbol('test')); 71 | 72 | const checkMaxBuffer = async (t, longValue, shortValue, maxBuffer) => { 73 | await t.throwsAsync(setupArrayBuffer([longValue], {maxBuffer}), {instanceOf: MaxBufferError}); 74 | await t.notThrowsAsync(setupArrayBuffer([shortValue], {maxBuffer})); 75 | }; 76 | 77 | test('maxBuffer throws when size is exceeded with an arrayBuffer', checkMaxBuffer, longArrayBuffer, fixtureArrayBuffer, fixtureLength); 78 | test('maxBuffer throws when size is exceeded with a typedArray', checkMaxBuffer, longTypedArray, fixtureTypedArray, fixtureLength); 79 | test('maxBuffer throws when size is exceeded with an uint16Array', checkMaxBuffer, longUint16Array, fixtureUint16Array, fixtureLength); 80 | test('maxBuffer throws when size is exceeded with a dataView', checkMaxBuffer, longDataView, fixtureDataView, fixtureLength); 81 | test('maxBuffer unit is bytes with getStreamAsArrayBuffer()', checkMaxBuffer, longMultibyteUint16Array, fixtureMultibyteUint16Array, fixtureMultibyteUint16Array.byteLength); 82 | 83 | const checkBufferedData = async (t, fixtureValue, expectedResult) => { 84 | const maxBuffer = expectedResult.byteLength; 85 | const {bufferedData} = await t.throwsAsync(setupArrayBuffer(fixtureValue, {maxBuffer}), {instanceOf: MaxBufferError}); 86 | t.is(bufferedData.byteLength, maxBuffer); 87 | t.deepEqual(expectedResult, bufferedData); 88 | }; 89 | 90 | test( 91 | 'set error.bufferedData when `maxBuffer` is hit, with a single chunk', 92 | checkBufferedData, 93 | [fixtureArrayBuffer], 94 | new Uint8Array(Buffer.from(fixtureString[0])).buffer, 95 | ); 96 | test( 97 | 'set error.bufferedData when `maxBuffer` is hit, with multiple chunks', 98 | checkBufferedData, 99 | [fixtureArrayBuffer, fixtureArrayBuffer], 100 | new Uint8Array(Buffer.from(`${fixtureString}${fixtureString[0]}`)).buffer, 101 | ); 102 | 103 | test('getStreamAsArrayBuffer() behaves like arrayBuffer()', async t => { 104 | const [nativeResult, customResult] = await Promise.all([ 105 | arrayBuffer(createStream([bigArrayBuffer])), 106 | setupArrayBuffer([bigArrayBuffer]), 107 | ]); 108 | t.deepEqual(nativeResult, customResult); 109 | }); 110 | 111 | test('getStreamAsArrayBuffer() can behave like blob()', async t => { 112 | const [nativeResult, customResult] = await Promise.all([ 113 | blob(createStream([bigArrayBuffer])), 114 | setupArrayBuffer([bigArrayBuffer]), 115 | ]); 116 | t.deepEqual(nativeResult, new Blob([customResult])); 117 | }); 118 | -------------------------------------------------------------------------------- /test/array.js: -------------------------------------------------------------------------------- 1 | import {compose} from 'node:stream'; 2 | import test from 'ava'; 3 | import streamJson from 'stream-json'; 4 | import streamJsonArray from 'stream-json/streamers/StreamArray.js'; 5 | import {getStreamAsArray, MaxBufferError} from '../source/index.js'; 6 | import {createStream, BIG_TEST_DURATION} from './helpers/index.js'; 7 | import { 8 | fixtureString, 9 | fixtureBuffer, 10 | fixtureTypedArray, 11 | fixtureArrayBuffer, 12 | fixtureUint16Array, 13 | fixtureDataView, 14 | fixtureMultiString, 15 | fixtureMultiBuffer, 16 | fixtureMultiTypedArray, 17 | fixtureMultiArrayBuffer, 18 | fixtureMultiUint16Array, 19 | fixtureMultiDataView, 20 | fixtureTypedArrayWithOffset, 21 | fixtureUint16ArrayWithOffset, 22 | fixtureDataViewWithOffset, 23 | bigArray, 24 | } from './fixtures/index.js'; 25 | 26 | const fixtureArray = [{}, {}]; 27 | 28 | const setupArray = (streamDefinition, options) => getStreamAsArray(createStream(streamDefinition), options); 29 | 30 | const getStreamToArray = async (t, fixtureValue) => { 31 | const result = await setupArray(fixtureValue); 32 | t.deepEqual(result, fixtureValue); 33 | }; 34 | 35 | test('get stream from string to array, with a single chunk', getStreamToArray, [fixtureString]); 36 | test('get stream from buffer to array, with a single chunk', getStreamToArray, [fixtureBuffer]); 37 | test('get stream from arrayBuffer to array, with a single chunk', getStreamToArray, [fixtureArrayBuffer]); 38 | test('get stream from typedArray to array, with a single chunk', getStreamToArray, [fixtureTypedArray]); 39 | test('get stream from typedArray with offset to array, with a single chunk', getStreamToArray, [fixtureTypedArrayWithOffset]); 40 | test('get stream from uint16Array to array, with a single chunk', getStreamToArray, [fixtureUint16Array]); 41 | test('get stream from uint16Array with offset to array, with a single chunk', getStreamToArray, [fixtureUint16ArrayWithOffset]); 42 | test('get stream from dataView to array, with a single chunk', getStreamToArray, [fixtureDataView]); 43 | test('get stream from dataView with offset to array, with a single chunk', getStreamToArray, [fixtureDataViewWithOffset]); 44 | 45 | test('get stream from string to array, with multiple chunks', getStreamToArray, fixtureMultiString); 46 | test('get stream from buffer to array, with multiple chunks', getStreamToArray, fixtureMultiBuffer); 47 | test('get stream from arrayBuffer to array, with multiple chunks', getStreamToArray, fixtureMultiArrayBuffer); 48 | test('get stream from typedArray to array, with multiple chunks', getStreamToArray, fixtureMultiTypedArray); 49 | test('get stream from uint16Array to array, with multiple chunks', getStreamToArray, fixtureMultiUint16Array); 50 | test('get stream from dataView to array, with multiple chunks', getStreamToArray, fixtureMultiDataView); 51 | 52 | const allowsAnyChunkType = async (t, fixtureValue) => { 53 | await t.notThrowsAsync(setupArray([fixtureValue])); 54 | }; 55 | 56 | test('get stream from object to array', allowsAnyChunkType, {}); 57 | test('get stream from array to array', allowsAnyChunkType, []); 58 | test('get stream from boolean to array', allowsAnyChunkType, false); 59 | test('get stream from number to array', allowsAnyChunkType, 0); 60 | test('get stream from bigint to array', allowsAnyChunkType, 0n); 61 | test('get stream from undefined to array', allowsAnyChunkType, undefined); 62 | test('get stream from symbol to array', allowsAnyChunkType, Symbol('test')); 63 | 64 | test('maxBuffer unit is each array element with getStreamAsArray()', async t => { 65 | const maxBuffer = fixtureArray.length; 66 | await t.throwsAsync(setupArray([...fixtureArray, ...fixtureArray], {maxBuffer}), {instanceOf: MaxBufferError}); 67 | await t.notThrowsAsync(setupArray(fixtureArray, {maxBuffer})); 68 | }); 69 | 70 | const checkBufferedData = async (t, fixtureValue, expectedResult) => { 71 | const maxBuffer = expectedResult.length; 72 | const {bufferedData} = await t.throwsAsync(setupArray(fixtureValue, {maxBuffer}), {instanceOf: MaxBufferError}); 73 | t.is(bufferedData.length, maxBuffer); 74 | t.deepEqual(expectedResult, bufferedData); 75 | }; 76 | 77 | test( 78 | 'set error.bufferedData when `maxBuffer` is hit, with a single chunk', 79 | checkBufferedData, 80 | fixtureArray, 81 | fixtureArray.slice(0, 1), 82 | ); 83 | test( 84 | 'set error.bufferedData when `maxBuffer` is hit, with multiple chunks', 85 | checkBufferedData, 86 | [...fixtureArray, ...fixtureArray], 87 | [...fixtureArray, ...fixtureArray.slice(0, 1)], 88 | ); 89 | 90 | test('getStreamAsArray() behaves like readable.toArray()', async t => { 91 | const [nativeResult, customResult] = await Promise.all([ 92 | createStream([bigArray]).toArray(), 93 | setupArray([bigArray]), 94 | ]); 95 | t.deepEqual(nativeResult, customResult); 96 | }); 97 | 98 | test('getStreamAsArray() can stream JSON', async t => { 99 | t.timeout(BIG_TEST_DURATION); 100 | const bigJson = bigArray.map(byte => ({byte})); 101 | const bigJsonString = JSON.stringify(bigJson); 102 | const result = await getStreamAsArray(compose( 103 | createStream([bigJsonString]), 104 | streamJson.parser(), 105 | streamJsonArray.streamArray(), 106 | )); 107 | t.is(result.length, bigJson.length); 108 | t.deepEqual(result.at(-1).value, bigJson.at(-1)); 109 | }); 110 | -------------------------------------------------------------------------------- /test/browser.js: -------------------------------------------------------------------------------- 1 | import {execFile} from 'node:child_process'; 2 | import path from 'node:path'; 3 | import {fileURLToPath} from 'node:url'; 4 | import {promisify} from 'node:util'; 5 | import test from 'ava'; 6 | import {fixtureString} from './fixtures/index.js'; 7 | 8 | const pExecFile = promisify(execFile); 9 | const cwd = path.dirname(fileURLToPath(import.meta.url)); 10 | const nodeStreamFixture = './fixtures/node-stream.js'; 11 | const webStreamFixture = './fixtures/web-stream.js'; 12 | const iterableFixture = './fixtures/iterable.js'; 13 | const nodeConditions = []; 14 | const browserConditions = ['--conditions=browser']; 15 | 16 | const testEntrypoint = async (t, fixture, conditions, expectedOutput = fixtureString) => { 17 | const {stdout, stderr} = await pExecFile('node', [...conditions, fixture], {cwd}); 18 | t.is(stderr, ''); 19 | t.is(stdout, expectedOutput); 20 | }; 21 | 22 | test('Node entrypoint works with Node streams', testEntrypoint, nodeStreamFixture, nodeConditions, `${fixtureString}${fixtureString}`); 23 | test('Browser entrypoint works with Node streams', testEntrypoint, nodeStreamFixture, browserConditions); 24 | test('Node entrypoint works with web streams', testEntrypoint, webStreamFixture, nodeConditions); 25 | test('Browser entrypoint works with web streams', testEntrypoint, webStreamFixture, browserConditions); 26 | test('Node entrypoint works with async iterables', testEntrypoint, iterableFixture, nodeConditions); 27 | test('Browser entrypoint works with async iterables', testEntrypoint, iterableFixture, browserConditions); 28 | -------------------------------------------------------------------------------- /test/buffer.js: -------------------------------------------------------------------------------- 1 | import {Buffer} from 'node:buffer'; 2 | import {buffer} from 'node:stream/consumers'; 3 | import test from 'ava'; 4 | import {getStreamAsBuffer, MaxBufferError} from '../source/index.js'; 5 | import {createStream} from './helpers/index.js'; 6 | import { 7 | fixtureString, 8 | fixtureLength, 9 | fixtureBuffer, 10 | fixtureTypedArray, 11 | fixtureArrayBuffer, 12 | fixtureUint16Array, 13 | fixtureDataView, 14 | fixtureMultiString, 15 | fixtureMultiBuffer, 16 | fixtureMultiTypedArray, 17 | fixtureMultiArrayBuffer, 18 | fixtureMultiUint16Array, 19 | fixtureMultiDataView, 20 | fixtureTypedArrayWithOffset, 21 | fixtureUint16ArrayWithOffset, 22 | fixtureDataViewWithOffset, 23 | longString, 24 | fixtureMultibyteString, 25 | longMultibyteString, 26 | bigArray, 27 | } from './fixtures/index.js'; 28 | 29 | const longBuffer = Buffer.from(longString); 30 | const fixtureMultibyteBuffer = Buffer.from(fixtureMultibyteString); 31 | const longMultibyteBuffer = Buffer.from(longMultibyteString); 32 | const bigBuffer = Buffer.from(bigArray); 33 | 34 | const setupBuffer = (streamDefinition, options) => getStreamAsBuffer(createStream(streamDefinition), options); 35 | 36 | const getStreamToBuffer = async (t, fixtureValue) => { 37 | const result = await setupBuffer(fixtureValue); 38 | t.true(Buffer.isBuffer(result)); 39 | t.true(result.equals(fixtureBuffer)); 40 | }; 41 | 42 | test('get stream from string to buffer, with a single chunk', getStreamToBuffer, [fixtureString]); 43 | test('get stream from buffer to buffer, with a single chunk', getStreamToBuffer, [fixtureBuffer]); 44 | test('get stream from arrayBuffer to buffer, with a single chunk', getStreamToBuffer, [fixtureArrayBuffer]); 45 | test('get stream from typedArray to buffer, with a single chunk', getStreamToBuffer, [fixtureTypedArray]); 46 | test('get stream from typedArray with offset to buffer, with a single chunk', getStreamToBuffer, [fixtureTypedArrayWithOffset]); 47 | test('get stream from uint16Array to buffer, with a single chunk', getStreamToBuffer, [fixtureUint16Array]); 48 | test('get stream from uint16Array with offset to buffer, with a single chunk', getStreamToBuffer, [fixtureUint16ArrayWithOffset]); 49 | test('get stream from dataView to buffer, with a single chunk', getStreamToBuffer, [fixtureDataView]); 50 | test('get stream from dataView with offset to buffer, with a single chunk', getStreamToBuffer, [fixtureDataViewWithOffset]); 51 | 52 | test('get stream from string to buffer, with multiple chunks', getStreamToBuffer, fixtureMultiString); 53 | test('get stream from buffer to buffer, with multiple chunks', getStreamToBuffer, fixtureMultiBuffer); 54 | test('get stream from arrayBuffer to buffer, with multiple chunks', getStreamToBuffer, fixtureMultiArrayBuffer); 55 | test('get stream from typedArray to buffer, with multiple chunks', getStreamToBuffer, fixtureMultiTypedArray); 56 | test('get stream from uint16Array to buffer, with multiple chunks', getStreamToBuffer, fixtureMultiUint16Array); 57 | test('get stream from dataView to buffer, with multiple chunks', getStreamToBuffer, fixtureMultiDataView); 58 | 59 | const throwOnInvalidChunkType = async (t, fixtureValue) => { 60 | await t.throwsAsync(setupBuffer([fixtureValue]), {message: /not supported/}); 61 | }; 62 | 63 | test('get stream from object to buffer', throwOnInvalidChunkType, {}); 64 | test('get stream from array to buffer', throwOnInvalidChunkType, []); 65 | test('get stream from boolean to buffer', throwOnInvalidChunkType, false); 66 | test('get stream from number to buffer', throwOnInvalidChunkType, 0); 67 | test('get stream from bigint to buffer', throwOnInvalidChunkType, 0n); 68 | test('get stream from undefined to buffer', throwOnInvalidChunkType, undefined); 69 | test('get stream from symbol to buffer', throwOnInvalidChunkType, Symbol('test')); 70 | 71 | const checkMaxBuffer = async (t, longValue, shortValue, maxBuffer) => { 72 | await t.throwsAsync(setupBuffer([longValue], {maxBuffer}), {instanceOf: MaxBufferError}); 73 | await t.notThrowsAsync(setupBuffer([shortValue], {maxBuffer})); 74 | }; 75 | 76 | test('maxBuffer throws when size is exceeded with a buffer', checkMaxBuffer, longBuffer, fixtureBuffer, fixtureLength); 77 | test('maxBuffer unit is bytes with getStreamAsBuffer()', checkMaxBuffer, longMultibyteBuffer, fixtureMultibyteBuffer, fixtureMultibyteBuffer.byteLength); 78 | 79 | const checkBufferedData = async (t, fixtureValue, expectedResult) => { 80 | const maxBuffer = expectedResult.length; 81 | const {bufferedData} = await t.throwsAsync(setupBuffer(fixtureValue, {maxBuffer}), {instanceOf: MaxBufferError}); 82 | t.is(bufferedData.length, maxBuffer); 83 | t.deepEqual(expectedResult, bufferedData); 84 | }; 85 | 86 | test( 87 | 'set error.bufferedData when `maxBuffer` is hit, with a single chunk', 88 | checkBufferedData, 89 | [fixtureBuffer], 90 | fixtureBuffer.slice(0, 1), 91 | ); 92 | test( 93 | 'set error.bufferedData when `maxBuffer` is hit, with multiple chunks', 94 | checkBufferedData, 95 | [fixtureBuffer, fixtureBuffer], 96 | Buffer.from([...fixtureBuffer, ...fixtureBuffer.slice(0, 1)]), 97 | ); 98 | 99 | test('getStreamAsBuffer() behaves like buffer()', async t => { 100 | const [nativeResult, customResult] = await Promise.all([ 101 | buffer(createStream([bigBuffer])), 102 | setupBuffer([bigBuffer]), 103 | ]); 104 | t.deepEqual(nativeResult, customResult); 105 | }); 106 | 107 | test('getStreamAsBuffer() only works in Node', async t => { 108 | const {Buffer} = globalThis; 109 | delete globalThis.Buffer; 110 | try { 111 | await t.throwsAsync(setupBuffer([fixtureString]), {message: /only supported in Node/}); 112 | } finally { 113 | globalThis.Buffer = Buffer; 114 | } 115 | }); 116 | -------------------------------------------------------------------------------- /test/contents.js: -------------------------------------------------------------------------------- 1 | import {setTimeout} from 'node:timers/promises'; 2 | import test from 'ava'; 3 | import getStream, {MaxBufferError} from '../source/index.js'; 4 | import {createStream} from './helpers/index.js'; 5 | import { 6 | fixtureString, 7 | fixtureBuffer, 8 | fixtureTypedArray, 9 | fixtureArrayBuffer, 10 | fixtureUint16Array, 11 | fixtureDataView, 12 | } from './fixtures/index.js'; 13 | 14 | const setupString = (streamDefinition, options) => getStream(createStream(streamDefinition), options); 15 | 16 | const generator = async function * () { 17 | yield 'a'; 18 | await setTimeout(0); 19 | yield 'b'; 20 | }; 21 | 22 | test('works with async iterable', async t => { 23 | const result = await getStream(generator()); 24 | t.is(result, 'ab'); 25 | }); 26 | 27 | const finallyGenerator = async function * (state) { 28 | try { 29 | yield {}; 30 | } catch (error) { 31 | state.error = true; 32 | throw error; 33 | } finally { 34 | state.finally = true; 35 | } 36 | }; 37 | 38 | test('async iterable .return() is called on error, but not .throw()', async t => { 39 | const state = {error: false, finally: false}; 40 | await t.throwsAsync(getStream(finallyGenerator(state))); 41 | t.false(state.error); 42 | t.true(state.finally); 43 | }); 44 | 45 | test('get stream with mixed chunk types', async t => { 46 | const fixtures = [fixtureString, fixtureBuffer, fixtureArrayBuffer, fixtureTypedArray, fixtureUint16Array, fixtureDataView]; 47 | const result = await setupString(fixtures); 48 | t.is(result, fixtureString.repeat(fixtures.length)); 49 | }); 50 | 51 | test('getStream should not affect additional listeners attached to the stream', async t => { 52 | t.plan(3); 53 | const fixture = createStream(['foo', 'bar']); 54 | fixture.on('data', chunk => t.true(typeof chunk === 'string')); 55 | t.is(await getStream(fixture), 'foobar'); 56 | }); 57 | 58 | const errorStream = async function * (error) { 59 | yield fixtureString; 60 | await setTimeout(0); 61 | throw error; 62 | }; 63 | 64 | const testErrorStream = async (t, error) => { 65 | const {bufferedData} = await t.throwsAsync(setupString(errorStream.bind(undefined, error))); 66 | t.is(bufferedData, fixtureString); 67 | }; 68 | 69 | test('set error.bufferedData when stream errors', testErrorStream, new Error('test')); 70 | test('set error.bufferedData when stream error string', testErrorStream, 'test'); 71 | test('set error.bufferedData when stream error null', testErrorStream, null); 72 | test('set error.bufferedData when stream error undefined', testErrorStream, undefined); 73 | 74 | const infiniteIteration = async function * () { 75 | while (true) { 76 | // eslint-disable-next-line no-await-in-loop 77 | await setTimeout(0); 78 | yield '.'; 79 | } 80 | }; 81 | 82 | test('handles infinite stream', async t => { 83 | await t.throwsAsync(setupString(infiniteIteration, {maxBuffer: 1}), {instanceOf: MaxBufferError}); 84 | }); 85 | 86 | const firstArgumentCheck = async (t, firstArgument) => { 87 | await t.throwsAsync(getStream(firstArgument), {message: /first argument/}); 88 | }; 89 | 90 | test('Throws if the first argument is undefined', firstArgumentCheck, undefined); 91 | test('Throws if the first argument is null', firstArgumentCheck, null); 92 | test('Throws if the first argument is a string', firstArgumentCheck, ''); 93 | test('Throws if the first argument is an array', firstArgumentCheck, []); 94 | -------------------------------------------------------------------------------- /test/fixtures/index.js: -------------------------------------------------------------------------------- 1 | import {Buffer} from 'node:buffer'; 2 | 3 | export const fixtureString = 'unicorn\n'; 4 | export const fixtureLength = fixtureString.length; 5 | export const fixtureBuffer = Buffer.from(fixtureString); 6 | export const fixtureTypedArray = new TextEncoder().encode(fixtureString); 7 | export const fixtureArrayBuffer = fixtureTypedArray.buffer; 8 | export const fixtureUint16Array = new Uint16Array(fixtureArrayBuffer); 9 | export const fixtureDataView = new DataView(fixtureArrayBuffer); 10 | export const fixtureUtf16 = Buffer.from(fixtureString, 'utf-16le'); 11 | 12 | export const fixtureMultiString = [...fixtureString]; 13 | const fixtureMultiBytes = [...fixtureBuffer]; 14 | export const fixtureMultiBuffer = fixtureMultiBytes.map(byte => Buffer.from([byte])); 15 | export const fixtureMultiTypedArray = fixtureMultiBytes.map(byte => new Uint8Array([byte])); 16 | export const fixtureMultiArrayBuffer = fixtureMultiTypedArray.map(({buffer}) => buffer); 17 | export const fixtureMultiUint16Array = Array.from({length: fixtureMultiBytes.length / 2}, (_, index) => 18 | new Uint16Array([((2 ** 8) * fixtureMultiBytes[(index * 2) + 1]) + fixtureMultiBytes[index * 2]]), 19 | ); 20 | export const fixtureMultiDataView = fixtureMultiArrayBuffer.map(arrayBuffer => new DataView(arrayBuffer)); 21 | 22 | const fixtureStringWide = ` ${fixtureString} `; 23 | const fixtureTypedArrayWide = new TextEncoder().encode(fixtureStringWide); 24 | const fixtureArrayBufferWide = fixtureTypedArrayWide.buffer; 25 | export const fixtureTypedArrayWithOffset = new Uint8Array(fixtureArrayBufferWide, 2, fixtureString.length); 26 | export const fixtureUint16ArrayWithOffset = new Uint16Array(fixtureArrayBufferWide, 2, fixtureString.length / 2); 27 | export const fixtureDataViewWithOffset = new DataView(fixtureArrayBufferWide, 2, fixtureString.length); 28 | 29 | export const longString = `${fixtureString}..`; 30 | export const fixtureMultibyteString = '\u1000'; 31 | export const longMultibyteString = `${fixtureMultibyteString}\u1000`; 32 | 33 | export const bigArray = Array.from({length: 1e5}, () => Math.floor(Math.random() * (2 ** 8))); 34 | 35 | export const prematureClose = {code: 'ERR_STREAM_PREMATURE_CLOSE'}; 36 | -------------------------------------------------------------------------------- /test/fixtures/iterable.js: -------------------------------------------------------------------------------- 1 | import process from 'node:process'; 2 | import getStream from 'get-stream'; 3 | import {createStream} from '../helpers/index.js'; 4 | import {fixtureString} from './index.js'; 5 | 6 | const generator = async function * () { 7 | yield fixtureString; 8 | }; 9 | 10 | const stream = createStream(generator); 11 | process.stdout.write(await getStream(stream)); 12 | -------------------------------------------------------------------------------- /test/fixtures/node-stream.js: -------------------------------------------------------------------------------- 1 | import process from 'node:process'; 2 | import getStream from 'get-stream'; 3 | import {createStream} from '../helpers/index.js'; 4 | import {fixtureString} from './index.js'; 5 | 6 | const stream = createStream([fixtureString]); 7 | const [output, secondOutput] = await Promise.all([getStream(stream), getStream(stream)]); 8 | process.stdout.write(`${output}${secondOutput}`); 9 | -------------------------------------------------------------------------------- /test/fixtures/web-stream.js: -------------------------------------------------------------------------------- 1 | import process from 'node:process'; 2 | import getStream from 'get-stream'; 3 | import {readableStreamFrom} from '../helpers/index.js'; 4 | import {fixtureString} from './index.js'; 5 | 6 | const stream = readableStreamFrom([fixtureString]); 7 | process.stdout.write(await getStream(stream)); 8 | -------------------------------------------------------------------------------- /test/helpers/index.js: -------------------------------------------------------------------------------- 1 | import {Duplex, Readable} from 'node:stream'; 2 | import {finished} from 'node:stream/promises'; 3 | 4 | // @todo Use ReadableStream.from() after dropping support for Node 18 5 | export {fromAnyIterable as readableStreamFrom} from '@sec-ant/readable-stream/ponyfill'; 6 | 7 | export const createStream = streamDefinition => typeof streamDefinition === 'function' 8 | ? Duplex.from(streamDefinition) 9 | : Readable.from(streamDefinition); 10 | 11 | // Tests related to big buffers/strings can be slow. We run them serially and 12 | // with a higher timeout to ensure they do not randomly fail. 13 | export const BIG_TEST_DURATION = '2m'; 14 | 15 | export const onFinishedStream = stream => finished(stream, {cleanup: true}); 16 | -------------------------------------------------------------------------------- /test/integration.js: -------------------------------------------------------------------------------- 1 | import {spawn} from 'node:child_process'; 2 | import {createReadStream} from 'node:fs'; 3 | import {open, opendir} from 'node:fs/promises'; 4 | import {version as nodeVersion} from 'node:process'; 5 | import {Duplex} from 'node:stream'; 6 | import test from 'ava'; 7 | import getStream, {getStreamAsBuffer, getStreamAsArray} from '../source/index.js'; 8 | import {fixtureString, fixtureBuffer, fixtureUtf16} from './fixtures/index.js'; 9 | 10 | const TEST_URL = 'https://nodejs.org/dist/index.json'; 11 | 12 | const createReadableStream = streamDefinition => Duplex.toWeb(Duplex.from(streamDefinition)).readable; 13 | 14 | test('works with opendir()', async t => { 15 | const directoryFiles = await opendir('.'); 16 | const entries = await getStreamAsArray(directoryFiles); 17 | t.true(entries.some(({name}) => name === 'package.json')); 18 | }); 19 | 20 | test('works with createReadStream() and buffers', async t => { 21 | const result = await getStreamAsBuffer(createReadStream('fixture')); 22 | t.true(result.equals(fixtureBuffer)); 23 | }); 24 | 25 | test('works with createReadStream() and utf8', async t => { 26 | const result = await getStream(createReadStream('fixture', 'utf8')); 27 | t.is(result, fixtureString); 28 | }); 29 | 30 | test('works with child_process.spawn()', async t => { 31 | const {stdout} = spawn('node', ['--version'], {stdio: ['ignore', 'pipe', 'ignore']}); 32 | const result = await getStream(stdout); 33 | t.is(result.trim(), nodeVersion); 34 | }); 35 | 36 | // @todo: remove this condition after dropping support for Node 16. 37 | // `ReadableStream` was added in Node 16.5.0. 38 | // `Duplex.toWeb()` and `fileHandle.readableWebStream` were added in Node 17.0.0. 39 | // `fetch()` without an experimental flag was added in Node 18.0.0. 40 | // However, `get-stream`'s implementation does not refer to any of those 41 | // variables and functions. Instead, it only supports specific chunk types 42 | // (`TypedArray`, `DataView`, `ArrayBuffer`) for any async iterable. 43 | // Doing so automatically works with `ReadableStream`s, regardless of whether 44 | // the environment supports them. 45 | if (!nodeVersion.startsWith('v16.')) { 46 | test('works with ReadableStream', async t => { 47 | const result = await getStream(createReadableStream(fixtureString)); 48 | t.is(result, fixtureString); 49 | }); 50 | 51 | const readableWebStream = async (t, type) => { 52 | const fileHandle = await open('fixture'); 53 | 54 | try { 55 | const result = await getStream(fileHandle.readableWebStream({type})); 56 | t.is(result, fixtureString); 57 | } finally { 58 | await fileHandle.close(); 59 | } 60 | }; 61 | 62 | test('works with readableWebStream({ type: undefined })', readableWebStream, undefined); 63 | test('works with readableWebStream({ type: "bytes" })', readableWebStream, 'bytes'); 64 | 65 | test('works with fetch()', async t => { 66 | const {body} = await fetch(TEST_URL); 67 | const result = await getStream(body); 68 | const parsedResult = JSON.parse(result); 69 | t.true(Array.isArray(parsedResult)); 70 | }); 71 | 72 | test('can use TextDecoderStream', async t => { 73 | const textDecoderStream = new TextDecoderStream('utf-16le'); 74 | const result = await getStream( 75 | createReadableStream(fixtureUtf16).pipeThrough(textDecoderStream), 76 | ); 77 | t.is(result, fixtureString); 78 | }); 79 | } 80 | -------------------------------------------------------------------------------- /test/stream.js: -------------------------------------------------------------------------------- 1 | import {once} from 'node:events'; 2 | import {Readable, Duplex} from 'node:stream'; 3 | import {scheduler, setTimeout as pSetTimeout} from 'node:timers/promises'; 4 | import test from 'ava'; 5 | import onetime from 'onetime'; 6 | import getStream, {getStreamAsArray, MaxBufferError} from '../source/index.js'; 7 | import {fixtureString, fixtureMultiString, prematureClose} from './fixtures/index.js'; 8 | import {onFinishedStream} from './helpers/index.js'; 9 | 10 | const noopMethods = {read() {}, write() {}}; 11 | 12 | // eslint-disable-next-line max-params 13 | const assertStream = ({readableEnded = false, writableEnded = false}, t, stream, StreamClass, error = null) => { 14 | t.is(stream.errored, error); 15 | t.true(stream.destroyed); 16 | t.false(stream.readable); 17 | t.is(stream.readableEnded, readableEnded); 18 | 19 | if (StreamClass === Duplex) { 20 | t.false(stream.writable); 21 | t.is(stream.writableEnded, writableEnded); 22 | } 23 | }; 24 | 25 | const assertSuccess = assertStream.bind(undefined, {readableEnded: true, writableEnded: true}); 26 | const assertReadFail = assertStream.bind(undefined, {writableEnded: true}); 27 | const assertWriteFail = assertStream.bind(undefined, {readableEnded: true}); 28 | const assertBothFail = assertStream.bind(undefined, {}); 29 | 30 | test('Can emit "error" event right after getStream()', async t => { 31 | const stream = Readable.from([fixtureString]); 32 | t.is(stream.listenerCount('error'), 0); 33 | const promise = getStream(stream); 34 | t.is(stream.listenerCount('error'), 1); 35 | 36 | const error = new Error('test'); 37 | stream.emit('error', error); 38 | t.is(await t.throwsAsync(promise), error); 39 | }); 40 | 41 | const testSuccess = async (t, StreamClass) => { 42 | const stream = StreamClass.from(fixtureMultiString); 43 | t.true(stream instanceof StreamClass); 44 | 45 | t.deepEqual(await getStreamAsArray(stream), fixtureMultiString); 46 | assertSuccess(t, stream, StreamClass); 47 | }; 48 | 49 | test('Can use Readable stream', testSuccess, Readable); 50 | test('Can use Duplex stream', testSuccess, Duplex); 51 | 52 | const testAlreadyEnded = async (t, StreamClass) => { 53 | const stream = StreamClass.from(fixtureMultiString); 54 | await stream.toArray(); 55 | assertSuccess(t, stream, StreamClass); 56 | 57 | t.deepEqual(await getStreamAsArray(stream), []); 58 | }; 59 | 60 | test('Can use already ended Readable', testAlreadyEnded, Readable); 61 | test('Can use already ended Duplex', testAlreadyEnded, Duplex); 62 | 63 | const testAlreadyAborted = async (t, StreamClass) => { 64 | const stream = StreamClass.from(fixtureMultiString); 65 | stream.destroy(); 66 | await t.throwsAsync(onFinishedStream(stream), prematureClose); 67 | assertReadFail(t, stream, StreamClass); 68 | 69 | const error = await t.throwsAsync(getStreamAsArray(stream), prematureClose); 70 | t.deepEqual(error.bufferedData, []); 71 | }; 72 | 73 | test('Throw if already aborted Readable', testAlreadyAborted, Readable); 74 | test('Throw if already aborted Duplex', testAlreadyAborted, Duplex); 75 | 76 | const testAlreadyErrored = async (t, StreamClass) => { 77 | const stream = StreamClass.from(fixtureMultiString); 78 | const error = new Error('test'); 79 | stream.destroy(error); 80 | t.is(await t.throwsAsync(onFinishedStream(stream)), error); 81 | assertReadFail(t, stream, StreamClass, error); 82 | 83 | t.is(await t.throwsAsync(getStreamAsArray(stream)), error); 84 | t.deepEqual(error.bufferedData, []); 85 | }; 86 | 87 | test('Throw if already errored Readable', testAlreadyErrored, Readable); 88 | test('Throw if already errored Duplex', testAlreadyErrored, Duplex); 89 | 90 | const testAbort = async (t, StreamClass) => { 91 | const stream = new StreamClass(noopMethods); 92 | setTimeout(() => { 93 | stream.destroy(); 94 | }, 0); 95 | const error = await t.throwsAsync(getStreamAsArray(stream), prematureClose); 96 | t.deepEqual(error.bufferedData, []); 97 | assertBothFail(t, stream, StreamClass); 98 | }; 99 | 100 | test('Throw when aborting Readable', testAbort, Readable); 101 | test('Throw when aborting Duplex', testAbort, Duplex); 102 | 103 | const testError = async (t, StreamClass) => { 104 | const stream = new StreamClass(noopMethods); 105 | const error = new Error('test'); 106 | setTimeout(() => { 107 | stream.destroy(error); 108 | }, 0); 109 | t.is(await t.throwsAsync(getStreamAsArray(stream)), error); 110 | t.deepEqual(error.bufferedData, []); 111 | assertBothFail(t, stream, StreamClass, error); 112 | }; 113 | 114 | test('Throw when erroring Readable', testError, Readable); 115 | test('Throw when erroring Duplex', testError, Duplex); 116 | 117 | const testErrorEvent = async (t, StreamClass, hasCause) => { 118 | const stream = new StreamClass(noopMethods); 119 | const error = new Error('test', hasCause ? {cause: new Error('inner')} : {}); 120 | setTimeout(() => { 121 | stream.emit('error', error); 122 | }, 0); 123 | t.is(await t.throwsAsync(getStreamAsArray(stream)), error); 124 | t.deepEqual(error.bufferedData, []); 125 | assertBothFail(t, stream, StreamClass); 126 | }; 127 | 128 | test('Throw when emitting "error" event with Readable', testErrorEvent, Readable, false); 129 | test('Throw when emitting "error" event with Duplex', testErrorEvent, Duplex, false); 130 | test('Throw when emitting "error" event with Readable and error.cause', testErrorEvent, Readable, true); 131 | test('Throw when emitting "error" event with Duplex and error.cause', testErrorEvent, Duplex, true); 132 | 133 | const testThrowRead = async (t, StreamClass) => { 134 | const error = new Error('test'); 135 | const stream = new StreamClass({ 136 | read() { 137 | throw error; 138 | }, 139 | }); 140 | t.is(await t.throwsAsync(getStreamAsArray(stream)), error); 141 | t.deepEqual(error.bufferedData, []); 142 | assertBothFail(t, stream, StreamClass, error); 143 | }; 144 | 145 | test('Throw when throwing error in Readable read()', testThrowRead, Readable); 146 | test('Throw when throwing error in Duplex read()', testThrowRead, Duplex); 147 | 148 | test('Throw when throwing error in Readable destroy()', async t => { 149 | const error = new Error('test'); 150 | const stream = new Readable({ 151 | read: onetime(function () { 152 | this.push(fixtureString); 153 | this.push(null); 154 | }), 155 | destroy(_, done) { 156 | done(error); 157 | }, 158 | }); 159 | 160 | t.is(await t.throwsAsync(getStream(stream)), error); 161 | t.deepEqual(error.bufferedData, fixtureString); 162 | assertSuccess(t, stream, Readable, error); 163 | }); 164 | 165 | test('Throw when throwing error in Duplex final()', async t => { 166 | const error = new Error('test'); 167 | const stream = new Duplex({ 168 | read: onetime(function () { 169 | this.push(null); 170 | }), 171 | final(done) { 172 | done(error); 173 | }, 174 | }); 175 | stream.end(); 176 | 177 | t.is(await t.throwsAsync(getStream(stream)), error); 178 | t.is(await t.throwsAsync(onFinishedStream(stream)), error); 179 | assertReadFail(t, stream, Duplex, error); 180 | }); 181 | 182 | test('Does not wait for Duplex writable side', async t => { 183 | const error = new Error('test'); 184 | const stream = new Duplex({ 185 | read: onetime(function () { 186 | this.push(null); 187 | }), 188 | destroy(_, done) { 189 | done(error); 190 | }, 191 | }); 192 | 193 | t.is(await getStream(stream), ''); 194 | t.is(await t.throwsAsync(onFinishedStream(stream)), error); 195 | assertWriteFail(t, stream, Duplex, error); 196 | }); 197 | 198 | test('Handle non-error instances', async t => { 199 | const stream = Readable.from(fixtureMultiString); 200 | const errorMessage = `< ${fixtureString} >`; 201 | stream.destroy(errorMessage); 202 | const [{reason}] = await Promise.allSettled([onFinishedStream(stream)]); 203 | t.is(reason, errorMessage); 204 | assertReadFail(t, stream, Readable, errorMessage); 205 | 206 | await t.throwsAsync(getStreamAsArray(stream), {message: errorMessage}); 207 | }); 208 | 209 | test('Handles objectMode errors', async t => { 210 | const stream = new Readable({ 211 | read: onetime(function () { 212 | this.push(fixtureString); 213 | this.push({}); 214 | }), 215 | objectMode: true, 216 | }); 217 | 218 | const error = await t.throwsAsync(getStream(stream), {message: /in object mode/}); 219 | t.is(error.bufferedData, fixtureString); 220 | assertReadFail(t, stream, Readable); 221 | }); 222 | 223 | test('Handles maxBuffer errors', async t => { 224 | const stream = new Readable({ 225 | read: onetime(function () { 226 | this.push(fixtureString); 227 | this.push(fixtureString); 228 | }), 229 | }); 230 | 231 | const error = await t.throwsAsync( 232 | getStream(stream, {maxBuffer: fixtureString.length}), 233 | {instanceOf: MaxBufferError}, 234 | ); 235 | t.is(error.bufferedData, fixtureString); 236 | assertReadFail(t, stream, Readable); 237 | }); 238 | 239 | test('Works if Duplex readable side ends before its writable side', async t => { 240 | const stream = new Duplex(noopMethods); 241 | stream.push(null); 242 | 243 | t.deepEqual(await getStreamAsArray(stream), []); 244 | assertWriteFail(t, stream, Duplex); 245 | }); 246 | 247 | test('Cleans up event listeners', async t => { 248 | const stream = Readable.from([]); 249 | t.is(stream.listenerCount('error'), 0); 250 | 251 | t.deepEqual(await getStreamAsArray(stream), []); 252 | 253 | t.is(stream.listenerCount('error'), 0); 254 | }); 255 | 256 | const testMultipleReads = async (t, wait) => { 257 | const size = 10; 258 | const stream = new Readable({ 259 | read: onetime(async function () { 260 | for (let index = 0; index < size; index += 1) { 261 | for (let index = 0; index < size; index += 1) { 262 | this.push(fixtureString); 263 | } 264 | 265 | // eslint-disable-next-line no-await-in-loop 266 | await wait(); 267 | } 268 | 269 | this.push(null); 270 | }), 271 | }); 272 | 273 | t.is(await getStream(stream), fixtureString.repeat(size * size)); 274 | assertSuccess(t, stream, Readable); 275 | }; 276 | 277 | test('Handles multiple successive fast reads', testMultipleReads, () => scheduler.yield()); 278 | test('Handles multiple successive slow reads', testMultipleReads, () => pSetTimeout(100)); 279 | 280 | test('Can call twice at the same time', async t => { 281 | const stream = Readable.from(fixtureMultiString); 282 | const [result, secondResult] = await Promise.all([ 283 | getStream(stream), 284 | getStream(stream), 285 | ]); 286 | t.deepEqual(result, fixtureString); 287 | t.deepEqual(secondResult, fixtureString); 288 | assertSuccess(t, stream, Readable); 289 | }); 290 | 291 | test('Can call and listen to "data" event at the same time', async t => { 292 | const stream = Readable.from([fixtureString]); 293 | const [result, secondResult] = await Promise.all([ 294 | getStream(stream), 295 | once(stream, 'data'), 296 | ]); 297 | t.deepEqual(result, fixtureString); 298 | t.deepEqual(secondResult.toString(), fixtureString); 299 | assertSuccess(t, stream, Readable); 300 | }); 301 | -------------------------------------------------------------------------------- /test/string.js: -------------------------------------------------------------------------------- 1 | import {Buffer, constants as BufferConstants} from 'node:buffer'; 2 | import {text} from 'node:stream/consumers'; 3 | import test from 'ava'; 4 | import getStream, {MaxBufferError} from '../source/index.js'; 5 | import {createStream, BIG_TEST_DURATION} from './helpers/index.js'; 6 | import { 7 | fixtureString, 8 | fixtureLength, 9 | fixtureBuffer, 10 | fixtureTypedArray, 11 | fixtureArrayBuffer, 12 | fixtureUint16Array, 13 | fixtureDataView, 14 | fixtureMultiString, 15 | fixtureMultiBuffer, 16 | fixtureMultiTypedArray, 17 | fixtureMultiArrayBuffer, 18 | fixtureMultiUint16Array, 19 | fixtureMultiDataView, 20 | fixtureTypedArrayWithOffset, 21 | fixtureUint16ArrayWithOffset, 22 | fixtureDataViewWithOffset, 23 | longString, 24 | fixtureMultibyteString, 25 | longMultibyteString, 26 | bigArray, 27 | } from './fixtures/index.js'; 28 | 29 | const bigString = Buffer.from(bigArray).toString(); 30 | const multiByteString = 'a\u1000'; 31 | const multiByteUint8Array = new TextEncoder().encode(multiByteString); 32 | const multiByteBuffer = [...multiByteUint8Array].map(byte => Buffer.from([byte])); 33 | const INVALID_UTF8_MARKER = '\uFFFD'; 34 | 35 | const setupString = (streamDefinition, options) => getStream(createStream(streamDefinition), options); 36 | 37 | const getStreamToString = async (t, fixtureValue) => { 38 | const result = await setupString(fixtureValue); 39 | t.is(typeof result, 'string'); 40 | t.is(result, fixtureString); 41 | }; 42 | 43 | test('get stream from string to string, with a single chunk', getStreamToString, [fixtureString]); 44 | test('get stream from buffer to string, with a single chunk', getStreamToString, [fixtureBuffer]); 45 | test('get stream from arrayBuffer to string, with a single chunk', getStreamToString, [fixtureArrayBuffer]); 46 | test('get stream from typedArray to string, with a single chunk', getStreamToString, [fixtureTypedArray]); 47 | test('get stream from typedArray with offset to string, with a single chunk', getStreamToString, [fixtureTypedArrayWithOffset]); 48 | test('get stream from uint16Array to string, with a single chunk', getStreamToString, [fixtureUint16Array]); 49 | test('get stream from uint16Array with offset to string, with a single chunk', getStreamToString, [fixtureUint16ArrayWithOffset]); 50 | test('get stream from dataView to string, with a single chunk', getStreamToString, [fixtureDataView]); 51 | test('get stream from dataView with offset to string, with a single chunk', getStreamToString, [fixtureDataViewWithOffset]); 52 | 53 | test('get stream from string to string, with multiple chunks', getStreamToString, fixtureMultiString); 54 | test('get stream from buffer to string, with multiple chunks', getStreamToString, fixtureMultiBuffer); 55 | test('get stream from arrayBuffer to string, with multiple chunks', getStreamToString, fixtureMultiArrayBuffer); 56 | test('get stream from typedArray to string, with multiple chunks', getStreamToString, fixtureMultiTypedArray); 57 | test('get stream from uint16Array to string, with multiple chunks', getStreamToString, fixtureMultiUint16Array); 58 | test('get stream from dataView to string, with multiple chunks', getStreamToString, fixtureMultiDataView); 59 | 60 | const throwOnInvalidChunkType = async (t, setupFunction, fixtureValue) => { 61 | await t.throwsAsync(setupFunction([fixtureValue]), {message: /not supported/}); 62 | }; 63 | 64 | test('get stream from object to string', throwOnInvalidChunkType, setupString, {}); 65 | test('get stream from array to string', throwOnInvalidChunkType, setupString, []); 66 | test('get stream from boolean to string', throwOnInvalidChunkType, setupString, false); 67 | test('get stream from number to string', throwOnInvalidChunkType, setupString, 0); 68 | test('get stream from bigint to string', throwOnInvalidChunkType, setupString, 0n); 69 | test('get stream from undefined to string', throwOnInvalidChunkType, setupString, undefined); 70 | test('get stream from symbol to string', throwOnInvalidChunkType, setupString, Symbol('test')); 71 | 72 | const checkMaxBuffer = async (t, longValue, shortValue, maxBuffer) => { 73 | await t.throwsAsync(setupString([longValue], {maxBuffer}), {instanceOf: MaxBufferError}); 74 | await t.notThrowsAsync(setupString([shortValue], {maxBuffer})); 75 | }; 76 | 77 | test('maxBuffer throws when size is exceeded with a string', checkMaxBuffer, longString, fixtureString, fixtureLength); 78 | test('maxBuffer unit is characters with getStream()', checkMaxBuffer, longMultibyteString, fixtureMultibyteString, fixtureMultibyteString.length); 79 | 80 | const checkBufferedData = async (t, fixtureValue, expectedResult) => { 81 | const maxBuffer = expectedResult.length; 82 | const {bufferedData} = await t.throwsAsync(setupString(fixtureValue, {maxBuffer}), {instanceOf: MaxBufferError}); 83 | t.is(bufferedData.length, maxBuffer); 84 | t.is(expectedResult, bufferedData); 85 | }; 86 | 87 | test( 88 | 'set error.bufferedData when `maxBuffer` is hit, with a single chunk', 89 | checkBufferedData, 90 | [fixtureString], 91 | fixtureString[0], 92 | ); 93 | test( 94 | 'set error.bufferedData when `maxBuffer` is hit, with multiple chunks', 95 | checkBufferedData, 96 | [fixtureString, fixtureString], 97 | `${fixtureString}${fixtureString[0]}`, 98 | ); 99 | 100 | test('handles streams larger than string max length', async t => { 101 | t.timeout(BIG_TEST_DURATION); 102 | const chunkCount = Math.floor(BufferConstants.MAX_STRING_LENGTH / CHUNK_SIZE * 2); 103 | const chunk = '.'.repeat(CHUNK_SIZE); 104 | const maxStringChunks = Array.from({length: chunkCount}, () => chunk); 105 | const {bufferedData} = await t.throwsAsync(setupString(maxStringChunks)); 106 | t.is(bufferedData[0], '.'); 107 | }); 108 | 109 | const CHUNK_SIZE = 2 ** 16; 110 | 111 | test('handles streams with a single chunk larger than string max length', async t => { 112 | const chunks = [Buffer.alloc(BufferConstants.MAX_STRING_LENGTH + 1)]; 113 | const {bufferedData} = await t.throwsAsync(setupString(chunks)); 114 | t.is(bufferedData, ''); 115 | }); 116 | 117 | test('getStream() behaves like text()', async t => { 118 | const [nativeResult, customResult] = await Promise.all([ 119 | text(createStream([bigString])), 120 | setupString([bigString]), 121 | ]); 122 | t.is(nativeResult, customResult); 123 | }); 124 | 125 | test('get stream with partial UTF-8 sequences', async t => { 126 | const result = await setupString(multiByteBuffer); 127 | t.is(result, multiByteString); 128 | }); 129 | 130 | test('get stream with truncated UTF-8 sequences', async t => { 131 | const result = await setupString(multiByteBuffer.slice(0, -1)); 132 | t.is(result, `${multiByteString.slice(0, -1)}${INVALID_UTF8_MARKER}`); 133 | }); 134 | 135 | test('handles truncated UTF-8 sequences over maxBuffer', async t => { 136 | const maxBuffer = multiByteString.length - 1; 137 | await t.throwsAsync(setupString(multiByteBuffer.slice(0, -1), {maxBuffer}), {instanceOf: MaxBufferError}); 138 | }); 139 | 140 | test('get stream with invalid UTF-8 sequences', async t => { 141 | const result = await setupString(multiByteBuffer.slice(1, 2)); 142 | t.is(result, INVALID_UTF8_MARKER); 143 | }); 144 | -------------------------------------------------------------------------------- /test/web-stream-ponyfill.js: -------------------------------------------------------------------------------- 1 | import test from 'ava'; 2 | 3 | // Emulate browsers that do not support those methods 4 | delete ReadableStream.prototype.values; 5 | delete ReadableStream.prototype[Symbol.asyncIterator]; 6 | 7 | // Run those tests, but emulating browsers 8 | await import('./web-stream.js'); 9 | 10 | test('Should not polyfill ReadableStream', t => { 11 | t.is(ReadableStream.prototype.values, undefined); 12 | t.is(ReadableStream.prototype[Symbol.asyncIterator], undefined); 13 | }); 14 | -------------------------------------------------------------------------------- /test/web-stream.js: -------------------------------------------------------------------------------- 1 | import test from 'ava'; 2 | import getStream from '../source/index.js'; 3 | import {fixtureString, fixtureMultiString} from './fixtures/index.js'; 4 | import {readableStreamFrom, onFinishedStream} from './helpers/index.js'; 5 | 6 | test('Can use ReadableStream', async t => { 7 | const stream = readableStreamFrom(fixtureMultiString); 8 | t.is(await getStream(stream), fixtureString); 9 | await onFinishedStream(stream); 10 | }); 11 | 12 | test('Can use already ended ReadableStream', async t => { 13 | const stream = readableStreamFrom(fixtureMultiString); 14 | t.is(await getStream(stream), fixtureString); 15 | t.is(await getStream(stream), ''); 16 | await onFinishedStream(stream); 17 | }); 18 | 19 | test('Can use already canceled ReadableStream', async t => { 20 | let canceledValue; 21 | const stream = new ReadableStream({ 22 | cancel(canceledError) { 23 | canceledValue = canceledError; 24 | }, 25 | }); 26 | const error = new Error('test'); 27 | await stream.cancel(error); 28 | t.is(canceledValue, error); 29 | t.is(await getStream(stream), ''); 30 | await onFinishedStream(stream); 31 | }); 32 | 33 | test('Can use already errored ReadableStream', async t => { 34 | const error = new Error('test'); 35 | const stream = new ReadableStream({ 36 | start(controller) { 37 | controller.error(error); 38 | }, 39 | }); 40 | t.is(await t.throwsAsync(getStream(stream)), error); 41 | t.is(await t.throwsAsync(onFinishedStream(stream)), error); 42 | }); 43 | 44 | test('Cancel ReadableStream when maxBuffer is hit', async t => { 45 | let canceled = false; 46 | const stream = new ReadableStream({ 47 | start(controller) { 48 | controller.enqueue(fixtureString); 49 | controller.enqueue(fixtureString); 50 | controller.close(); 51 | }, 52 | cancel() { 53 | canceled = true; 54 | }, 55 | }); 56 | const error = await t.throwsAsync( 57 | getStream(stream, {maxBuffer: 1}), 58 | {message: /maxBuffer exceeded/}, 59 | ); 60 | t.deepEqual(error.bufferedData, fixtureString[0]); 61 | await onFinishedStream(stream); 62 | t.true(canceled); 63 | }); 64 | --------------------------------------------------------------------------------