├── .gitignore ├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ └── tests.yml ├── src ├── utils │ ├── reduce.d.ts │ ├── reduce.js │ ├── skip.js │ ├── take.js │ ├── takeWithSkip.js │ ├── scan.js │ ├── skip.d.ts │ ├── lines.d.ts │ ├── fixUtf8Stream.d.ts │ ├── skipWhile.d.ts │ ├── take.d.ts │ ├── batch.js │ ├── skipWhile.js │ ├── lines.js │ ├── fold.js │ ├── scan.d.ts │ ├── batch.d.ts │ ├── takeWhile.js │ ├── fold.d.ts │ ├── takeWhile.d.ts │ ├── takeWithSkip.d.ts │ ├── fixUtf8Stream.js │ ├── readableFrom.d.ts │ ├── reduceStream.js │ ├── reduceStream.d.ts │ └── readableFrom.js ├── typed-streams.js ├── jsonl │ ├── parserStream.js │ ├── parser.js │ ├── parserStream.d.ts │ ├── parser.d.ts │ ├── stringerStream.js │ └── stringerStream.d.ts ├── asStream.d.ts ├── fun.d.ts ├── typed-streams.d.ts ├── gen.d.ts ├── gen.js ├── defs.js ├── fun.js ├── asStream.js ├── index.d.ts ├── index.js └── defs.d.ts ├── .gitmodules ├── tests ├── data │ └── sample.jsonl.gz ├── test-errors.mjs ├── test-batch.mjs ├── manual │ ├── asStreamTest.js │ └── streamEventsTest.js ├── test-web-stream.mjs ├── test-skip.mjs ├── helpers.mjs ├── test-take.mjs ├── test-readableFrom.mjs ├── test-dataSource.mjs ├── test-readWrite.mjs ├── test-demo.mjs ├── test-asStream.mjs ├── test-jsonl-parser.mjs ├── test-simple.mjs ├── test-fold.mjs ├── test-jsonl-parserStream.mjs ├── test-defs.mjs ├── test-transducers.mjs ├── test-jsonl-stringerStream.mjs ├── test-fun.mjs └── test-gen.mjs ├── .vscode ├── settings.json └── launch.json ├── .prettierrc ├── .editorconfig ├── ts-check ├── asStream.ts ├── json.ts ├── utils.ts ├── fun.ts ├── gen.ts ├── demo.ts └── defs.ts ├── bench ├── gen-fun.mjs └── gen-opt.mjs ├── LICENSE ├── package.json ├── ts-test ├── demo.mts └── defs.mts ├── tsconfig.json └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | .AppleDouble 3 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: uhop 2 | buy_me_a_coffee: uhop 3 | -------------------------------------------------------------------------------- /src/utils/reduce.d.ts: -------------------------------------------------------------------------------- 1 | import fold from './fold'; 2 | 3 | export = fold; 4 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "wiki"] 2 | path = wiki 3 | url = git@github.com:uhop/stream-chain.wiki 4 | -------------------------------------------------------------------------------- /src/utils/reduce.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./reduce.d.ts" 2 | 3 | module.exports = require('./fold'); 4 | -------------------------------------------------------------------------------- /tests/data/sample.jsonl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uhop/stream-chain/HEAD/tests/data/sample.jsonl.gz -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "cSpell.words": [ 3 | "backpressure", 4 | "streamable", 5 | "unbundling" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 100, 3 | "singleQuote": true, 4 | "bracketSpacing": false, 5 | "arrowParens": "avoid", 6 | "trailingComma": "none" 7 | } 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | indent_style = space 9 | indent_size = 2 10 | -------------------------------------------------------------------------------- /ts-check/asStream.ts: -------------------------------------------------------------------------------- 1 | import asStream from 'stream-chain/asStream.js'; 2 | 3 | const a0 = asStream((x: number) => x * x); 4 | const a1 = asStream((x: boolean) => Promise.resolve(String(x))); 5 | 6 | void a0; 7 | void a1; 8 | -------------------------------------------------------------------------------- /src/utils/skip.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./skip.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {none} = require('../defs'); 6 | 7 | const skip = n => value => (n > 0 ? (--n, none) : value); 8 | 9 | module.exports = skip; 10 | -------------------------------------------------------------------------------- /src/utils/take.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./take.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {none} = require('../defs'); 6 | 7 | const take = (n, finalValue = none) => value => (n > 0 ? (--n, value) : finalValue); 8 | 9 | module.exports = take; 10 | -------------------------------------------------------------------------------- /src/utils/takeWithSkip.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./takeWithSkip.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {none} = require('../defs'); 6 | 7 | const takeWithSkip = (n, skip = 0, finalValue = none) => value => 8 | skip > 0 ? (--skip, none) : n > 0 ? (--n, value) : finalValue; 9 | 10 | module.exports = takeWithSkip; 11 | -------------------------------------------------------------------------------- /src/utils/scan.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./scan.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const scan = (fn, acc) => value => { 6 | const result = fn(acc, value); 7 | if (result && typeof result.then == 'function') { 8 | return result.then(result => (acc = result)); 9 | } 10 | return (acc = result); 11 | }; 12 | 13 | module.exports = scan; 14 | -------------------------------------------------------------------------------- /src/utils/skip.d.ts: -------------------------------------------------------------------------------- 1 | import {none} from '../defs'; 2 | 3 | export = skip; 4 | 5 | /** 6 | * Creates a function that skips `n` elements. 7 | * @param n number of elements to skip 8 | * @returns a function that takes a value and returns a value or {@link none} when skipping 9 | */ 10 | declare function skip(n: number): (value: T) => T | typeof none; 11 | -------------------------------------------------------------------------------- /src/typed-streams.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./typed-streams.d.ts" 2 | 3 | const {Duplex, Readable, Transform, Writable} = require('stream'); 4 | 5 | class TypedDuplex extends Duplex {}; 6 | class TypedReadable extends Readable {}; 7 | class TypedTransform extends Transform {}; 8 | class TypedWritable extends Writable {}; 9 | 10 | module.exports = {TypedDuplex, TypedReadable, TypedTransform, TypedWritable}; 11 | -------------------------------------------------------------------------------- /src/utils/lines.d.ts: -------------------------------------------------------------------------------- 1 | import {none} from '../defs'; 2 | 3 | export = lines; 4 | 5 | /** 6 | * The flushable function that outputs text in lines. 7 | */ 8 | type LinesOutput = (value: string | typeof none) => Generator; 9 | 10 | /** 11 | * Creates a flushable function that outputs text in lines. 12 | * @returns a splitter function 13 | */ 14 | declare function lines(): LinesOutput; 15 | -------------------------------------------------------------------------------- /tests/test-errors.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import chain from '../src/index.js'; 6 | 7 | test('errors: no streams', t => { 8 | t.throws(() => { 9 | chain([]); 10 | t.fail("shouldn't be here"); 11 | }); 12 | }); 13 | 14 | test('errors: wrong stream', t => { 15 | t.throws(() => { 16 | chain([1]); 17 | t.fail("shouldn't be here"); 18 | }); 19 | }); 20 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Tests", 11 | "program": "${workspaceFolder}/tests/tests.js" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /src/jsonl/parserStream.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./parserStream.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const asStream = require('../asStream'); 6 | const parser = require('./parser.js'); 7 | 8 | const parserStream = options => { 9 | const reviver = options && options.reviver, 10 | ignoreErrors = options && options.ignoreErrors; 11 | return asStream( 12 | parser({reviver, ignoreErrors}), 13 | Object.assign({writableObjectMode: false, readableObjectMode: true}, options) 14 | ); 15 | }; 16 | 17 | module.exports = parserStream; 18 | -------------------------------------------------------------------------------- /src/utils/fixUtf8Stream.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import {none} from '../defs'; 4 | 5 | export = fixUtf8Stream; 6 | 7 | /** 8 | * Converts buffers to UTF-8 strings and outputs them on the correct character boundaries. 9 | */ 10 | type FixOutput = (chunk: string | Buffer | typeof none) => string; 11 | 12 | /** 13 | * Creates a function that converts buffers to UTF-8 strings and outputs them on the correct character boundaries. 14 | * @returns a converter function 15 | */ 16 | declare function fixUtf8Stream(): FixOutput; 17 | -------------------------------------------------------------------------------- /src/utils/skipWhile.d.ts: -------------------------------------------------------------------------------- 1 | import {none} from '../defs'; 2 | 3 | export = skipWhile; 4 | 5 | /** 6 | * Creates a function that skips values while `fn` returns `true`. 7 | * @param fn a function that takes a value and returns a boolean 8 | * @returns a function that takes a value and returns a value or {@link none} when skipping 9 | */ 10 | declare function skipWhile(fn: (value: T) => boolean): (value: T) => T | typeof none; 11 | declare function skipWhile( 12 | fn: (value: T) => Promise 13 | ): (value: T) => Promise; 14 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "npm" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /src/asStream.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import {Duplex, DuplexOptions} from 'node:stream'; 4 | import {TypedDuplex} from './typed-streams'; 5 | import {Arg0, Ret} from './defs'; 6 | 7 | export = asStream; 8 | 9 | /** 10 | * Wraps a function in a duplex stream 11 | * @param fn function to wrap 12 | * @param options options for the wrapping duplex stream 13 | * @returns a duplex stream 14 | */ 15 | declare function asStream unknown>( 16 | fn: F, 17 | options?: DuplexOptions 18 | ): TypedDuplex, Ret>; 19 | -------------------------------------------------------------------------------- /src/utils/take.d.ts: -------------------------------------------------------------------------------- 1 | import {none, stop} from '../defs'; 2 | 3 | export = take; 4 | 5 | /** 6 | * Creates a function that takes `n` elements. 7 | * @param n number of elements 8 | * @param finalValue a value that is returned when `n` elements are taken. It can be {@link none} or {@link stop}. It defaults to {@link none}. 9 | * @returns a function that takes a value and returns a value or {@link finalValue} when `n` elements are taken 10 | */ 11 | declare function take( 12 | n: number, 13 | finalValue?: typeof none | typeof stop = none 14 | ): (value: T) => T | typeof finalValue; 15 | -------------------------------------------------------------------------------- /src/utils/batch.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./batch.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {none, flushable} = require('../defs'); 6 | 7 | const batch = (n = 100) => { 8 | let buffer = []; 9 | return flushable(value => { 10 | if (value === none) { 11 | // clean up buffer 12 | if (!buffer.length) return none; 13 | const result = buffer; 14 | buffer = null; 15 | return result; 16 | } 17 | buffer.push(value); 18 | if (buffer.length < n) return none; 19 | const result = buffer; 20 | buffer = []; 21 | return result; 22 | }); 23 | }; 24 | 25 | module.exports = batch; 26 | -------------------------------------------------------------------------------- /src/utils/skipWhile.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./skipWhile.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {none} = require('../defs'); 6 | 7 | const skipWhile = fn => { 8 | let test = true; 9 | return value => { 10 | if (!test) return value; 11 | const result = fn(value); 12 | if (result && typeof result.then == 'function') { 13 | return result.then(result => { 14 | if (result) return none; 15 | test = false; 16 | return value; 17 | }); 18 | } 19 | if (result) return none; 20 | test = false; 21 | return value; 22 | }; 23 | }; 24 | 25 | module.exports = skipWhile; 26 | -------------------------------------------------------------------------------- /src/utils/lines.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./lines.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {none, flushable} = require('../defs'); 6 | 7 | const lines = () => { 8 | let rest = ''; 9 | return flushable(function* (value) { 10 | if (value === none) { 11 | if (!rest) return; 12 | const result = rest; 13 | rest = ''; 14 | yield result; 15 | return; 16 | } 17 | const lines = value.split(/\r?\n/g); 18 | rest += lines[0]; 19 | if (lines.length < 2) return; 20 | lines[0] = rest; 21 | rest = lines.pop(); 22 | yield* lines; 23 | }); 24 | }; 25 | 26 | module.exports = lines; 27 | -------------------------------------------------------------------------------- /src/utils/fold.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./fold.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {none, flushable} = require('../defs'); 6 | 7 | const fold = (fn, acc) => 8 | flushable(value => { 9 | if (value === none) { 10 | // clean up acc 11 | const result = acc; 12 | acc = null; 13 | return result; 14 | } 15 | const result = fn(acc, value); 16 | if (result && typeof result.then == 'function') { 17 | return result.then(result => { 18 | acc = result; 19 | return none; 20 | }); 21 | } 22 | acc = result; 23 | return none; 24 | }); 25 | 26 | module.exports = fold; 27 | -------------------------------------------------------------------------------- /src/utils/scan.d.ts: -------------------------------------------------------------------------------- 1 | export = scan; 2 | 3 | /** 4 | * Creates a function that scans values into an accumulator. 5 | * @param fn a function that takes an accumulator and a value and returns an accumulator 6 | * @param acc an initial accumulator 7 | * @returns a function that takes a value and returns an accumulator 8 | * @remarks It is a companion for `fold()`. Unlike `fold()` it returns the current accumulator for each value. 9 | */ 10 | declare function scan(fn: (acc: A, value: T) => A, acc: A): (value: T) => A; 11 | declare function scan( 12 | fn: (acc: A, value: T) => Promise, 13 | acc: A 14 | ): (value: T) => Promise; 15 | -------------------------------------------------------------------------------- /src/utils/batch.d.ts: -------------------------------------------------------------------------------- 1 | import {none} from '../defs'; 2 | 3 | export = batch; 4 | 5 | /** 6 | * Batch values into arrays of `n` elements. 7 | */ 8 | type BatchOutput = (value: T | typeof none) => (T[] | typeof none); 9 | 10 | /** 11 | * Creates a function that batches values into arrays of `n` elements. 12 | * @param n number of elements in a batch 13 | * @returns a flushable function that batches values 14 | * @remarks The returned function is a {@link BatchOutput}. It collects values into batches (arrays) of `n` elements. The last batch can have less than `n` elements. 15 | */ 16 | declare function batch(n?: number): BatchOutput; 17 | -------------------------------------------------------------------------------- /src/utils/takeWhile.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./takeWhile.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {none} = require('../defs'); 6 | 7 | const takeWhile = (fn, finalValue = none) => { 8 | let test = true; 9 | return value => { 10 | if (!test) return finalValue; 11 | const result = fn(value); 12 | if (result && typeof result.then == 'function') { 13 | return result.then(result => { 14 | if (result) return value; 15 | test = false; 16 | return finalValue; 17 | }); 18 | } 19 | if (result) return value; 20 | test = false; 21 | return finalValue; 22 | }; 23 | }; 24 | 25 | module.exports = takeWhile; 26 | -------------------------------------------------------------------------------- /bench/gen-fun.mjs: -------------------------------------------------------------------------------- 1 | import gen from 'stream-chain/gen.js'; 2 | import fun from 'stream-chain/fun.js'; 3 | import {getManyValues} from 'stream-chain/defs.js'; 4 | 5 | const fns = [x => x - 2, x => x + 1, x => 2 * x, x => x + 2, x => x >> 1]; 6 | 7 | const g = gen(...fns), 8 | f = fun(...fns); 9 | 10 | export default { 11 | async gen(n) { 12 | let acc = 0; 13 | for (let i = 0; i < n; ++i) { 14 | for await (const x of g(i)) { 15 | acc += x; 16 | } 17 | } 18 | return acc; 19 | }, 20 | async fun(n) { 21 | let acc = 0; 22 | for (let i = 0; i < n; ++i) { 23 | for (const x of getManyValues(await f(i))) { 24 | acc += x; 25 | } 26 | } 27 | return acc; 28 | } 29 | }; 30 | -------------------------------------------------------------------------------- /src/fun.d.ts: -------------------------------------------------------------------------------- 1 | import type {Many, AsFlatList, Arg0, Ret, Fn} from './defs'; 2 | import type {FnList} from './gen'; 3 | 4 | export = fun; 5 | 6 | /** 7 | * Returns a wrapped identity function. Rarely used. 8 | */ 9 | declare function fun(): (arg: any) => Promise>; 10 | 11 | /** 12 | * Returns a function that applies the given functions in sequence wrapping them as 13 | * an asynchronous function. 14 | * @param fns functions to be wrapped 15 | * @returns an asynchronous function 16 | * @remark It collects values and return them as a {@link Many}. 17 | */ 18 | declare function fun( 19 | ...fns: FnList, L> 20 | ): AsFlatList extends readonly [Fn, ...Fn[]] 21 | ? (arg: Arg0) => Promise>> 22 | : (arg: any) => Promise>; 23 | -------------------------------------------------------------------------------- /src/utils/fold.d.ts: -------------------------------------------------------------------------------- 1 | import {none} from '../defs'; 2 | 3 | export = fold; 4 | 5 | /** 6 | * Folds values into an accumulator. Returns the accumulator when the source is exhausted. 7 | * @param fn function that takes an accumulator and a value and returns an accumulator 8 | * @param acc initial accumulator 9 | * @returns a function that takes a value and returns an accumulator or {@link none}. 10 | * @remarks It is modelled on the [reduce](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/reduce) method. 11 | */ 12 | declare function fold( 13 | fn: (acc: A, value: T) => A, 14 | acc: A 15 | ): (value: T | typeof none) => A | typeof none; 16 | declare function fold( 17 | fn: (acc: A, value: T) => Promise, 18 | acc: A 19 | ): (value: T | typeof none) => Promise; 20 | -------------------------------------------------------------------------------- /ts-check/json.ts: -------------------------------------------------------------------------------- 1 | import chain, {type Arg0, type Ret, type ChainItem} from 'stream-chain'; 2 | 3 | import parser from 'stream-chain/jsonl/parser.js'; 4 | import parserStream from 'stream-chain/jsonl/parserStream.js'; 5 | import stringerStream from 'stream-chain/jsonl/stringerStream.js'; 6 | 7 | import fs, { createReadStream, createWriteStream } from 'node:fs'; 8 | 9 | const pipeline1 = chain([ 10 | createReadStream('input.json'), 11 | parser(), 12 | ({value}: {value: number}) => value + 1, 13 | stringerStream(), 14 | createWriteStream('output.json') 15 | ] as const); 16 | 17 | void pipeline1; 18 | 19 | const pipeline2 = chain([ 20 | createReadStream('input.json'), 21 | parserStream(), 22 | ({value}: {value: number}) => value + 1, 23 | stringerStream(), 24 | createWriteStream('output.json') 25 | ] as const); 26 | 27 | void pipeline2; 28 | -------------------------------------------------------------------------------- /src/utils/takeWhile.d.ts: -------------------------------------------------------------------------------- 1 | import {none, stop} from './defs'; 2 | 3 | export = takeWhile; 4 | 5 | /** 6 | * Creates a function that takes values while `fn` returns `true`. 7 | * @param fn a function that takes a value and returns a boolean 8 | * @param finalValue a value that is returned when `fn` returns `false`. It can be {@link none} or {@link stop}. It defaults to {@link none}. 9 | * @returns a function that takes a value and returns a value or {@link finalValue} when {@link fn} returns `false` 10 | */ 11 | declare function takeWhile( 12 | fn: (value: T) => boolean, 13 | finalValue?: typeof none | typeof stop = none 14 | ): (value: unknown) => T | typeof finalValue; 15 | declare function takeWhile( 16 | fn: (value: T) => Promise, 17 | finalValue?: typeof none | typeof stop = none 18 | ): (value: unknown) => Promise; 19 | -------------------------------------------------------------------------------- /src/utils/takeWithSkip.d.ts: -------------------------------------------------------------------------------- 1 | import {none, stop} from '../defs'; 2 | 3 | export = takeWithSkip; 4 | 5 | /** 6 | * Creates a function that takes `n` elements after skipping `skip` elements. 7 | * @param n number of elements to take 8 | * @param skip number of elements to skip (defaults to 0) 9 | * @param finalValue a value that is returned when `n` elements are taken. It can be {@link none} or {@link stop}. It defaults to {@link none}. 10 | * @returns a function that takes a value and returns a value or {@link finalValue} when `n` elements are taken. It returns {@link none} when `skip` elements are skipped. 11 | * @remarks This function is more efficient than `skip()` followed by `take()`. 12 | */ 13 | declare function takeWithSkip( 14 | n: number, 15 | skip?: number, 16 | finalValue?: typeof none | typeof stop = none 17 | ): (value: T) => T | typeof finalValue; 18 | -------------------------------------------------------------------------------- /tests/test-batch.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import chain from '../src/index.js'; 6 | 7 | import {readString} from './helpers.mjs'; 8 | import parser from '../src/jsonl/parser.js'; 9 | 10 | import batch from '../src/utils/batch.js'; 11 | 12 | test.asPromise('batch: smoke test', (t, resolve) => { 13 | const pattern = [0, 1, true, false, null, {}, [], {a: 'b'}, ['c']], 14 | result = [], 15 | pipeline = chain([ 16 | readString(pattern.map(value => JSON.stringify(value)).join('\n')), 17 | parser(), 18 | batch(2) 19 | ]); 20 | 21 | pipeline.output.on('data', batch => { 22 | t.ok(batch.length == 2 || batch.length == 1); 23 | batch.forEach(object => (result[object.key] = object.value)); 24 | }); 25 | pipeline.output.on('end', () => { 26 | t.deepEqual(pattern, result); 27 | resolve(); 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /src/jsonl/parser.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./parser.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {none} = require('../defs.js'); 6 | const gen = require('../gen.js'); 7 | const fixUtf8Stream = require('../utils/fixUtf8Stream'); 8 | const lines = require('../utils/lines'); 9 | 10 | const parse = reviver => string => JSON.parse(string, reviver); 11 | 12 | const checkedParse = reviver => string => { 13 | try { 14 | return JSON.parse(string, reviver); 15 | } catch (_) { 16 | // squelch 17 | return none; 18 | } 19 | }; 20 | 21 | const parser = options => { 22 | const reviver = (options && options.reviver) || options, 23 | ignoreErrors = options && options.ignoreErrors, 24 | parseFn = ignoreErrors ? checkedParse(reviver) : parse(reviver); 25 | let counter = 0; 26 | return gen(fixUtf8Stream(), lines(), string => ({key: counter++, value: parseFn(string)})); 27 | }; 28 | 29 | module.exports = parser; 30 | -------------------------------------------------------------------------------- /tests/manual/asStreamTest.js: -------------------------------------------------------------------------------- 1 | 2 | const {PassThrough} = require('node:stream'); 3 | 4 | const defs = require('../../src/defs'); 5 | const asStream = require('../../src/AsStream'); 6 | 7 | const s = asStream(x => x * x); 8 | // const s = asStream(async x => x * x); 9 | // const s = asStream(function* (x) { for (let i = 0; i < x; ++i) yield i; }); 10 | // const s = asStream(async function* (x) { for (let i = 0; i < x; ++i) yield i; }); 11 | 12 | // const s = asStream(x => defs.none); 13 | // const s = asStream(x => defs.finalValue(42)); 14 | // const s = asStream(x => defs.many(['a', x, 'b'])); 15 | // const s = asStream(x => defs.stop); 16 | 17 | const h = new PassThrough({writableObjectMode: true, readableObjectMode: true}); 18 | const p = h.pipe(s); 19 | 20 | p.on('data', data => console.log('DATA:', data)); 21 | p.on('end', () => console.log('END')); 22 | 23 | h.write(1); 24 | h.write(2); 25 | h.write(3); 26 | h.write(4); 27 | h.end(); 28 | -------------------------------------------------------------------------------- /src/utils/fixUtf8Stream.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./fixUtf8Stream.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {StringDecoder} = require('string_decoder'); 6 | 7 | const {none, flushable} = require('../defs'); 8 | 9 | const fixUtf8Stream = () => { 10 | const stringDecoder = new StringDecoder(); 11 | let input = ''; 12 | return flushable(chunk => { 13 | if (chunk === none) { 14 | const result = input + stringDecoder.end(); 15 | input = ''; 16 | return result; 17 | } 18 | if (typeof chunk == 'string') { 19 | if (!input) return chunk; 20 | const result = input + chunk; 21 | input = ''; 22 | return result; 23 | } 24 | if (chunk instanceof Buffer) { 25 | const result = input + stringDecoder.write(chunk); 26 | input = ''; 27 | return result; 28 | } 29 | throw new TypeError('Expected a string or a Buffer'); 30 | }); 31 | }; 32 | 33 | module.exports = fixUtf8Stream; 34 | -------------------------------------------------------------------------------- /src/jsonl/parserStream.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import {Duplex, DuplexOptions} from 'node:stream'; 4 | import {TypedDuplex} from '../typed-streams'; 5 | 6 | export = parserStream; 7 | 8 | /** 9 | * Options for the parser stream based on `DuplexOptions` with some additional properties. 10 | */ 11 | interface ParserOptions extends DuplexOptions { 12 | /** 13 | * An optional reviver function suitable for `JSON.parse()`. 14 | * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse 15 | */ 16 | reviver?: (this: unknown, key: string, value: unknown) => unknown; 17 | /** Whether to ignore errors silently. It defaults to `false`. */ 18 | ignoreErrors?: boolean; 19 | } 20 | 21 | /** 22 | * Returns a JSONL parser as a duplex stream. 23 | * @param options options for the parser stream (see {@link ParserOptions}) 24 | * @returns a duplex stream 25 | */ 26 | declare function parserStream( 27 | options?: ParserOptions 28 | ): TypedDuplex; 29 | -------------------------------------------------------------------------------- /src/utils/readableFrom.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import {Readable, ReadableOptions} from 'node:stream'; 4 | import {TypedReadable} from '../typed-streams'; 5 | 6 | export = readableFrom; 7 | 8 | /** 9 | * A function or an iterable that will be used as a data source. 10 | */ 11 | type Iter = (() => T) | (() => Promise) | Iterable | AsyncIterable; 12 | 13 | /** 14 | * Options for the `readableFrom` function based on `ReadableOptions` with some additional properties. 15 | */ 16 | interface ReadableFromOptions extends ReadableOptions { 17 | /** An iterable or a function that will be used as a data source. */ 18 | iterable?: Iter; 19 | } 20 | 21 | /** 22 | * Creates a readable stream from an iterable or a function that will be used as a data source. 23 | * @param options readable options (see {@link ReadableFromOptions}) or an iterable or a function that will be used as a data source. 24 | * @returns a readable stream 25 | */ 26 | declare function readableFrom(options: Iter | ReadableFromOptions): TypedReadable; 27 | -------------------------------------------------------------------------------- /ts-check/utils.ts: -------------------------------------------------------------------------------- 1 | import chain from 'stream-chain'; 2 | 3 | import skip from 'stream-chain/utils/skip.js'; 4 | import skipWhile from 'stream-chain/utils/skipWhile.js'; 5 | import take from 'stream-chain/utils/take.js'; 6 | import takeWhile from 'stream-chain/utils/takeWhile.js'; 7 | import takeWithSkip from 'stream-chain/utils/takeWithSkip.js'; 8 | 9 | import fold from 'stream-chain/utils/fold.js'; 10 | import scan from 'stream-chain/utils/scan.js'; 11 | 12 | import reduceStream from 'stream-chain/utils/reduceStream.js'; 13 | 14 | import batch from 'stream-chain/utils/batch.js'; 15 | 16 | const pipeline1 = chain([ 17 | skipWhile((x: number) => x < 3), 18 | takeWhile((x: number) => x > 0), 19 | skip(2), 20 | take(10), 21 | takeWithSkip(5, 2), 22 | scan((acc: number, x: number) => acc + x, 0), 23 | fold((acc: string, x: number) => acc + x, '') 24 | ] as const); 25 | void pipeline1; 26 | 27 | const pipeline2 = chain([ 28 | batch(10), 29 | reduceStream((acc: number, x: number[]) => acc + x.length, 0) 30 | ] as const); 31 | void pipeline2; 32 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: Node.js CI 5 | 6 | on: 7 | push: 8 | branches: ['*'] 9 | pull_request: 10 | branches: [master] 11 | 12 | jobs: 13 | tests: 14 | name: Node.js ${{matrix.node-version}} on ${{matrix.os}} 15 | runs-on: ${{matrix.os}} 16 | 17 | strategy: 18 | matrix: 19 | os: [ubuntu-latest] 20 | node-version: [18, 20, 22] 21 | 22 | steps: 23 | - uses: actions/checkout@v4 24 | with: 25 | submodules: true 26 | - name: Setup Node.js ${{matrix.node-version}} 27 | uses: actions/setup-node@v4 28 | with: 29 | node-version: ${{matrix.node-version}} 30 | - name: Install the package and run tests 31 | run: | 32 | npm ci 33 | npm run build --if-present 34 | npm test 35 | npm run ts-check --if-present 36 | -------------------------------------------------------------------------------- /tests/test-web-stream.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import chain from '../src/index.js'; 6 | 7 | test.asPromise('web stream: transform', (t, resolve) => { 8 | if (typeof Bun == 'object') { 9 | // these tests are only for Node.js 10 | resolve(); 11 | return; 12 | } 13 | 14 | if (!globalThis.ReadableStream) { 15 | resolve(); 16 | return; 17 | } 18 | 19 | const output = [], 20 | c = chain([ 21 | new ReadableStream({ 22 | start(controller) { 23 | controller.enqueue(1); 24 | controller.enqueue(2); 25 | controller.enqueue(3); 26 | controller.close(); 27 | } 28 | }), 29 | new TransformStream({ 30 | transform(x, controller) { 31 | controller.enqueue(x * x); 32 | } 33 | }), 34 | x => 2 * x + 1, 35 | new WritableStream({ 36 | write(x) { 37 | output.push(x); 38 | } 39 | }) 40 | ]); 41 | 42 | c.on('end', () => { 43 | t.deepEqual(output, [3, 9, 19]); 44 | resolve(); 45 | }); 46 | }); 47 | -------------------------------------------------------------------------------- /src/typed-streams.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import {Duplex, Readable, Transform, Writable} from 'node:stream'; 4 | 5 | /** 6 | * Technical class to add input/output types to duplex streams. 7 | */ 8 | export declare class TypedDuplex extends Duplex { 9 | __streamTypeR(): R { 10 | return null as R; 11 | } 12 | __streamTypeW(): W { 13 | return null as W; 14 | } 15 | } 16 | 17 | /** 18 | * Technical class to add output type to readable streams. 19 | */ 20 | export declare class TypedReadable extends Readable { 21 | __streamTypeR(): R { 22 | return null as R; 23 | } 24 | } 25 | 26 | /** 27 | * Technical class to add input/output types to transform streams. 28 | */ 29 | export declare class TypedTransform extends Transform { 30 | __streamTypeR(): R { 31 | return null as R; 32 | } 33 | __streamTypeW(): W { 34 | return null as W; 35 | } 36 | } 37 | 38 | /** 39 | * Technical class to add input type to writable streams. 40 | */ 41 | export declare class TypedWritable extends Writable { 42 | __streamTypeW(): W { 43 | return null as W; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /ts-check/fun.ts: -------------------------------------------------------------------------------- 1 | import fun from 'stream-chain/fun.js'; 2 | 3 | const f0 = fun(); 4 | const f1 = fun((x: number) => x * x); 5 | const f2 = fun( 6 | (x: number) => x * x, 7 | (x: number) => String(x).split(' ') 8 | ); 9 | const f3 = fun( 10 | (x: number) => x * x, 11 | [f0, f1, f2] as const, 12 | (x: string[]) => x[0], 13 | [null, undefined] as const, 14 | (x: string) => !x.split(' '), 15 | (x: boolean) => !x 16 | ); 17 | const f4 = fun([ 18 | (x: number) => x * x, 19 | [f0, f1, f2], 20 | (x: string[]) => x[0], 21 | [null, undefined], 22 | (x: string) => !x.split(' '), 23 | (x: boolean) => !x 24 | ] as const); 25 | const f5 = fun( 26 | async (x: number) => x * x, 27 | function* (x: number) { 28 | yield* [x - 1, x, x + 1]; 29 | }, 30 | async function* (x: number) { 31 | for (let i = x; i > 0; --i) { 32 | yield i; 33 | } 34 | }, 35 | (x: any) => String(x) 36 | ); 37 | 38 | const fns: ((arg: any) => any)[] = [f3, (x: boolean) => Number(x), f4]; 39 | const f6 = fun(...fns); 40 | const f7 = fun( 41 | function* () { 42 | for (let i = 0; i < 10; ++i) { 43 | yield i; 44 | } 45 | }, 46 | f6 47 | ); 48 | 49 | void f5; 50 | void f7; 51 | -------------------------------------------------------------------------------- /tests/test-skip.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {streamToArray, delay} from './helpers.mjs'; 6 | import chain from '../src/index.js'; 7 | import fromIterable from '../src/utils/readableFrom.js'; 8 | 9 | import skip from '../src/utils/skip.js'; 10 | import skipWhile from '../src/utils/skipWhile.js'; 11 | 12 | test.asPromise('skip: smoke test', (t, resolve) => { 13 | const output = [], 14 | c = chain([fromIterable([1, 2, 3, 4, 5]), skip(2), streamToArray(output)]); 15 | 16 | c.on('end', () => { 17 | t.deepEqual(output, [3, 4, 5]); 18 | resolve(); 19 | }); 20 | }); 21 | 22 | test.asPromise('skip: while', (t, resolve) => { 23 | const output = [], 24 | c = chain([fromIterable([1, 2, 3, 4, 5]), skipWhile(x => x != 3), streamToArray(output)]); 25 | 26 | c.on('end', () => { 27 | t.deepEqual(output, [3, 4, 5]); 28 | resolve(); 29 | }); 30 | }); 31 | 32 | test.asPromise('skip: while async', (t, resolve) => { 33 | const output = [], 34 | c = chain([ 35 | fromIterable([1, 2, 3, 4, 5]), 36 | skipWhile(delay(x => x != 3)), 37 | streamToArray(output) 38 | ]); 39 | 40 | c.on('end', () => { 41 | t.deepEqual(output, [3, 4, 5]); 42 | resolve(); 43 | }); 44 | }); 45 | -------------------------------------------------------------------------------- /src/jsonl/parser.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import {Buffer} from 'node:buffer'; 4 | 5 | export = parser; 6 | 7 | /** 8 | * The JSONL parser output. 9 | */ 10 | interface OutputItem { 11 | /** The key: a sequential number starting from 0. */ 12 | key: number; 13 | /** The parsed value. */ 14 | value: any; 15 | } 16 | 17 | /** 18 | * The reviver function prototype required by `JSON.parse()`. 19 | * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse 20 | */ 21 | type Reviver = (this: unknown, key: string, value: unknown) => unknown; 22 | 23 | type ParserOptions = { 24 | /** An optional reviver function for `JSON.parse()`. */ 25 | reviver?: Reviver; 26 | /** Whether to ignore errors silently. It defaults to `false`. */ 27 | ignoreErrors?: boolean; 28 | }; 29 | 30 | /** 31 | * The JSONL parser as a streamable generator. 32 | * @param reviver an optional reviver function (see {@link Reviver}) or an {@link ParserOptions} 33 | * @returns an asynchronous generator 34 | * @remark parsers JSON lines items returning them as {@link OutputItem}. 35 | */ 36 | declare function parser( 37 | reviver?: Reviver | ParserOptions 38 | ): (x: string | Buffer) => AsyncGenerator; 39 | -------------------------------------------------------------------------------- /bench/gen-opt.mjs: -------------------------------------------------------------------------------- 1 | import gen from 'stream-chain/gen.js'; 2 | import {clearFunctionList} from 'stream-chain/defs.js'; 3 | 4 | const g1 = gen( 5 | x => x - 2, 6 | x => x + 1, 7 | x => 2 * x, 8 | x => x + 2, 9 | x => x >> 1 10 | ), 11 | g2 = gen( 12 | x => x - 2, 13 | gen( 14 | x => x + 1, 15 | x => 2 * x, 16 | x => x + 2 17 | ), 18 | x => x >> 1 19 | ), 20 | g3 = gen( 21 | x => x - 2, 22 | clearFunctionList( 23 | gen( 24 | x => x + 1, 25 | x => 2 * x, 26 | x => x + 2 27 | ) 28 | ), 29 | x => x >> 1 30 | ); 31 | 32 | export default { 33 | async ['simple list'](n) { 34 | let acc = 0; 35 | for (let i = 0; i < n; ++i) { 36 | for await (const x of g1(i)) { 37 | acc += x; 38 | } 39 | } 40 | return acc; 41 | }, 42 | async ['optimization on'](n) { 43 | let acc = 0; 44 | for (let i = 0; i < n; ++i) { 45 | for await (const x of g2(i)) { 46 | acc += x; 47 | } 48 | } 49 | return acc; 50 | }, 51 | async ['optimization off'](n) { 52 | let acc = 0; 53 | for (let i = 0; i < n; ++i) { 54 | for await (const x of g3(i)) { 55 | acc += x; 56 | } 57 | } 58 | return acc; 59 | } 60 | }; 61 | -------------------------------------------------------------------------------- /src/utils/reduceStream.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./reduceStream.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {Writable} = require('node:stream'); 6 | 7 | const defaultInitial = 0; 8 | const defaultReducer = (acc, value) => value; 9 | 10 | const reduceStream = (options, initial) => { 11 | if (!options || !options.reducer) { 12 | options = {reducer: options, initial}; 13 | } 14 | let accumulator = defaultInitial, 15 | reducer = defaultReducer; 16 | if (options) { 17 | 'initial' in options && (accumulator = options.initial); 18 | 'reducer' in options && (reducer = options.reducer); 19 | } 20 | 21 | const stream = new Writable( 22 | Object.assign({objectMode: true}, options, { 23 | write(chunk, _, callback) { 24 | const result = reducer.call(this, this.accumulator, chunk); 25 | if (result && typeof result.then == 'function') { 26 | result.then( 27 | value => { 28 | this.accumulator = value; 29 | callback(null); 30 | }, 31 | error => callback(error) 32 | ); 33 | } else { 34 | this.accumulator = result; 35 | callback(null); 36 | } 37 | } 38 | }) 39 | ); 40 | stream.accumulator = accumulator; 41 | 42 | return stream; 43 | }; 44 | 45 | module.exports = reduceStream; 46 | -------------------------------------------------------------------------------- /tests/helpers.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import {Readable, Writable} from 'node:stream'; 4 | 5 | export const streamToArray = array => 6 | new Writable({ 7 | objectMode: true, 8 | write(chunk, _, callback) { 9 | array.push(chunk); 10 | callback(null); 11 | } 12 | }); 13 | 14 | export const readString = (string, quant) => new Readable({ 15 | read() { 16 | if (isNaN(quant) || quant < 1) { 17 | this.push(string); 18 | } else if (string instanceof Buffer) { 19 | for (let i = 0; i < string.length; i += quant) { 20 | this.push(string.slice(i, i + quant)); 21 | } 22 | } else { 23 | for (let i = 0; i < string.length; i += quant) { 24 | this.push(string.substr(i, quant)); 25 | } 26 | } 27 | this.push(null); 28 | } 29 | }); 30 | 31 | export const writeToArray = array => new Writable({ 32 | write(chunk, _, callback) { 33 | if (typeof chunk == 'string') { 34 | array.push(chunk); 35 | } else { 36 | array.push(chunk.toString('utf8')); 37 | } 38 | callback(null); 39 | } 40 | }); 41 | 42 | export const delay = (fn, ms = 20) => (...args) => 43 | new Promise((resolve, reject) => { 44 | setTimeout(() => { 45 | try { 46 | resolve(fn(...args)); 47 | } catch (error) { 48 | reject(error); 49 | } 50 | }, ms); 51 | }); 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2005-2024 Eugene Lazutkin 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 12 | -------------------------------------------------------------------------------- /ts-check/gen.ts: -------------------------------------------------------------------------------- 1 | import gen from 'stream-chain/gen.js'; 2 | 3 | const g0 = gen(); 4 | const g1 = gen((x: number) => x * x); 5 | const g2 = gen( 6 | (x: number) => x * x, 7 | (x: number) => String(x).split(' ') 8 | ); 9 | const g3 = gen( 10 | (x: number) => x * x, 11 | [g0, g1, g2] as const, 12 | (x: string[]) => x[0], 13 | [null, undefined] as const, 14 | (x: string) => !x.split(' '), 15 | (x: boolean) => !x 16 | ); 17 | const g4 = gen([ 18 | (x: number) => x * x, 19 | [g0, g1, g2], 20 | (x: string[]) => x[0], 21 | [null, undefined], 22 | (x: string) => !x.split(' '), 23 | (x: boolean) => !x 24 | ] as const); 25 | const g5 = gen( 26 | async (x: number) => x * x, 27 | function* (x: number) { 28 | yield* [x - 1, x, x + 1]; 29 | }, 30 | async function* (x: number) { 31 | for (let i = x; i > 0; --i) { 32 | yield i; 33 | } 34 | }, 35 | (x: any) => String(x) 36 | ); 37 | 38 | const fns: ((arg: any) => any)[] = [g3, (x: boolean) => Number(x), g4]; 39 | const g6 = gen(...fns); 40 | const g7 = gen( 41 | function* () { 42 | for (let i = 0; i < 10; ++i) { 43 | yield i; 44 | } 45 | }, 46 | g6 47 | ); 48 | 49 | void g5; 50 | void g7; 51 | 52 | const g8 = gen(...[g3, (x: boolean) => Number(x), g4]); 53 | const g9 = gen(...[g3, (x: boolean) => Number(x), g4] as ((arg: any) => any)[]); 54 | 55 | void g8; 56 | void g9; 57 | 58 | const g10 = gen(null); 59 | const g11 = gen(undefined, null); 60 | 61 | void g10; 62 | void g11; 63 | -------------------------------------------------------------------------------- /src/jsonl/stringerStream.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./stringerStream.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {Transform} = require('node:stream'); 6 | 7 | const stringer = options => { 8 | let first = true, 9 | prefix = '', 10 | suffix = '', 11 | separator = '\n', 12 | emptyValue, 13 | replacer, 14 | space; 15 | if (options) { 16 | if (typeof options.prefix == 'string') prefix = options.prefix; 17 | if (typeof options.suffix == 'string') suffix = options.suffix; 18 | if (typeof options.separator == 'string') separator = options.separator; 19 | if (typeof options.emptyValue == 'string') emptyValue = options.emptyValue; 20 | replacer = options.replacer; 21 | space = options.space; 22 | } 23 | return new Transform( 24 | Object.assign({writableObjectMode: true}, options, { 25 | transform(value, _, callback) { 26 | let result = JSON.stringify(value, replacer, space); 27 | if (first) { 28 | first = false; 29 | result = prefix + result; 30 | } else { 31 | result = separator + result; 32 | } 33 | this.push(result); 34 | callback(null); 35 | }, 36 | flush(callback) { 37 | let output; 38 | if (first) { 39 | output = typeof emptyValue == 'string' ? emptyValue : prefix + suffix; 40 | } else { 41 | output = suffix; 42 | } 43 | output && this.push(output); 44 | callback(null); 45 | } 46 | }) 47 | ); 48 | }; 49 | 50 | module.exports = stringer; 51 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "stream-chain", 3 | "version": "3.4.0", 4 | "description": "Chain functions as transform streams.", 5 | "type": "commonjs", 6 | "main": "./src/index.js", 7 | "types": "./src/index.d.ts", 8 | "exports": { 9 | ".": "./src/index.js", 10 | "./*": "./src/*" 11 | }, 12 | "scripts": { 13 | "debug": "node --inspect-brk tests/tests.js", 14 | "test": "tape6 --flags FO", 15 | "test:bun": "tape6-bun --flags FO", 16 | "test:proc": "tape6-proc --flags FO", 17 | "test:proc:bun": "bun run `npx tape6-proc --self` --flags FO", 18 | "ts-check": "tsc --noEmit", 19 | "ts-demo": "node --experimental-strip-types ts-test/demo.mts" 20 | }, 21 | "repository": { 22 | "type": "git", 23 | "url": "git+https://github.com/uhop/stream-chain.git" 24 | }, 25 | "keywords": [ 26 | "stream", 27 | "chain" 28 | ], 29 | "author": "Eugene Lazutkin (https://www.lazutkin.com/)", 30 | "funding": "https://github.com/sponsors/uhop", 31 | "license": "BSD-3-Clause", 32 | "bugs": { 33 | "url": "https://github.com/uhop/stream-chain/issues" 34 | }, 35 | "homepage": "https://github.com/uhop/stream-chain#readme", 36 | "files": [ 37 | "src", 38 | "LICENSE", 39 | "README.md" 40 | ], 41 | "tape6": { 42 | "tests": [ 43 | "/tests/test-*.mjs" 44 | ] 45 | }, 46 | "devDependencies": { 47 | "@types/node": "^22.10.7", 48 | "nano-benchmark": "^1.0.4", 49 | "tape-six": "^1.0.3", 50 | "tape-six-proc": "^1.0.1", 51 | "typescript": "^5.7.3" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/jsonl/stringerStream.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import {Transform} from 'node:stream'; 4 | import {TypedTransform} from '../typed-streams'; 5 | 6 | export = stringer; 7 | 8 | /** 9 | * Options for the stringer stream used to control the output. 10 | */ 11 | interface StringerOptions { 12 | /** The prefix string. It is prepended to the output. Defaults to `""`. */ 13 | prefix?: string; 14 | /** The suffix string. It is appended to the output. Defaults to `""`. */ 15 | suffix?: string; 16 | /** The separator string used between items. Defaults to `"\n"`. */ 17 | separator?: string; 18 | /** 19 | * The empty value string. It is used when no values were streamed. Defaults to `prefix + suffix`. 20 | * See {@link StringerOptions.prefix} and {@link StringerOptions.suffix}. 21 | */ 22 | emptyValue?: string; 23 | /** 24 | * The optional replacer function used by `JSON.stringify()`. 25 | * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify 26 | */ 27 | replacer?: (this: unknown, key: string, value: unknown) => unknown; 28 | /** 29 | * The optional space string or number used by `JSON.stringify()`. 30 | * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify 31 | */ 32 | space?: string | number; 33 | } 34 | 35 | /** 36 | * Returns a JSONL stringer as a duplex stream. 37 | * @param options options for the stringer stream (see {@link StringerOptions}) 38 | * @returns a duplex stream 39 | */ 40 | declare function stringer(options?: any): TypedTransform; 41 | -------------------------------------------------------------------------------- /ts-test/demo.mts: -------------------------------------------------------------------------------- 1 | import chain, {asStream, many, none} from 'stream-chain'; 2 | import {TypedTransform} from 'stream-chain/typed-streams.js'; 3 | import readableFrom from 'stream-chain/utils/readableFrom.js'; 4 | 5 | import {Transform} from 'node:stream'; 6 | 7 | const getTotalFromDatabaseByKey = async (x: number) => 8 | new Promise(resolve => { 9 | setTimeout(() => { 10 | resolve(Math.min(x % 10, 3)); 11 | }, 20); 12 | }); 13 | 14 | const c = chain([ 15 | // transforms a value 16 | (x: number) => x * x, 17 | // returns several values 18 | (x: number) => many([x - 1, x, x + 1]), 19 | // waits for an asynchronous operation 20 | async (x: number) => await getTotalFromDatabaseByKey(x), 21 | // or: (x: number) => getTotalFromDatabaseByKey(x), 22 | // returns multiple values with a generator 23 | function* (x: number) { 24 | for (let i = x; i >= 0; --i) { 25 | yield i; 26 | } 27 | }, 28 | // filters out even values 29 | (x: number) => (x % 2 ? x : none), 30 | // uses an arbitrary transform stream 31 | new Transform({ 32 | objectMode: true, 33 | transform(x, _, callback) { 34 | callback(null, x + 1); 35 | } 36 | }), 37 | // can skip falsy values 38 | [null, undefined], 39 | // uses a typed transform stream 40 | new TypedTransform({ 41 | objectMode: true, 42 | transform(x, _, callback) { 43 | callback(null, String(x + 1)); 44 | } 45 | }), 46 | // uses a wrapped function 47 | asStream((x: string) => !x) 48 | ] as const), 49 | output: boolean[] = []; 50 | c.on('data', (data: boolean) => output.push(data)); 51 | c.on('finish', () => console.log(output)); 52 | 53 | readableFrom([1, 2, 3]).pipe(c); 54 | -------------------------------------------------------------------------------- /src/gen.d.ts: -------------------------------------------------------------------------------- 1 | import type {Arg0, Ret, AsFlatList, Fn} from './defs'; 2 | 3 | export = gen; 4 | 5 | /** 6 | * Returns a type, which was expected from a list item. 7 | * It is used to highlight mismatches between argument types and return types in a list. 8 | */ 9 | export type FnItem = F extends readonly [infer F1, ...infer R] 10 | ? F1 extends (null | undefined) 11 | ? readonly [F1, ...FnList] 12 | : readonly [FnItem, ...FnList, R>] 13 | : F extends readonly unknown[] 14 | ? readonly [FnItem] 15 | : F extends Fn 16 | ? I extends Arg0 17 | ? F 18 | : (arg: I, ...rest: readonly unknown[]) => ReturnType 19 | : F extends (null | undefined) 20 | ? F 21 | : never; 22 | 23 | /** 24 | * Replicates a tuple verifying the types of the list items so arguments match returns. 25 | * The replicated tuple is used to highlight mismatches between list items. 26 | */ 27 | export type FnList = L extends readonly [infer F1, ...infer R] 28 | ? F1 extends (null | undefined) 29 | ? readonly [F1, ...FnList] 30 | : readonly [FnItem, ...FnList, R>] 31 | : L; 32 | 33 | /** 34 | * Returns a wrapped identity function. Rarely used. 35 | */ 36 | declare function gen(): (arg: any) => AsyncGenerator; 37 | /** 38 | * Returns a function that applies the given functions in sequence wrapping them as 39 | * an asynchronous generator. 40 | * @param fns functions to be wrapped 41 | * @returns an asynchronous generator 42 | */ 43 | declare function gen( 44 | ...fns: FnList, L> 45 | ): AsFlatList extends readonly [Fn, ...Fn[]] 46 | ? (arg: Arg0) => AsyncGenerator, void, unknown> 47 | : (arg: any) => AsyncGenerator; 48 | -------------------------------------------------------------------------------- /src/utils/reduceStream.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import {Writable, WritableOptions} from 'stream'; 4 | import {TypedWritable} from '../typed-streams'; 5 | 6 | export = reduceStream; 7 | 8 | /** A reducer function prototype */ 9 | type Reducer = (this: ReduceStreamOutput, acc: A, value: T) => A; 10 | /** An asynchronous reducer function prototype */ 11 | type ReducerPromise = (this: ReduceStreamOutput, acc: A, value: T) => Promise; 12 | 13 | /** 14 | * Options for the `reduceStream` function based on `WritableOptions` with some additional properties. 15 | */ 16 | interface ReduceStreamOptions extends WritableOptions { 17 | /** A reducer function. */ 18 | reducer?: Reducer | ReducerPromise; 19 | /** An initial accumulator. */ 20 | initial?: A; 21 | } 22 | 23 | /** 24 | * A writable stream that contains an accumulator as a property. 25 | */ 26 | interface ReduceStreamOutput extends TypedWritable { 27 | accumulator: A; 28 | } 29 | 30 | /** 31 | * Creates a writable stream that contains an accumulator as a property. 32 | * @param options options for the reduceStream (see {@link ReduceStreamOptions}) 33 | * @returns a writable stream 34 | * @remarks It is modelled on the [reduce](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/reduce) method. 35 | */ 36 | declare function reduceStream( 37 | options: ReduceStreamOptions 38 | ): ReduceStreamOutput; 39 | /** 40 | * Creates a writable stream that contains an accumulator as a property. 41 | * @param reducer a reducer function 42 | * @param initial an initial accumulator 43 | * @returns a writable stream 44 | * @remarks It is modelled on the [reduce](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/reduce) method. 45 | */ 46 | declare function reduceStream( 47 | reducer: Reducer | ReducerPromise, 48 | initial: A 49 | ): ReduceStreamOutput; 50 | -------------------------------------------------------------------------------- /tests/test-take.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {streamToArray, delay} from './helpers.mjs'; 6 | import chain, {stop} from '../src/index.js'; 7 | import fromIterable from '../src/utils/readableFrom.js'; 8 | 9 | import take from '../src/utils/take.js'; 10 | import takeWhile from '../src/utils/takeWhile.js'; 11 | import takeWithSkip from '../src/utils/takeWithSkip.js'; 12 | 13 | test.asPromise('take: smoke test', (t, resolve) => { 14 | const output = [], 15 | c = chain([fromIterable([1, 2, 3, 4, 5]), take(2), streamToArray(output)]); 16 | 17 | c.on('end', () => { 18 | t.deepEqual(output, [1, 2]); 19 | resolve(); 20 | }); 21 | }); 22 | 23 | test.asPromise('simple: with skip', (t, resolve) => { 24 | const output = [], 25 | c = chain([fromIterable([1, 2, 3, 4, 5]), takeWithSkip(2, 2), streamToArray(output)]); 26 | 27 | c.on('end', () => { 28 | t.deepEqual(output, [3, 4]); 29 | resolve(); 30 | }); 31 | }); 32 | 33 | test.asPromise('simple: while', (t, resolve) => { 34 | const output = [], 35 | c = chain([fromIterable([1, 2, 3, 4, 5]), takeWhile(x => x != 3), streamToArray(output)]); 36 | 37 | c.on('end', () => { 38 | t.deepEqual(output, [1, 2]); 39 | resolve(); 40 | }); 41 | }); 42 | 43 | test.asPromise('simple: while async', (t, resolve) => { 44 | const output = [], 45 | c = chain([ 46 | fromIterable([1, 2, 3, 4, 5]), 47 | takeWhile(delay(x => x != 3)), 48 | streamToArray(output) 49 | ]); 50 | 51 | c.on('end', () => { 52 | t.deepEqual(output, [1, 2]); 53 | resolve(); 54 | }); 55 | }); 56 | 57 | test.asPromise('simple: stop', (t, resolve) => { 58 | const output = [], 59 | c = chain([fromIterable([1, 2, 3, 4, 5]), take(2, stop), streamToArray(output)]); 60 | 61 | c.on('end', () => { 62 | t.deepEqual(output, [1, 2]); 63 | resolve(); 64 | }); 65 | }); 66 | 67 | test.asPromise('simple: stop with skip', (t, resolve) => { 68 | const output = [], 69 | c = chain([fromIterable([1, 2, 3, 4, 5]), takeWithSkip(2, 2, stop), streamToArray(output)]); 70 | 71 | c.on('end', () => { 72 | t.deepEqual(output, [3, 4]); 73 | resolve(); 74 | }); 75 | }); 76 | -------------------------------------------------------------------------------- /tests/test-readableFrom.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {streamToArray, delay} from './helpers.mjs'; 6 | import chain from '../src/index.js'; 7 | 8 | import readableFrom from '../src/utils/readableFrom.js'; 9 | 10 | test.asPromise('readableFrom: smoke test', (t, resolve) => { 11 | const output = [], 12 | c = chain([readableFrom([1, 2, 3]), streamToArray(output)]); 13 | 14 | c.on('end', () => { 15 | t.deepEqual(output, [1, 2, 3]); 16 | resolve(); 17 | }); 18 | }); 19 | 20 | test.asPromise('readableFrom: function', (t, resolve) => { 21 | const output = [], 22 | c = chain([readableFrom(() => 0), streamToArray(output)]); 23 | 24 | c.on('end', () => { 25 | t.deepEqual(output, [0]); 26 | resolve(); 27 | }); 28 | }); 29 | 30 | test.asPromise('readableFrom: async function', (t, resolve) => { 31 | const output = [], 32 | c = chain([readableFrom(delay(() => 0)), streamToArray(output)]); 33 | 34 | c.on('end', () => { 35 | t.deepEqual(output, [0]); 36 | resolve(); 37 | }); 38 | }); 39 | 40 | test.asPromise('readableFrom: generator', (t, resolve) => { 41 | const output = [], 42 | c = chain([ 43 | readableFrom(function* () { 44 | yield 0; 45 | yield 1; 46 | }), 47 | streamToArray(output) 48 | ]); 49 | 50 | c.on('end', () => { 51 | t.deepEqual(output, [0, 1]); 52 | resolve(); 53 | }); 54 | }); 55 | 56 | test.asPromise('readableFrom: async generator', (t, resolve) => { 57 | const output = [], 58 | c = chain([ 59 | readableFrom(async function* () { 60 | yield delay(() => 0)(); 61 | yield delay(() => 1)(); 62 | }), 63 | streamToArray(output) 64 | ]); 65 | 66 | c.on('end', () => { 67 | t.deepEqual(output, [0, 1]); 68 | resolve(); 69 | }); 70 | }); 71 | 72 | test.asPromise('readableFrom: nextable', (t, resolve) => { 73 | const output = [], 74 | c = chain([ 75 | readableFrom( 76 | (function* () { 77 | yield 0; 78 | yield 1; 79 | })() 80 | ), 81 | streamToArray(output) 82 | ]); 83 | 84 | c.on('end', () => { 85 | t.deepEqual(output, [0, 1]); 86 | resolve(); 87 | }); 88 | }); 89 | -------------------------------------------------------------------------------- /tests/test-dataSource.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {streamToArray, delay} from './helpers.mjs'; 6 | import chain, {dataSource} from '../src/index.js'; 7 | 8 | test.asPromise('dataSource: smoke test', (t, resolve) => { 9 | const output = [], 10 | c = chain([dataSource([1, 2, 3]), streamToArray(output)]); 11 | 12 | c.on('end', () => { 13 | t.deepEqual(output, [1, 2, 3]); 14 | resolve(); 15 | }); 16 | 17 | c.end(1); // start the chain 18 | }); 19 | 20 | test.asPromise('dataSource: function', (t, resolve) => { 21 | const output = [], 22 | c = chain([dataSource(() => 0), streamToArray(output)]); 23 | 24 | c.on('end', () => { 25 | t.deepEqual(output, [0]); 26 | resolve(); 27 | }); 28 | 29 | c.end(1); // start the chain 30 | }); 31 | 32 | test.asPromise('dataSource: async function', (t, resolve) => { 33 | const output = [], 34 | c = chain([dataSource(delay(() => 0)), streamToArray(output)]); 35 | 36 | c.on('end', () => { 37 | t.deepEqual(output, [0]); 38 | resolve(); 39 | }); 40 | 41 | c.end(1); // start the chain 42 | }); 43 | 44 | test.asPromise('dataSource: generator', (t, resolve) => { 45 | const output = [], 46 | c = chain([ 47 | dataSource(function* () { 48 | yield 0; 49 | yield 1; 50 | }), 51 | streamToArray(output) 52 | ]); 53 | 54 | c.on('end', () => { 55 | t.deepEqual(output, [0, 1]); 56 | resolve(); 57 | }); 58 | 59 | c.end(1); // start the chain 60 | }); 61 | 62 | test.asPromise('dataSource: async generator', (t, resolve) => { 63 | const output = [], 64 | c = chain([ 65 | dataSource(async function* () { 66 | yield delay(() => 0)(); 67 | yield delay(() => 1)(); 68 | }), 69 | streamToArray(output) 70 | ]); 71 | 72 | c.on('end', () => { 73 | t.deepEqual(output, [0, 1]); 74 | resolve(); 75 | }); 76 | 77 | c.end(1); // start the chain 78 | }); 79 | 80 | test.asPromise('dataSource: nextable', (t, resolve) => { 81 | const output = [], 82 | c = chain([ 83 | dataSource( 84 | (function* () { 85 | yield 0; 86 | yield 1; 87 | })() 88 | ), 89 | streamToArray(output) 90 | ]); 91 | 92 | c.on('end', () => { 93 | t.deepEqual(output, [0, 1]); 94 | resolve(); 95 | }); 96 | 97 | c.end(1); // start the chain 98 | }); 99 | -------------------------------------------------------------------------------- /src/gen.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./gen.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const defs = require('./defs'); 6 | 7 | const next = async function* (value, fns, index) { 8 | for (let i = index; i <= fns.length; ++i) { 9 | if (value && typeof value.then == 'function') { 10 | // thenable 11 | value = await value; 12 | } 13 | if (value === defs.none) break; 14 | if (value === defs.stop) throw new defs.Stop(); 15 | if (defs.isFinalValue(value)) { 16 | yield defs.getFinalValue(value); 17 | break; 18 | } 19 | if (defs.isMany(value)) { 20 | const values = defs.getManyValues(value); 21 | if (i == fns.length) { 22 | yield* values; 23 | } else { 24 | for (let j = 0; j < values.length; ++j) { 25 | yield* next(values[j], fns, i); 26 | } 27 | } 28 | break; 29 | } 30 | if (value && typeof value.next == 'function') { 31 | // generator 32 | for (;;) { 33 | let data = value.next(); 34 | if (data && typeof data.then == 'function') { 35 | data = await data; 36 | } 37 | if (data.done) break; 38 | if (i == fns.length) { 39 | yield data.value; 40 | } else { 41 | yield* next(data.value, fns, i); 42 | } 43 | } 44 | break; 45 | } 46 | if (i == fns.length) { 47 | yield value; 48 | break; 49 | } 50 | const f = fns[i]; 51 | value = f(value); 52 | } 53 | }; 54 | 55 | const gen = (...fns) => { 56 | fns = fns 57 | .filter(fn => fn) 58 | .flat(Infinity) 59 | .map(fn => (defs.isFunctionList(fn) ? defs.getFunctionList(fn) : fn)) 60 | .flat(Infinity); 61 | if (!fns.length) { 62 | fns = [x => x]; 63 | } 64 | let flushed = false; 65 | let g = async function* (value) { 66 | if (flushed) throw Error('Call to a flushed pipe.'); 67 | if (value !== defs.none) { 68 | yield* next(value, fns, 0); 69 | } else { 70 | flushed = true; 71 | for (let i = 0; i < fns.length; ++i) { 72 | const f = fns[i]; 73 | if (defs.isFlushable(f)) { 74 | yield* next(f(defs.none), fns, i + 1); 75 | } 76 | } 77 | } 78 | }; 79 | const needToFlush = fns.some(fn => defs.isFlushable(fn)); 80 | if (needToFlush) g = defs.flushable(g); 81 | return defs.setFunctionList(g, fns); 82 | }; 83 | 84 | module.exports = gen; 85 | 86 | module.exports.next = next; 87 | -------------------------------------------------------------------------------- /tests/test-readWrite.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {streamToArray} from './helpers.mjs'; 6 | import chain, {dataSource} from '../src/index.js'; 7 | import fromIterable from '../src/utils/readableFrom.js'; 8 | 9 | test.asPromise('readWrite: readable', (t, resolve) => { 10 | const output1 = [], 11 | output2 = [], 12 | c = chain([fromIterable([1, 2, 3]), x => x * x]); 13 | 14 | c.pipe(streamToArray(output1)); 15 | 16 | c.on('data', value => output2.push(value)); 17 | c.on('end', () => { 18 | t.deepEqual(output1, [1, 4, 9]); 19 | t.deepEqual(output2, [1, 4, 9]); 20 | resolve(); 21 | }); 22 | }); 23 | 24 | test.asPromise('readWrite: writable', (t, resolve) => { 25 | const output = [], 26 | c = chain([x => x * x, streamToArray(output)]); 27 | 28 | fromIterable([1, 2, 3]).pipe(c); 29 | 30 | c.on('end', () => { 31 | t.deepEqual(output, [1, 4, 9]); 32 | resolve(); 33 | }); 34 | }); 35 | 36 | test.asPromise('readWrite: readable and writable', (t, resolve) => { 37 | const output = [], 38 | c = chain([fromIterable([1, 2, 3]), x => x * x, streamToArray(output)]); 39 | 40 | c.on('end', () => { 41 | t.deepEqual(output, [1, 4, 9]); 42 | resolve(); 43 | }); 44 | }); 45 | 46 | test.asPromise('readWrite: single readable', (t, resolve) => { 47 | const output1 = [], 48 | output2 = [], 49 | c = chain([fromIterable([1, 2, 3])]); 50 | 51 | c.pipe(streamToArray(output1)); 52 | 53 | c.on('data', value => output2.push(value)); 54 | c.on('end', () => { 55 | t.deepEqual(output1, [1, 2, 3]); 56 | t.deepEqual(output2, [1, 2, 3]); 57 | resolve(); 58 | }); 59 | }); 60 | 61 | test.asPromise('readWrite: single writable', (t, resolve) => { 62 | const output = [], 63 | c = chain([streamToArray(output)]); 64 | 65 | fromIterable([1, 2, 3]).pipe(c); 66 | 67 | c.on('end', () => { 68 | t.deepEqual(output, [1, 2, 3]); 69 | resolve(); 70 | }); 71 | }); 72 | 73 | test.asPromise('readWrite: pipeable', (t, resolve, reject) => { 74 | const output1 = [], 75 | output2 = [], 76 | c = chain([dataSource([1, 2, 3]), streamToArray(output1)]); 77 | 78 | fromIterable([4, 5, 6]).pipe(c).pipe(streamToArray(output2)); 79 | 80 | c.on('end', () => { 81 | t.deepEqual(output1, [1, 2, 3, 1, 2, 3, 1, 2, 3]); 82 | t.deepEqual(output2, []); 83 | resolve(); 84 | }); 85 | c.on('error', error => reject(error)); 86 | }); 87 | -------------------------------------------------------------------------------- /tests/manual/streamEventsTest.js: -------------------------------------------------------------------------------- 1 | const {Writable, Transform} = require('node:stream'); 2 | 3 | const makeStreamT = id => { 4 | const stream = new Transform({ 5 | readableObjectMode: true, 6 | writableObjectMode: true, 7 | transform(chunk, encoding, callback) { 8 | console.log(id + ':', 'transform', chunk, encoding); 9 | const flag = this.push(chunk, encoding); 10 | console.log(id + ':', 'transform-push', flag); 11 | callback(null); 12 | }, 13 | flush(callback) { 14 | console.log(id + ':', 'flush'); 15 | callback(null); 16 | } 17 | }); 18 | stream._id = id; 19 | stream.on('error', error => console.log(id + ':', 'event-error', error)); 20 | stream.on('end', () => console.log(id + ':', 'event-end')); 21 | stream.on('finish', () => console.log(id + ':', 'event-finish')); 22 | stream.on('close', () => console.log(id + ':', 'event-close')); 23 | stream.on('pipe', src => console.log(id + ':', 'event-pipe', src._id)); 24 | stream.on('unpipe', src => console.log(id + ':', 'event-unpipe', src._id)); 25 | return stream; 26 | }; 27 | 28 | const makeStreamW = id => { 29 | const stream = new Writable({ 30 | objectMode: true, 31 | write(chunk, encoding, callback) { 32 | console.log(id + ':', 'write', chunk, encoding); 33 | callback(null); 34 | }, 35 | final(callback) { 36 | console.log(id + ':', 'final'); 37 | callback(null); 38 | }, 39 | destroy(error, callback) { 40 | console.log(id + ':', 'destroy', error); 41 | callback(null); 42 | } 43 | }); 44 | stream._id = id; 45 | stream.on('error', error => console.log(id + ':', 'event-error', error)); 46 | stream.on('finish', () => console.log(id + ':', 'event-finish')); 47 | stream.on('close', () => console.log(id + ':', 'event-close')); 48 | stream.on('pipe', src => console.log(id + ':', 'event-pipe', src._id)); 49 | stream.on('unpipe', src => console.log(id + ':', 'event-unpipe', src._id)); 50 | return stream; 51 | }; 52 | 53 | console.log('Creating streams ...'); 54 | 55 | const a = makeStreamT('A'), 56 | b = makeStreamT('B'), 57 | c = makeStreamT('C'), 58 | w = makeStreamW('W'); 59 | 60 | console.log('Connecting streams ...'); 61 | 62 | a.pipe(b).pipe(c).pipe(w); 63 | 64 | console.log('Passing a value ...'); 65 | 66 | a.write({a: 1}); 67 | // a.end(); 68 | 69 | console.log('Destroying B ...'); 70 | 71 | // a.destroy(); 72 | a.unpipe(b); 73 | b.end(); 74 | 75 | console.log('Done.'); 76 | -------------------------------------------------------------------------------- /tests/test-demo.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {Transform} from 'node:stream'; 6 | import chain from '../src/index.js'; 7 | import readableFrom from '../src/utils/readableFrom.js'; 8 | 9 | const getTotalFromDatabaseByKey = async x => 10 | new Promise(resolve => { 11 | setTimeout(() => { 12 | resolve(Math.min(x % 10, 3)); 13 | }, 20); 14 | }); 15 | 16 | test.asPromise('demo: default', (t, resolve) => { 17 | const c = chain([ 18 | // transforms a value 19 | x => x * x, 20 | // returns several values 21 | x => chain.many([x - 1, x, x + 1]), 22 | // waits for an asynchronous operation 23 | async x => await getTotalFromDatabaseByKey(x), 24 | // returns multiple values with a generator 25 | function* (x) { 26 | for (let i = x; i > 0; --i) { 27 | yield i; 28 | } 29 | return 0; 30 | }, 31 | // filters out even values 32 | x => (x % 2 ? x : null), 33 | // uses an arbitrary transform stream 34 | new Transform({ 35 | objectMode: true, 36 | transform(x, _, callback) { 37 | callback(null, x + 1); 38 | } 39 | }) 40 | ]), 41 | output = []; 42 | c.on('data', data => output.push(data)); 43 | c.on('end', () => { 44 | t.deepEqual(output, [2, 2, 4, 2, 4, 2, 4, 2, 4, 2, 4, 2]); 45 | resolve(); 46 | }); 47 | 48 | readableFrom([1, 2, 3]).pipe(c); 49 | }); 50 | 51 | test.asPromise('demo: no grouping', (t, resolve) => { 52 | const c = chain( 53 | [ 54 | // transforms a value 55 | x => x * x, 56 | // returns several values 57 | x => chain.many([x - 1, x, x + 1]), 58 | // waits for an asynchronous operation 59 | async x => await getTotalFromDatabaseByKey(x), 60 | // returns multiple values with a generator 61 | function* (x) { 62 | for (let i = x; i > 0; --i) { 63 | yield i; 64 | } 65 | return 0; 66 | }, 67 | // filters out even values 68 | x => (x % 2 ? x : null), 69 | // uses an arbitrary transform stream 70 | new Transform({ 71 | objectMode: true, 72 | transform(x, _, callback) { 73 | callback(null, x + 1); 74 | } 75 | }) 76 | ], 77 | {noGrouping: true} 78 | ), 79 | output = []; 80 | c.on('data', data => output.push(data)); 81 | c.on('end', () => { 82 | t.deepEqual(output, [2, 2, 4, 2, 4, 2, 4, 2, 4, 2, 4, 2]); 83 | resolve(); 84 | }); 85 | 86 | readableFrom([1, 2, 3]).pipe(c); 87 | }); 88 | -------------------------------------------------------------------------------- /ts-test/defs.mts: -------------------------------------------------------------------------------- 1 | import { 2 | none, 3 | many, 4 | isMany, 5 | toMany, 6 | normalizeMany, 7 | getManyValues, 8 | combineMany, 9 | combineManyMut 10 | } from 'stream-chain/defs.js'; 11 | 12 | { 13 | const a = many([]), 14 | b = many([1]), 15 | c = many([1, 2]); 16 | 17 | console.assert(getManyValues(a).length === 0); 18 | console.assert(getManyValues(b).length === 1); 19 | console.assert(getManyValues(c).length === 2); 20 | 21 | console.assert(normalizeMany(a) === none); 22 | console.assert(normalizeMany(b) === 1); 23 | 24 | const x = normalizeMany(c); 25 | console.assert(isMany(x)); 26 | if (isMany(x)) console.assert(getManyValues(x).length === 2); 27 | } 28 | 29 | { 30 | const a = toMany(none), 31 | b = toMany(1), 32 | c = toMany(many([])), 33 | d = toMany(many([1, 2, 3])); 34 | 35 | console.assert(getManyValues(a).length === 0); 36 | console.assert(getManyValues(b).length === 1); 37 | console.assert(getManyValues(c).length === 0); 38 | console.assert(getManyValues(d).length === 3); 39 | } 40 | 41 | { 42 | const a = combineMany(none, none), 43 | b = combineMany(none, 1), 44 | c = combineMany(1, none), 45 | d = combineMany(1, '2'), 46 | e = combineMany(many([]), many([])), 47 | f = combineMany(many([]), many([1, 2, 3])), 48 | g = combineMany(many([1, 2, 3]), many([])), 49 | h = combineMany(many([1, '2', 3]), many([4, 5, 6])); 50 | 51 | console.assert(getManyValues(a).length === 0); 52 | console.assert(getManyValues(b).length === 1); 53 | console.assert(getManyValues(c).length === 1); 54 | console.assert(getManyValues(d).length === 2); 55 | console.assert(getManyValues(e).length === 0); 56 | console.assert(getManyValues(f).length === 3); 57 | console.assert(getManyValues(g).length === 3); 58 | console.assert(getManyValues(h).length === 6); 59 | } 60 | 61 | { 62 | const a = combineManyMut(none, none), 63 | b = combineManyMut(none, 1), 64 | c = combineManyMut(1, none), 65 | d = combineManyMut(1, '2'), 66 | e = combineManyMut(many([]), many([])), 67 | f = combineManyMut(many([]), many([1, 2, 3])), 68 | g = combineManyMut(many([1, 2, 3]), many([])), 69 | h = combineManyMut(many([1, '2', 3]), many([4, 5, 6])); 70 | 71 | console.assert(getManyValues(a).length === 0); 72 | console.assert(getManyValues(b).length === 1); 73 | console.assert(getManyValues(c).length === 1); 74 | console.assert(getManyValues(d).length === 2); 75 | console.assert(getManyValues(e).length === 0); 76 | console.assert(getManyValues(f).length === 3); 77 | console.assert(getManyValues(g).length === 3); 78 | console.assert(getManyValues(h).length === 6); 79 | } 80 | -------------------------------------------------------------------------------- /tests/test-asStream.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {streamToArray, delay} from './helpers.mjs'; 6 | import {none} from '../src/defs.js'; 7 | 8 | import asStream from '../src/asStream.js'; 9 | 10 | test.asPromise('asStream: smoke test', (t, resolve) => { 11 | const pattern = [0, 1, true, false, {}, [], {a: 'b'}, ['c']], 12 | result = [], 13 | stream = asStream(x => x), 14 | pipeline = stream.pipe(streamToArray(result)); 15 | 16 | pipeline.on('finish', () => { 17 | t.deepEqual(result, pattern); 18 | resolve(); 19 | }); 20 | 21 | pattern.forEach(value => stream.write(value)); 22 | stream.end(); 23 | }); 24 | 25 | test.asPromise('asStream: function', (t, resolve) => { 26 | const pattern = [0, 1, true, false, {}, [], {a: 'b'}, ['c']], 27 | result = [], 28 | stream = asStream(x => (x ? x : none)), 29 | pipeline = stream.pipe(streamToArray(result)); 30 | 31 | pipeline.on('finish', () => { 32 | t.deepEqual( 33 | result, 34 | pattern.filter(x => x) 35 | ); 36 | resolve(); 37 | }); 38 | 39 | pattern.forEach(value => stream.write(value)); 40 | stream.end(); 41 | }); 42 | 43 | test.asPromise('asStream: async function', (t, resolve) => { 44 | const pattern = [0, 1, true, false, {}, [], {a: 'b'}, ['c']], 45 | result = [], 46 | stream = asStream(delay(x => (x ? x : none))), 47 | pipeline = stream.pipe(streamToArray(result)); 48 | 49 | pipeline.on('finish', () => { 50 | t.deepEqual( 51 | result, 52 | pattern.filter(x => x) 53 | ); 54 | resolve(); 55 | }); 56 | 57 | pattern.forEach(value => stream.write(value)); 58 | stream.end(); 59 | }); 60 | 61 | test.asPromise('asStream: generator', (t, resolve) => { 62 | const pattern = [1, 2, 3], 63 | result = [], 64 | stream = asStream(function* () { 65 | yield* pattern; 66 | }), 67 | pipeline = stream.pipe(streamToArray(result)); 68 | 69 | pipeline.on('finish', () => { 70 | t.deepEqual(result, pattern); 71 | resolve(); 72 | }); 73 | 74 | stream.end(1); 75 | }); 76 | 77 | test.asPromise('asStream: async generator', (t, resolve) => { 78 | const pattern = [1, 2, 3], 79 | result = [], 80 | stream = asStream(async function* () { 81 | const fn = delay(x => x); 82 | yield* pattern.map(value => fn(value)); 83 | }), 84 | pipeline = stream.pipe(streamToArray(result)); 85 | 86 | pipeline.on('finish', () => { 87 | t.deepEqual(result, pattern); 88 | resolve(); 89 | }); 90 | 91 | stream.end(1); 92 | }); 93 | 94 | test('asStream: wrong argument', t => { 95 | t.throws(() => { 96 | asStream(1); 97 | t.fail("shouldn't be here"); 98 | }); 99 | }); 100 | -------------------------------------------------------------------------------- /ts-check/demo.ts: -------------------------------------------------------------------------------- 1 | import chain, {chainUnchecked, asStream, many, none} from 'stream-chain'; 2 | import {TypedTransform} from 'stream-chain/typed-streams.js'; 3 | import readableFrom from 'stream-chain/utils/readableFrom.js'; 4 | 5 | import {Transform} from 'node:stream'; 6 | 7 | const getTotalFromDatabaseByKey = async (x: number) => 8 | new Promise(resolve => { 9 | setTimeout(() => { 10 | resolve(Math.min(x % 10, 3)); 11 | }, 20); 12 | }); 13 | 14 | const c = chain([ 15 | // transforms a value 16 | (x: number) => x * x, 17 | // returns several values 18 | (x: number) => many([x - 1, x, x + 1]), 19 | // waits for an asynchronous operation 20 | async (x: number) => await getTotalFromDatabaseByKey(x), 21 | // or: (x: number) => getTotalFromDatabaseByKey(x), 22 | // returns multiple values with a generator 23 | function* (x: number) { 24 | for (let i = x; i >= 0; --i) { 25 | yield i; 26 | } 27 | }, 28 | // filters out even values 29 | (x: number) => (x % 2 ? x : none), 30 | // uses an arbitrary transform stream 31 | new Transform({ 32 | objectMode: true, 33 | transform(x, _, callback) { 34 | callback(null, x + 1); 35 | } 36 | }), 37 | // can skip falsy values 38 | [null, undefined], 39 | // uses a typed transform stream 40 | new TypedTransform({ 41 | objectMode: true, 42 | transform(x, _, callback) { 43 | callback(null, String(x + 1)); 44 | } 45 | }), 46 | // uses a wrapped function 47 | asStream((x: string) => !x) 48 | ] as const), 49 | output: boolean[] = []; 50 | c.on('data', (data: boolean) => output.push(data)); 51 | 52 | readableFrom([1, 2, 3]).pipe(c); 53 | 54 | // unchecked 55 | 56 | const c2 = chainUnchecked([ 57 | // transforms a value 58 | (x: number) => x * x, 59 | // returns several values 60 | (x: number) => many([x - 1, x, x + 1]), 61 | // waits for an asynchronous operation 62 | async (x: number) => await getTotalFromDatabaseByKey(x), 63 | // or: (x: number) => getTotalFromDatabaseByKey(x), 64 | // returns multiple values with a generator 65 | function* (x: number) { 66 | for (let i = x; i >= 0; --i) { 67 | yield i; 68 | } 69 | }, 70 | // filters out even values 71 | (x: number) => (x % 2 ? x : none), 72 | // uses an arbitrary transform stream 73 | new Transform({ 74 | objectMode: true, 75 | transform(x, _, callback) { 76 | callback(null, x + 1); 77 | } 78 | }), 79 | // can skip falsy values 80 | [null, undefined], 81 | // uses a typed transform stream 82 | new TypedTransform({ 83 | objectMode: true, 84 | transform(x, _, callback) { 85 | callback(null, String(x + 1)); 86 | } 87 | }), 88 | // uses a wrapped function 89 | asStream((x: string) => !x) 90 | ]), 91 | output2: boolean[] = []; 92 | c2.on('data', (data: boolean) => output2.push(data)); 93 | 94 | readableFrom([1, 2, 3]).pipe(c2); 95 | -------------------------------------------------------------------------------- /tests/test-jsonl-parser.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import fs from 'node:fs'; 6 | import path from 'node:path'; 7 | import zlib from 'node:zlib'; 8 | import {Writable} from 'node:stream'; 9 | 10 | import {readString} from './helpers.mjs'; 11 | import chain from '../src/index.js'; 12 | 13 | import parser from '../src/jsonl/parser.js'; 14 | 15 | const roundtrip = (t, resolve, len, quant) => { 16 | const objects = []; 17 | for (let n = 0; n < len; n += 1) { 18 | objects.push({ 19 | stringWithTabsAndNewlines: "Did it work?\nNo...\t\tI don't think so...", 20 | anArray: [n + 1, n + 2, true, 'tabs?\t\t\t\u0001\u0002\u0003', false], 21 | n 22 | }); 23 | } 24 | 25 | const json = []; 26 | for (let n = 0; n < objects.length; n += 1) { 27 | json.push(JSON.stringify(objects[n])); 28 | } 29 | 30 | const input = json.join('\n'), 31 | result = []; 32 | chain([ 33 | readString(input, quant), 34 | parser(), 35 | new Writable({ 36 | objectMode: true, 37 | write(chunk, _, callback) { 38 | result.push(chunk.value); 39 | callback(null); 40 | }, 41 | final(callback) { 42 | t.deepEqual(objects, result); 43 | resolve(); 44 | callback(null); 45 | } 46 | }) 47 | ]); 48 | }; 49 | 50 | test.asPromise('jsonl parser: smoke test', (t, resolve) => roundtrip(t, resolve)); 51 | 52 | for (let i = 1; i <= 12; ++i) { 53 | test.asPromise('jsonl parser: roundtrip with a set of objects - ' + i, (t, resolve) => { 54 | roundtrip(t, resolve, i); 55 | }); 56 | } 57 | 58 | for (let i = 1; i <= 12; ++i) { 59 | test.asPromise('jsonl parser: roundtrip with different window sizes - ' + i, (t, resolve) => { 60 | roundtrip(t, resolve, 10, i); 61 | }); 62 | } 63 | 64 | test.asPromise('jsonl parser: read file', (t, resolve) => { 65 | if (!/^file:\/\//.test(import.meta.url)) throw Error('Cannot get the current working directory'); 66 | const isWindows = path.sep === '\\', 67 | fileName = path.join( 68 | path.dirname(import.meta.url.substring(isWindows ? 8 : 7)), 69 | './data/sample.jsonl.gz' 70 | ); 71 | let count = 0; 72 | chain([ 73 | fs.createReadStream(fileName), 74 | zlib.createGunzip(), 75 | parser(), 76 | new Writable({ 77 | objectMode: true, 78 | write(chunk, _, callback) { 79 | t.equal(count, chunk.key); 80 | ++count; 81 | callback(null); 82 | }, 83 | final(callback) { 84 | t.equal(count, 100); 85 | resolve(); 86 | callback(null); 87 | } 88 | }) 89 | ]); 90 | }); 91 | 92 | test.asPromise('jsonl parser: bad json', (t, resolve) => { 93 | const pipeline = chain([readString(' not json '), parser()]); 94 | 95 | pipeline.on('data', () => t.fail("We shouldn't be here.")); 96 | pipeline.on('error', e => { 97 | t.ok(e); 98 | resolve(); 99 | }); 100 | pipeline.on('end', value => { 101 | t.fail("We shouldn't be here."); 102 | resolve(); 103 | }); 104 | }); 105 | -------------------------------------------------------------------------------- /tests/test-simple.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {Transform} from 'node:stream'; 6 | import {streamToArray, delay} from './helpers.mjs'; 7 | import chain from '../src/index.js'; 8 | import fromIterable from '../src/utils/readableFrom.js'; 9 | 10 | test.asPromise('simple: smoke test', (t, resolve) => { 11 | const c = chain([x => x * x]), 12 | output1 = [], 13 | output2 = []; 14 | 15 | fromIterable([1, 2, 3]).pipe(c).pipe(streamToArray(output1)); 16 | 17 | c.on('data', value => output2.push(value)); 18 | c.on('end', () => { 19 | t.deepEqual(output1, [1, 4, 9]); 20 | t.deepEqual(output2, [1, 4, 9]); 21 | resolve(); 22 | }); 23 | }); 24 | 25 | test.asPromise('simple: generator', (t, resolve) => { 26 | const output = [], 27 | c = chain([ 28 | fromIterable([1, 2, 3]), 29 | function* (x) { 30 | yield x * x; 31 | yield x * x * x; 32 | yield 2 * x; 33 | }, 34 | streamToArray(output) 35 | ]); 36 | 37 | c.on('end', () => { 38 | t.deepEqual(output, [1, 1, 2, 4, 8, 4, 9, 27, 6]); 39 | resolve(); 40 | }); 41 | }); 42 | 43 | test.asPromise('simple: async function', (t, resolve) => { 44 | const output = [], 45 | c = chain([fromIterable([1, 2, 3]), delay(x => x + 1), streamToArray(output)]); 46 | 47 | c.on('end', () => { 48 | t.deepEqual(output, [2, 3, 4]); 49 | resolve(); 50 | }); 51 | }); 52 | 53 | test.asPromise('simple: async function', (t, resolve) => { 54 | const output = [], 55 | c = chain([ 56 | fromIterable([1, 2, 3]), 57 | x => chain.many([x * x, x * x * x, 2 * x]), 58 | streamToArray(output) 59 | ]); 60 | 61 | c.on('end', () => { 62 | t.deepEqual(output, [1, 1, 2, 4, 8, 4, 9, 27, 6]); 63 | resolve(); 64 | }); 65 | }); 66 | 67 | test.asPromise('simple: chain', (t, resolve) => { 68 | const output = [], 69 | c = chain([fromIterable([1, 2, 3]), x => x * x, x => 2 * x + 1, streamToArray(output)]); 70 | 71 | c.on('end', () => { 72 | t.deepEqual(output, [3, 9, 19]); 73 | resolve(); 74 | }); 75 | }); 76 | 77 | test.asPromise('simple: stream', (t, resolve) => { 78 | const output = [], 79 | c = chain([ 80 | fromIterable([1, 2, 3]), 81 | new Transform({ 82 | objectMode: true, 83 | transform(x, _, callback) { 84 | callback(null, x * x); 85 | } 86 | }), 87 | x => 2 * x + 1, 88 | streamToArray(output) 89 | ]); 90 | 91 | c.on('end', () => { 92 | t.deepEqual(output, [3, 9, 19]); 93 | resolve(); 94 | }); 95 | }); 96 | 97 | test.asPromise('simple: factory', (t, resolve) => { 98 | const output = [], 99 | c = chain([ 100 | fromIterable([1, 2, 3]), 101 | function* (x) { 102 | yield x * x; 103 | yield x * x * x; 104 | yield 2 * x; 105 | }, 106 | streamToArray(output) 107 | ]); 108 | 109 | c.on('end', () => { 110 | t.deepEqual(output, [1, 1, 2, 4, 8, 4, 9, 27, 6]); 111 | resolve(); 112 | }); 113 | }); 114 | -------------------------------------------------------------------------------- /tests/test-fold.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {streamToArray, delay} from './helpers.mjs'; 6 | import chain from '../src/index.js'; 7 | import readableFrom from '../src/utils/readableFrom.js'; 8 | 9 | import fold from '../src/utils/fold.js'; 10 | import scan from '../src/utils/scan.js'; 11 | import reduce from '../src/utils/reduce.js'; 12 | import reduceStream from '../src/utils/reduceStream.js'; 13 | 14 | test.asPromise('fold: smoke test', (t, resolve) => { 15 | const output = [], 16 | c = chain([readableFrom([1, 2, 3]), fold((acc, x) => acc + x, 0), streamToArray(output)]); 17 | 18 | c.on('end', () => { 19 | t.deepEqual(output, [6]); 20 | resolve(); 21 | }); 22 | }); 23 | 24 | test.asPromise('fold: async', (t, resolve) => { 25 | const output = [], 26 | c = chain([ 27 | readableFrom([1, 2, 3]), 28 | fold( 29 | delay((acc, x) => acc + x), 30 | 0 31 | ), 32 | streamToArray(output) 33 | ]); 34 | 35 | c.on('end', () => { 36 | t.deepEqual(output, [6]); 37 | resolve(); 38 | }); 39 | }); 40 | 41 | test.asPromise('fold: scan', (t, resolve) => { 42 | const output = [], 43 | c = chain([readableFrom([1, 2, 3]), scan((acc, x) => acc + x, 0), streamToArray(output)]); 44 | 45 | c.on('end', () => { 46 | t.deepEqual(output, [1, 3, 6]); 47 | resolve(); 48 | }); 49 | }); 50 | 51 | test.asPromise('fold: scan async', (t, resolve) => { 52 | const output = [], 53 | c = chain([ 54 | readableFrom([1, 2, 3]), 55 | scan( 56 | delay((acc, x) => acc + x), 57 | 0 58 | ), 59 | streamToArray(output) 60 | ]); 61 | 62 | c.on('end', () => { 63 | t.deepEqual(output, [1, 3, 6]); 64 | resolve(); 65 | }); 66 | }); 67 | 68 | test.asPromise('fold: reduce', (t, resolve) => { 69 | const output = [], 70 | c = chain([readableFrom([1, 2, 3]), fold((acc, x) => acc + x, 0), streamToArray(output)]); 71 | 72 | c.on('end', () => { 73 | t.deepEqual(output, [6]); 74 | resolve(); 75 | }); 76 | }); 77 | 78 | test.asPromise('fold: reduce async', (t, resolve) => { 79 | const output = [], 80 | c = chain([ 81 | readableFrom([1, 2, 3]), 82 | reduce( 83 | delay((acc, x) => acc + x), 84 | 0 85 | ), 86 | streamToArray(output) 87 | ]); 88 | 89 | c.on('end', () => { 90 | t.deepEqual(output, [6]); 91 | resolve(); 92 | }); 93 | }); 94 | 95 | test.asPromise('fold: reduce stream', (t, resolve) => { 96 | const r = reduceStream((acc, x) => acc + x, 0); 97 | 98 | readableFrom([1, 2, 3]).pipe(r); 99 | 100 | r.on('finish', () => { 101 | t.deepEqual(r.accumulator, 6); 102 | resolve(); 103 | }); 104 | }); 105 | 106 | test.asPromise('fold: reduce stream async', (t, resolve) => { 107 | const r = reduceStream({reducer: delay((acc, x) => acc + x), initial: 0}); 108 | 109 | readableFrom([1, 2, 3]).pipe(r); 110 | 111 | r.on('finish', () => { 112 | t.deepEqual(r.accumulator, 6); 113 | resolve(); 114 | }); 115 | }); 116 | -------------------------------------------------------------------------------- /tests/test-jsonl-parserStream.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import fs from 'node:fs'; 6 | import path from 'node:path'; 7 | import zlib from 'node:zlib'; 8 | import {Writable} from 'node:stream'; 9 | 10 | import {readString} from './helpers.mjs'; 11 | 12 | import parserStream from '../src/jsonl/parserStream.js'; 13 | 14 | const roundtrip = (t, resolve, len, quant) => { 15 | const objects = []; 16 | for (let n = 0; n < len; n += 1) { 17 | objects.push({ 18 | stringWithTabsAndNewlines: "Did it work?\nNo...\t\tI don't think so...", 19 | anArray: [n + 1, n + 2, true, 'tabs?\t\t\t\u0001\u0002\u0003', false], 20 | n 21 | }); 22 | } 23 | 24 | const json = []; 25 | for (let n = 0; n < objects.length; n += 1) { 26 | json.push(JSON.stringify(objects[n])); 27 | } 28 | 29 | const input = json.join('\n'), 30 | result = []; 31 | readString(input, quant) 32 | .pipe(parserStream()) 33 | .pipe( 34 | new Writable({ 35 | objectMode: true, 36 | write(chunk, _, callback) { 37 | result.push(chunk.value); 38 | callback(null); 39 | }, 40 | final(callback) { 41 | t.deepEqual(objects, result); 42 | resolve(); 43 | callback(null); 44 | } 45 | }) 46 | ); 47 | }; 48 | 49 | test.asPromise('jsonl parserStream: smoke test', (t, resolve) => roundtrip(t, resolve)); 50 | 51 | for (let i = 1; i <= 12; ++i) { 52 | test.asPromise('jsonl parserStream: roundtrip with a set of objects - ' + i, (t, resolve) => { 53 | roundtrip(t, resolve, i); 54 | }); 55 | } 56 | 57 | for (let i = 1; i <= 12; ++i) { 58 | test.asPromise( 59 | 'jsonl parserStream: roundtrip with different window sizes - ' + i, 60 | (t, resolve) => { 61 | roundtrip(t, resolve, 10, i); 62 | } 63 | ); 64 | } 65 | 66 | test.asPromise('jsonl parserStream: read file', (t, resolve) => { 67 | if (!/^file:\/\//.test(import.meta.url)) throw Error('Cannot get the current working directory'); 68 | const isWindows = path.sep === '\\', 69 | fileName = path.join( 70 | path.dirname(import.meta.url.substring(isWindows ? 8 : 7)), 71 | './data/sample.jsonl.gz' 72 | ); 73 | let count = 0; 74 | fs.createReadStream(fileName) 75 | .pipe(zlib.createGunzip()) 76 | .pipe(parserStream()) 77 | .pipe( 78 | new Writable({ 79 | objectMode: true, 80 | write(chunk, _, callback) { 81 | t.equal(count, chunk.key); 82 | ++count; 83 | callback(null); 84 | }, 85 | final(callback) { 86 | t.equal(count, 100); 87 | resolve(); 88 | callback(null); 89 | } 90 | }) 91 | ); 92 | }); 93 | 94 | test.asPromise('jsonl parserStream: bad json', (t, resolve) => { 95 | const pipeline = readString(' not json ').pipe(parserStream()); 96 | 97 | pipeline.on('data', () => t.fail("We shouldn't be here.")); 98 | pipeline.on('error', e => { 99 | t.ok(e); 100 | resolve(); 101 | }); 102 | pipeline.on('end', value => { 103 | t.fail("We shouldn't be here."); 104 | resolve(); 105 | }); 106 | }); 107 | -------------------------------------------------------------------------------- /tests/test-defs.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import { 6 | getManyValues, 7 | many, 8 | none, 9 | toMany, 10 | normalizeMany, 11 | combineMany, 12 | combineManyMut 13 | } from '../src/defs.js'; 14 | 15 | test('defs: toMany()', t => { 16 | t.deepEqual(getManyValues(toMany(none)), []); 17 | t.deepEqual(getManyValues(toMany(1)), [1]); 18 | t.deepEqual(getManyValues(toMany(many([]))), []); 19 | t.deepEqual(getManyValues(toMany(many([1, 2, 3]))), [1, 2, 3]); 20 | }); 21 | 22 | test('defs: normalizeMany()', t => { 23 | t.deepEqual(normalizeMany(none), none); 24 | t.deepEqual(normalizeMany(1), 1); 25 | t.deepEqual(normalizeMany(many([])), none); 26 | t.deepEqual(normalizeMany(many([1])), 1); 27 | t.deepEqual(getManyValues(normalizeMany(many([1, 2, 3]))), [1, 2, 3]); 28 | }); 29 | 30 | test('defs: combineMany()', t => { 31 | t.deepEqual(getManyValues(combineMany(none, none)), []); 32 | t.deepEqual(getManyValues(combineMany(none, 2)), [2]); 33 | t.deepEqual(getManyValues(combineMany(1, none)), [1]); 34 | t.deepEqual(getManyValues(combineMany(none, many([]))), []); 35 | t.deepEqual(getManyValues(combineMany(1, many([]))), [1]); 36 | t.deepEqual(getManyValues(combineMany(none, many([1, 2, 3]))), [1, 2, 3]); 37 | t.deepEqual(getManyValues(combineMany(0, many([1, 2, 3]))), [0, 1, 2, 3]); 38 | t.deepEqual(getManyValues(combineMany(many([]), none)), []); 39 | t.deepEqual(getManyValues(combineMany(many([]), 1)), [1]); 40 | t.deepEqual(getManyValues(combineMany(many([1]), 2)), [1, 2]); 41 | t.deepEqual(getManyValues(combineMany(many([1, 2, 3]), many([]))), [1, 2, 3]); 42 | t.deepEqual(getManyValues(combineMany(many([1, 2, 3]), many([4, 5, 6]))), [1, 2, 3, 4, 5, 6]); 43 | }); 44 | 45 | test('defs: combineMany() - immutability', t => { 46 | const a = many([1, 2, 3]), 47 | b = many([4, 5, 6]), 48 | c = combineMany(a, b); 49 | t.deepEqual(getManyValues(a), [1, 2, 3]); 50 | t.deepEqual(getManyValues(b), [4, 5, 6]); 51 | t.deepEqual(getManyValues(c), [1, 2, 3, 4, 5, 6]); 52 | }); 53 | 54 | test('defs: combineManyMut()', t => { 55 | t.deepEqual(getManyValues(combineManyMut(none, none)), []); 56 | t.deepEqual(getManyValues(combineManyMut(none, 2)), [2]); 57 | t.deepEqual(getManyValues(combineManyMut(1, none)), [1]); 58 | t.deepEqual(getManyValues(combineManyMut(none, many([]))), []); 59 | t.deepEqual(getManyValues(combineManyMut(1, many([]))), [1]); 60 | t.deepEqual(getManyValues(combineManyMut(none, many([1, 2, 3]))), [1, 2, 3]); 61 | t.deepEqual(getManyValues(combineManyMut(0, many([1, 2, 3]))), [0, 1, 2, 3]); 62 | t.deepEqual(getManyValues(combineManyMut(many([]), none)), []); 63 | t.deepEqual(getManyValues(combineManyMut(many([]), 1)), [1]); 64 | t.deepEqual(getManyValues(combineManyMut(many([1]), 2)), [1, 2]); 65 | t.deepEqual(getManyValues(combineManyMut(many([1, 2, 3]), many([]))), [1, 2, 3]); 66 | t.deepEqual(getManyValues(combineManyMut(many([1, 2, 3]), many([4, 5, 6]))), [1, 2, 3, 4, 5, 6]); 67 | }); 68 | 69 | test('defs: combineManyMut() - mutability', t => { 70 | const a = many([1, 2, 3]), 71 | b = many([4, 5, 6]), 72 | c = combineManyMut(a, b); 73 | t.deepEqual(getManyValues(a), [1, 2, 3, 4, 5, 6]); 74 | t.deepEqual(getManyValues(b), [4, 5, 6]); 75 | t.deepEqual(getManyValues(c), [1, 2, 3, 4, 5, 6]); 76 | }); 77 | -------------------------------------------------------------------------------- /src/defs.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./defs.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const none = Symbol.for('object-stream.none'); 6 | const stop = Symbol.for('object-stream.stop'); 7 | 8 | const finalSymbol = Symbol.for('object-stream.final'); 9 | const manySymbol = Symbol.for('object-stream.many'); 10 | const flushSymbol = Symbol.for('object-stream.flush'); 11 | const fListSymbol = Symbol.for('object-stream.fList'); 12 | 13 | const finalValue = value => ({[finalSymbol]: 1, value}); 14 | const many = values => ({[manySymbol]: 1, values}); 15 | 16 | const isFinalValue = o => o && o[finalSymbol] === 1; 17 | const isMany = o => o && o[manySymbol] === 1; 18 | const isFlushable = o => o && o[flushSymbol] === 1; 19 | const isFunctionList = o => o && o[fListSymbol] === 1; 20 | 21 | const getFinalValue = o => o.value; 22 | const getManyValues = o => o.values; 23 | const getFunctionList = o => o.fList; 24 | 25 | const flushable = (write, final = null) => { 26 | const fn = final ? value => (value === none ? final() : write(value)) : write; 27 | fn[flushSymbol] = 1; 28 | return fn; 29 | }; 30 | 31 | const setFunctionList = (o, fns) => { 32 | o.fList = fns; 33 | o[fListSymbol] = 1; 34 | return o; 35 | }; 36 | 37 | const clearFunctionList = o => { 38 | delete o.fList; 39 | delete o[fListSymbol]; 40 | return o; 41 | }; 42 | 43 | class Stop extends Error {} 44 | 45 | const toMany = value => 46 | value === none ? many([]) : value && value[manySymbol] === 1 ? value : many([value]); 47 | 48 | const normalizeMany = o => { 49 | if (o?.[manySymbol] === 1) { 50 | switch (o.values.length) { 51 | case 0: 52 | return none; 53 | case 1: 54 | return o.values[0]; 55 | } 56 | } 57 | return o; 58 | }; 59 | 60 | const combineMany = (a, b) => { 61 | const values = a === none ? [] : a?.[manySymbol] === 1 ? a.values.slice() : [a]; 62 | if (b === none) { 63 | // do nothing 64 | } else if (b?.[manySymbol] === 1) { 65 | values.push(...b.values); 66 | } else { 67 | values.push(b); 68 | } 69 | return many(values); 70 | }; 71 | 72 | const combineManyMut = (a, b) => { 73 | const values = a === none ? [] : a?.[manySymbol] === 1 ? a.values : [a]; 74 | if (b === none) { 75 | // do nothing 76 | } else if (b?.[manySymbol] === 1) { 77 | values.push(...b.values); 78 | } else { 79 | values.push(b); 80 | } 81 | return many(values); 82 | }; 83 | 84 | // old aliases 85 | const final = finalValue; 86 | 87 | module.exports.none = none; 88 | module.exports.stop = stop; 89 | module.exports.Stop = Stop; 90 | 91 | module.exports.finalSymbol = finalSymbol; 92 | module.exports.finalValue = finalValue; 93 | module.exports.final = final; 94 | module.exports.isFinalValue = isFinalValue; 95 | module.exports.getFinalValue = getFinalValue; 96 | 97 | module.exports.manySymbol = manySymbol; 98 | module.exports.many = many; 99 | module.exports.isMany = isMany; 100 | module.exports.getManyValues = getManyValues; 101 | module.exports.getFunctionList = getFunctionList; 102 | 103 | module.exports.flushSymbol = flushSymbol; 104 | module.exports.flushable = flushable; 105 | module.exports.isFlushable = isFlushable; 106 | 107 | module.exports.fListSymbol = fListSymbol; 108 | module.exports.isFunctionList = isFunctionList; 109 | module.exports.getFunctionList = getFunctionList; 110 | module.exports.setFunctionList = setFunctionList; 111 | module.exports.clearFunctionList = clearFunctionList; 112 | 113 | module.exports.toMany = toMany; 114 | module.exports.normalizeMany = normalizeMany; 115 | module.exports.combineMany = combineMany; 116 | module.exports.combineManyMut = combineManyMut; 117 | -------------------------------------------------------------------------------- /tests/test-transducers.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {streamToArray} from './helpers.mjs'; 6 | import chain, {gen, none, finalValue, clearFunctionList} from '../src/index.js'; 7 | import fromIterable from '../src/utils/readableFrom.js'; 8 | 9 | test.asPromise('transducers: smoke test', (t, resolve) => { 10 | const output = [], 11 | c = chain([ 12 | fromIterable([1, 2, 3]), 13 | gen( 14 | x => x * x, 15 | x => 2 * x + 1 16 | ), 17 | streamToArray(output) 18 | ]); 19 | 20 | c.on('end', () => { 21 | t.deepEqual(output, [3, 9, 19]); 22 | resolve(); 23 | }); 24 | }); 25 | 26 | test.asPromise('transducers: final', (t, resolve) => { 27 | const output = [], 28 | c = chain([ 29 | fromIterable([1, 2, 3]), 30 | gen( 31 | x => x * x, 32 | x => finalValue(x), 33 | x => 2 * x + 1 34 | ), 35 | streamToArray(output) 36 | ]); 37 | 38 | c.on('end', () => { 39 | t.deepEqual(output, [1, 4, 9]); 40 | resolve(); 41 | }); 42 | }); 43 | 44 | test.asPromise('transducers: nothing', (t, resolve) => { 45 | const output = [], 46 | c = chain([ 47 | fromIterable([1, 2, 3]), 48 | gen( 49 | x => x * x, 50 | () => none, 51 | x => 2 * x + 1 52 | ), 53 | streamToArray(output) 54 | ]); 55 | 56 | c.on('end', () => { 57 | t.deepEqual(output, []); 58 | resolve(); 59 | }); 60 | }); 61 | 62 | test.asPromise('transducers: empty', (t, resolve) => { 63 | const output = [], 64 | c = chain([fromIterable([1, 2, 3]), x => x * x, gen(), streamToArray(output)]); 65 | 66 | c.on('end', () => { 67 | t.deepEqual(output, [1, 4, 9]); 68 | resolve(); 69 | }); 70 | }); 71 | 72 | test.asPromise('transducers: one', (t, resolve) => { 73 | const output = [], 74 | c = chain([fromIterable([1, 2, 3]), x => x * x, gen(x => 2 * x + 1), streamToArray(output)]); 75 | 76 | c.on('end', () => { 77 | t.deepEqual(output, [3, 9, 19]); 78 | resolve(); 79 | }); 80 | }); 81 | 82 | test.asPromise('transducers: array', (t, resolve) => { 83 | const output = [], 84 | c = chain([fromIterable([1, 2, 3]), [x => x * x, x => 2 * x + 1], streamToArray(output)]); 85 | 86 | c.on('end', () => { 87 | t.deepEqual(output, [3, 9, 19]); 88 | resolve(); 89 | }); 90 | }); 91 | 92 | test.asPromise('transducers: embedded arrays', (t, resolve) => { 93 | const output = [], 94 | c = chain([fromIterable([1, 2, 3]), [x => x * x, [x => 2 * x + 1, []]], streamToArray(output)]); 95 | 96 | c.on('end', () => { 97 | t.deepEqual(output, [3, 9, 19]); 98 | resolve(); 99 | }); 100 | }); 101 | 102 | test.asPromise('transducers: optimize function lists', (t, resolve) => { 103 | const output = [], 104 | c = chain([ 105 | fromIterable([1, 2, 3]), 106 | gen( 107 | x => x * x, 108 | x => finalValue(x), 109 | x => 2 * x + 1 110 | ), 111 | x => x + 1, 112 | streamToArray(output) 113 | ]); 114 | 115 | c.on('end', () => { 116 | t.deepEqual(output, [1, 4, 9]); 117 | resolve(); 118 | }); 119 | }); 120 | 121 | test.asPromise("transducers: don't optimize function lists", (t, resolve) => { 122 | const output = [], 123 | c = chain([ 124 | fromIterable([1, 2, 3]), 125 | clearFunctionList( 126 | gen( 127 | x => x * x, 128 | x => finalValue(x), 129 | x => 2 * x + 1 130 | ) 131 | ), 132 | x => x + 1, 133 | streamToArray(output) 134 | ]); 135 | 136 | c.on('end', () => { 137 | t.deepEqual(output, [2, 5, 10]); 138 | resolve(); 139 | }); 140 | }); 141 | -------------------------------------------------------------------------------- /src/fun.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./fun.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const defs = require('./defs'); 6 | 7 | const next = async (value, fns, index, collect) => { 8 | let cleanIndex; 9 | try { 10 | for (let i = index; i <= fns.length; ++i) { 11 | if (value && typeof value.then == 'function') { 12 | // thenable 13 | value = await value; 14 | } 15 | if (value === defs.none) break; 16 | if (value === defs.stop) { 17 | cleanIndex = i - 1; 18 | throw new defs.Stop(); 19 | } 20 | if (defs.isFinalValue(value)) { 21 | collect(defs.getFinalValue(value)); 22 | break; 23 | } 24 | if (defs.isMany(value)) { 25 | const values = defs.getManyValues(value); 26 | if (i == fns.length) { 27 | values.forEach(val => collect(val)); 28 | } else { 29 | for (let j = 0; j < values.length; ++j) { 30 | await next(values[j], fns, i, collect); 31 | } 32 | } 33 | break; 34 | } 35 | if (value && typeof value.next == 'function') { 36 | // generator 37 | for (;;) { 38 | let data = value.next(); 39 | if (data && typeof data.then == 'function') { 40 | data = await data; 41 | } 42 | if (data.done) break; 43 | if (i == fns.length) { 44 | collect(data.value); 45 | } else { 46 | await next(data.value, fns, i, collect); 47 | } 48 | } 49 | break; 50 | } 51 | if (i == fns.length) { 52 | collect(value); 53 | break; 54 | } 55 | cleanIndex = i + 1; 56 | const f = fns[i]; 57 | value = f(value); 58 | } 59 | } catch (error) { 60 | if (error instanceof defs.Stop) { 61 | await flush(fns, cleanIndex, collect); 62 | } 63 | throw error; 64 | } 65 | }; 66 | 67 | const flush = async (fns, index, collect) => { 68 | for (let i = index; i < fns.length; ++i) { 69 | const f = fns[i]; 70 | if (defs.isFlushable(f)) { 71 | await next(f(defs.none), fns, i + 1, collect); 72 | } 73 | } 74 | }; 75 | 76 | const collect = (collect, fns) => { 77 | fns = fns 78 | .filter(fn => fn) 79 | .flat(Infinity) 80 | .map(fn => (defs.isFunctionList(fn) ? defs.getFunctionList(fn) : fn)) 81 | .flat(Infinity); 82 | if (!fns.length) { 83 | fns = [x => x]; 84 | } 85 | let flushed = false; 86 | let g = async value => { 87 | if (flushed) throw Error('Call to a flushed pipe.'); 88 | if (value !== defs.none) { 89 | await next(value, fns, 0, collect); 90 | } else { 91 | flushed = true; 92 | await flush(fns, 0, collect); 93 | } 94 | }; 95 | const needToFlush = fns.some(fn => defs.isFlushable(fn)); 96 | if (needToFlush) g = defs.flushable(g); 97 | return defs.setFunctionList(g, fns); 98 | }; 99 | 100 | const asArray = (...fns) => { 101 | let results = null; 102 | const f = collect(value => results.push(value), fns); 103 | let g = async value => { 104 | results = []; 105 | await f(value); 106 | const r = results; 107 | results = null; 108 | return r; 109 | }; 110 | if (defs.isFlushable(f)) g = defs.flushable(g); 111 | return defs.setFunctionList(g, defs.getFunctionList(f)); 112 | }; 113 | 114 | const fun = (...fns) => { 115 | const f = asArray(...fns); 116 | let g = value => f(value).then(results => defs.many(results)); 117 | if (defs.isFlushable(f)) g = defs.flushable(g); 118 | return defs.setFunctionList(g, defs.getFunctionList(f)); 119 | }; 120 | 121 | module.exports = fun; 122 | 123 | module.exports.next = next; 124 | module.exports.collect = collect; 125 | module.exports.asArray = asArray; 126 | -------------------------------------------------------------------------------- /src/utils/readableFrom.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./readableFrom.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {Readable} = require('node:stream'); 6 | const defs = require('../defs'); 7 | 8 | const readableFrom = options => { 9 | if (!options || !options.iterable) { 10 | options = {iterable: options}; 11 | } 12 | let fn = options && options.iterable; 13 | if (fn && typeof fn != 'function') { 14 | if (typeof fn[Symbol.asyncIterator] == 'function') { 15 | fn = fn[Symbol.asyncIterator].bind(fn); 16 | } else if (typeof fn[Symbol.iterator] == 'function') { 17 | fn = fn[Symbol.iterator].bind(fn); 18 | } else { 19 | fn = null; 20 | } 21 | } 22 | if (!fn) 23 | throw TypeError( 24 | 'Only a function or an object with an iterator is accepted as the first argument' 25 | ); 26 | 27 | // pump variables 28 | let paused = Promise.resolve(), 29 | resolvePaused = null; 30 | const queue = []; 31 | 32 | // pause/resume 33 | const resume = () => { 34 | if (!resolvePaused) return; 35 | resolvePaused(); 36 | resolvePaused = null; 37 | paused = Promise.resolve(); 38 | }; 39 | const pause = () => { 40 | if (resolvePaused) return; 41 | paused = new Promise(resolve => (resolvePaused = resolve)); 42 | }; 43 | 44 | let stream = null; // will be assigned later 45 | 46 | // data processing 47 | const pushResults = values => { 48 | if (values && typeof values.next == 'function') { 49 | // generator 50 | queue.push(values); 51 | return; 52 | } 53 | // array 54 | queue.push(values[Symbol.iterator]()); 55 | }; 56 | const pump = async () => { 57 | while (queue.length) { 58 | await paused; 59 | const gen = queue[queue.length - 1]; 60 | let result = gen.next(); 61 | if (result && typeof result.then == 'function') { 62 | result = await result; 63 | } 64 | if (result.done) { 65 | queue.pop(); 66 | continue; 67 | } 68 | let value = result.value; 69 | if (value && typeof value.then == 'function') { 70 | value = await value; 71 | } 72 | await sanitize(value); 73 | } 74 | }; 75 | const sanitize = async value => { 76 | if (value === undefined || value === null || value === defs.none) return; 77 | if (value === defs.stop) throw new defs.Stop(); 78 | 79 | if (defs.isMany(value)) { 80 | pushResults(defs.getManyValues(value)); 81 | return pump(); 82 | } 83 | 84 | if (defs.isFinalValue(value)) { 85 | // a final value is not supported, it is treated as a regular value 86 | value = defs.getFinalValue(value); 87 | return processValue(value); 88 | } 89 | 90 | if (!stream.push(value)) { 91 | pause(); 92 | } 93 | }; 94 | const startPump = async () => { 95 | try { 96 | const value = fn(); 97 | await processValue(value); 98 | stream.push(null); 99 | } catch (error) { 100 | if (error instanceof defs.Stop) { 101 | stream.push(null); 102 | stream.destroy(); 103 | return; 104 | } 105 | throw error; 106 | } 107 | }; 108 | const processValue = async value => { 109 | if (value && typeof value.then == 'function') { 110 | // thenable 111 | return value.then(value => processValue(value)); 112 | } 113 | if (value && typeof value.next == 'function') { 114 | // generator 115 | pushResults(value); 116 | return pump(); 117 | } 118 | return sanitize(value); 119 | }; 120 | 121 | stream = new Readable( 122 | Object.assign({objectMode: true}, options, { 123 | read() { 124 | resume(); 125 | } 126 | }) 127 | ); 128 | 129 | startPump(); 130 | return stream; 131 | }; 132 | 133 | module.exports = readableFrom; 134 | -------------------------------------------------------------------------------- /src/asStream.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./asStream.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {Duplex} = require('node:stream'); 6 | const defs = require('./defs'); 7 | 8 | const asStream = (fn, options) => { 9 | if (typeof fn != 'function') 10 | throw TypeError( 11 | 'Only a function is accepted as the first argument' 12 | ); 13 | 14 | // pump variables 15 | let paused = Promise.resolve(), 16 | resolvePaused = null; 17 | const queue = []; 18 | 19 | // pause/resume 20 | const resume = () => { 21 | if (!resolvePaused) return; 22 | resolvePaused(); 23 | resolvePaused = null; 24 | paused = Promise.resolve(); 25 | }; 26 | const pause = () => { 27 | if (resolvePaused) return; 28 | paused = new Promise(resolve => (resolvePaused = resolve)); 29 | }; 30 | 31 | let stream = null; // will be assigned later 32 | 33 | // data processing 34 | const pushResults = values => { 35 | if (values && typeof values.next == 'function') { 36 | // generator 37 | queue.push(values); 38 | return; 39 | } 40 | // array 41 | queue.push(values[Symbol.iterator]()); 42 | }; 43 | const pump = async () => { 44 | while (queue.length) { 45 | await paused; 46 | const gen = queue[queue.length - 1]; 47 | let result = gen.next(); 48 | if (result && typeof result.then == 'function') { 49 | result = await result; 50 | } 51 | if (result.done) { 52 | queue.pop(); 53 | continue; 54 | } 55 | let value = result.value; 56 | if (value && typeof value.then == 'function') { 57 | value = await value; 58 | } 59 | await sanitize(value); 60 | } 61 | }; 62 | const sanitize = async value => { 63 | if (value === undefined || value === null || value === defs.none) return; 64 | if (value === defs.stop) throw new defs.Stop(); 65 | 66 | if (defs.isMany(value)) { 67 | pushResults(defs.getManyValues(value)); 68 | return pump(); 69 | } 70 | 71 | if (defs.isFinalValue(value)) { 72 | // a final value is not supported, it is treated as a regular value 73 | value = defs.getFinalValue(value); 74 | return processValue(value); 75 | } 76 | 77 | if (!stream.push(value)) { 78 | pause(); 79 | } 80 | }; 81 | const processChunk = async (chunk, encoding) => { 82 | try { 83 | const value = fn(chunk, encoding); 84 | await processValue(value); 85 | } catch (error) { 86 | if (error instanceof defs.Stop) { 87 | stream.push(null); 88 | stream.destroy(); 89 | return; 90 | } 91 | throw error; 92 | } 93 | }; 94 | const processValue = async value => { 95 | if (value && typeof value.then == 'function') { 96 | // thenable 97 | return value.then(value => processValue(value)); 98 | } 99 | if (value && typeof value.next == 'function') { 100 | // generator 101 | pushResults(value); 102 | return pump(); 103 | } 104 | return sanitize(value); 105 | }; 106 | 107 | stream = new Duplex( 108 | Object.assign({writableObjectMode: true, readableObjectMode: true}, options, { 109 | write(chunk, encoding, callback) { 110 | processChunk(chunk, encoding).then( 111 | () => callback(null), 112 | error => callback(error) 113 | ); 114 | }, 115 | final(callback) { 116 | if (!defs.isFlushable(fn)) { 117 | stream.push(null); 118 | callback(null); 119 | return; 120 | } 121 | processChunk(defs.none, null).then( 122 | () => (stream.push(null), callback(null)), 123 | error => callback(error) 124 | ); 125 | }, 126 | read() { 127 | resume(); 128 | } 129 | }) 130 | ); 131 | 132 | return stream; 133 | }; 134 | 135 | module.exports = asStream; 136 | -------------------------------------------------------------------------------- /ts-check/defs.ts: -------------------------------------------------------------------------------- 1 | import { 2 | none, 3 | isFinalValue, 4 | finalValue, 5 | getFinalValue, 6 | isMany, 7 | many, 8 | getManyValues, 9 | isFlushable, 10 | flushable, 11 | isFunctionList, 12 | getFunctionList, 13 | setFunctionList, 14 | clearFunctionList, 15 | toMany, 16 | normalizeMany, 17 | combineMany, 18 | combineManyMut 19 | } from 'stream-chain/defs.js'; 20 | 21 | { 22 | const x = finalValue(42); 23 | if (isFinalValue(x)) { 24 | const t = getFinalValue(x); 25 | void t; 26 | } 27 | const z = getFinalValue(x); 28 | void z; 29 | 30 | const w = {}; 31 | if (isFinalValue(w)) { 32 | const t = getFinalValue(w); 33 | void t; 34 | } 35 | // const v = getFinalValue(w); 36 | // void v; 37 | } 38 | 39 | { 40 | const x = many([1, 2, 3]); 41 | if (isMany(x)) { 42 | const t = getManyValues(x); 43 | void t; 44 | } 45 | const z = getManyValues(x); 46 | void z; 47 | 48 | const w = {}; 49 | if (isMany(w)) { 50 | const t = getManyValues(w); 51 | void t; 52 | } 53 | // const v = getManyValues(w); 54 | // void v; 55 | } 56 | 57 | { 58 | const x = flushable((x: number) => x * x); 59 | if (isFlushable(x)) { 60 | const t = x(42); 61 | void t; 62 | } 63 | const z = x(42); 64 | void z; 65 | 66 | const w = (x: string) => x + 'x'; 67 | if (isFlushable(w)) { 68 | const t = w('42'); 69 | void t; 70 | } 71 | } 72 | 73 | { 74 | const x = setFunctionList((x: number) => x * x, [() => 42]); 75 | if (isFunctionList(x)) { 76 | const t = getFunctionList(x); 77 | void t; 78 | } 79 | const z = getFunctionList(x); 80 | void z; 81 | 82 | const y = (x: string) => x + 'x'; 83 | if (isFunctionList(y)) { 84 | const t = getFunctionList(y); 85 | void t; 86 | } 87 | 88 | const w = clearFunctionList(x); 89 | if (isFunctionList(w)) { 90 | const t = getFunctionList(w); 91 | void t; 92 | } 93 | 94 | // const v = getFunctionList(w); 95 | // void v; 96 | 97 | void w; 98 | } 99 | 100 | { 101 | const x1 = toMany(none), 102 | t1 = getManyValues(x1); 103 | console.assert(t1.length === 0); 104 | 105 | const x2 = toMany(1), 106 | t2 = getManyValues(x2); 107 | console.assert(t2.length === 1); 108 | 109 | const x3 = toMany(many([1, 2, 3])), 110 | t3 = getManyValues(x3); 111 | console.assert(t3.length === 3); 112 | } 113 | 114 | { 115 | const x1 = normalizeMany(many([])); 116 | console.assert(x1 === none); 117 | 118 | const x2 = normalizeMany(many([1])); 119 | console.assert(x2 === 1); 120 | 121 | const x3 = normalizeMany(many([1, 2, 3])); 122 | if (isMany(x3)) { 123 | console.assert(getManyValues(x3).length === 3); 124 | } 125 | console.assert(isMany(x3)); 126 | } 127 | 128 | { 129 | const x1 = combineMany(none, none); 130 | console.assert(getManyValues(x1).length === 0); 131 | 132 | const x2 = combineMany(none, many([1, 2, 3])); 133 | console.assert(getManyValues(x2).length === 3); 134 | 135 | const x3 = combineMany(many([1, 2, 3]), none); 136 | console.assert(getManyValues(x3).length === 3); 137 | 138 | const x4 = combineMany(many([1, 2, 3]), many([4, 5, 6])); 139 | console.assert(getManyValues(x4).length === 6); 140 | 141 | const x5 = combineMany(0, many([1, 2, 3])); 142 | console.assert(getManyValues(x5).length === 4); 143 | 144 | const x6 = combineMany(many([1, 2, 3]), 4); 145 | console.assert(getManyValues(x6).length === 4); 146 | 147 | const x7 = combineMany(1, 2); 148 | console.assert(getManyValues(x7).length === 2); 149 | } 150 | 151 | { 152 | const x1 = combineManyMut(none, none); 153 | console.assert(getManyValues(x1).length === 0); 154 | 155 | const x2 = combineManyMut(none, many([1, 2, 3])); 156 | console.assert(getManyValues(x2).length === 3); 157 | 158 | const x3 = combineManyMut(many([1, 2, 3]), none); 159 | console.assert(getManyValues(x3).length === 3); 160 | 161 | const x4 = combineManyMut(many([1, 2, 3]), many([4, 5, 6])); 162 | console.assert(getManyValues(x4).length === 6); 163 | 164 | const x5 = combineManyMut(0, many([1, 2, 3])); 165 | console.assert(getManyValues(x5).length === 4); 166 | 167 | const x6 = combineManyMut(many([1, 2, 3]), 4); 168 | console.assert(getManyValues(x6).length === 4); 169 | 170 | const x7 = combineManyMut(1, 2); 171 | console.assert(getManyValues(x7).length === 2); 172 | } 173 | -------------------------------------------------------------------------------- /tests/test-jsonl-stringerStream.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {Writable, Transform} from 'node:stream'; 6 | 7 | import {readString, writeToArray} from './helpers.mjs'; 8 | 9 | import parserStream from '../src/jsonl/parserStream.js'; 10 | import stringerStream from '../src/jsonl/stringerStream.js'; 11 | 12 | test.asPromise('jsonl stringerStream: smoke test', (t, resolve) => { 13 | const pattern = { 14 | a: [[[]]], 15 | b: {a: 1}, 16 | c: {a: 1, b: 2}, 17 | d: [true, 1, "'x\"y'", null, false, true, {}, [], ''], 18 | e: 1, 19 | f: '', 20 | g: true, 21 | h: false, 22 | i: null, 23 | j: [], 24 | k: {} 25 | }, 26 | string = JSON.stringify(pattern); 27 | 28 | let buffer = ''; 29 | readString(string) 30 | .pipe(parserStream()) 31 | .pipe( 32 | new Transform({ 33 | writableObjectMode: true, 34 | readableObjectMode: true, 35 | transform(chunk, _, callback) { 36 | this.push(chunk.value); 37 | callback(null); 38 | } 39 | }) 40 | ) 41 | .pipe(stringerStream()) 42 | .pipe( 43 | new Writable({ 44 | write(chunk, _, callback) { 45 | buffer += chunk; 46 | callback(null); 47 | }, 48 | final(callback) { 49 | t.deepEqual(string, buffer); 50 | resolve(); 51 | callback(null); 52 | } 53 | }) 54 | ); 55 | }); 56 | 57 | test.asPromise('jsonl stringerStream: multiple', (t, resolve) => { 58 | const pattern = { 59 | a: [[[]]], 60 | b: {a: 1}, 61 | c: {a: 1, b: 2}, 62 | d: [true, 1, "'x\"y'", null, false, true, {}, [], ''], 63 | e: 1, 64 | f: '', 65 | g: true, 66 | h: false, 67 | i: null, 68 | j: [], 69 | k: {} 70 | }; 71 | 72 | let string = JSON.stringify(pattern), 73 | buffer = ''; 74 | string = string + '\n' + string + '\n' + string; 75 | 76 | readString(string + '\n') 77 | .pipe(parserStream()) 78 | .pipe( 79 | new Transform({ 80 | writableObjectMode: true, 81 | readableObjectMode: true, 82 | transform(chunk, _, callback) { 83 | this.push(chunk.value); 84 | callback(null); 85 | } 86 | }) 87 | ) 88 | .pipe(stringerStream()) 89 | .pipe( 90 | new Writable({ 91 | write(chunk, _, callback) { 92 | buffer += chunk; 93 | callback(null); 94 | }, 95 | final(callback) { 96 | t.deepEqual(string, buffer); 97 | resolve(); 98 | callback(null); 99 | } 100 | }) 101 | ); 102 | }); 103 | 104 | test.asPromise('jsonl stringerStream: custom separators - one value', (t, resolve) => { 105 | const output = [], 106 | stringer = stringerStream({emptyValue: '{}', prefix: '[', suffix: ']', separator: ','}), 107 | pipeline = stringer.pipe(writeToArray(output)); 108 | 109 | pipeline.on('finish', () => { 110 | t.equal(output.join(''), '[1]'); 111 | resolve(); 112 | }); 113 | 114 | stringer.end(1); 115 | }); 116 | 117 | test.asPromise('jsonl stringerStream: custom separators - two value', (t, resolve) => { 118 | const output = [], 119 | stringer = stringerStream({emptyValue: '{}', prefix: '[', suffix: ']', separator: ','}), 120 | pipeline = stringer.pipe(writeToArray(output)); 121 | 122 | pipeline.on('finish', () => { 123 | t.equal(output.join(''), '[2,1]'); 124 | resolve(); 125 | }); 126 | 127 | stringer.write(2); 128 | stringer.end(1); 129 | }); 130 | 131 | test.asPromise('jsonl stringerStream: custom separators - no value', (t, resolve) => { 132 | const output = [], 133 | stringer = stringerStream({emptyValue: '{}', prefix: '[', suffix: ']', separator: ','}), 134 | pipeline = stringer.pipe(writeToArray(output)); 135 | 136 | pipeline.on('finish', () => { 137 | t.equal(output.join(''), '{}'); 138 | resolve(); 139 | }); 140 | 141 | stringer.end(); 142 | }); 143 | 144 | test.asPromise('jsonl stringerStream: custom separators - no value (default)', (t, resolve) => { 145 | const output = [], 146 | stringer = stringerStream({prefix: '[', suffix: ']', separator: ','}), 147 | pipeline = stringer.pipe(writeToArray(output)); 148 | 149 | pipeline.on('finish', () => { 150 | t.equal(output.join(''), '[]'); 151 | resolve(); 152 | }); 153 | 154 | stringer.end(); 155 | }); 156 | -------------------------------------------------------------------------------- /tests/test-fun.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {streamToArray, delay} from './helpers.mjs'; 6 | import chain, {none, finalValue, many} from '../src/index.js'; 7 | import fromIterable from '../src/utils/readableFrom.js'; 8 | 9 | import fun from '../src/fun.js'; 10 | 11 | test.asPromise('fun: smoke test', (t, resolve) => { 12 | const output = [], 13 | c = chain([ 14 | fromIterable([1, 2, 3]), 15 | fun( 16 | x => x * x, 17 | x => 2 * x + 1 18 | ), 19 | streamToArray(output) 20 | ]); 21 | 22 | c.on('end', () => { 23 | t.deepEqual(output, [3, 9, 19]); 24 | resolve(); 25 | }); 26 | }); 27 | 28 | test.asPromise('fun: final', (t, resolve) => { 29 | const output = [], 30 | c = chain([ 31 | fromIterable([1, 2, 3]), 32 | fun( 33 | x => x * x, 34 | x => finalValue(x), 35 | x => 2 * x + 1 36 | ), 37 | streamToArray(output) 38 | ]); 39 | 40 | c.on('end', () => { 41 | t.deepEqual(output, [1, 4, 9]); 42 | resolve(); 43 | }); 44 | }); 45 | 46 | test.asPromise('fun: nothing', (t, resolve) => { 47 | const output = [], 48 | c = chain([ 49 | fromIterable([1, 2, 3]), 50 | fun( 51 | x => x * x, 52 | () => none, 53 | x => 2 * x + 1 54 | ), 55 | streamToArray(output) 56 | ]); 57 | 58 | c.on('end', () => { 59 | t.deepEqual(output, []); 60 | resolve(); 61 | }); 62 | }); 63 | 64 | test.asPromise('fun: empty', (t, resolve) => { 65 | const output = [], 66 | c = chain([fromIterable([1, 2, 3]), x => x * x, fun(), streamToArray(output)]); 67 | 68 | c.on('end', () => { 69 | t.deepEqual(output, [1, 4, 9]); 70 | resolve(); 71 | }); 72 | }); 73 | 74 | test.asPromise('fun: async', (t, resolve) => { 75 | const output = [], 76 | c = chain([ 77 | fromIterable([1, 2, 3]), 78 | fun( 79 | delay(x => x * x), 80 | x => 2 * x + 1 81 | ), 82 | streamToArray(output) 83 | ]); 84 | 85 | c.on('end', () => { 86 | t.deepEqual(output, [3, 9, 19]); 87 | resolve(); 88 | }); 89 | }); 90 | 91 | test.asPromise('fun: generator', (t, resolve) => { 92 | const output = [], 93 | c = chain([ 94 | fromIterable([1, 2, 3]), 95 | fun( 96 | x => x * x, 97 | function* (x) { 98 | yield x; 99 | yield x + 1; 100 | yield x + 2; 101 | }, 102 | x => 2 * x + 1 103 | ), 104 | streamToArray(output) 105 | ]); 106 | 107 | c.on('end', () => { 108 | t.deepEqual(output, [3, 5, 7, 9, 11, 13, 19, 21, 23]); 109 | resolve(); 110 | }); 111 | }); 112 | 113 | test.asPromise('fun: many', (t, resolve) => { 114 | const output = [], 115 | c = chain([ 116 | fromIterable([1, 2, 3]), 117 | fun( 118 | x => x * x, 119 | x => many([x, x + 1, x + 2]), 120 | x => 2 * x + 1 121 | ), 122 | streamToArray(output) 123 | ]); 124 | 125 | c.on('end', () => { 126 | t.deepEqual(output, [3, 5, 7, 9, 11, 13, 19, 21, 23]); 127 | resolve(); 128 | }); 129 | }); 130 | 131 | test.asPromise('fun: combined', (t, resolve) => { 132 | const output = [], 133 | c = chain([ 134 | fromIterable([1, 2]), 135 | fun( 136 | delay(x => -x), 137 | x => many([x, x * 10]), 138 | function* (x) { 139 | yield x; 140 | yield x - 1; 141 | }, 142 | x => -x 143 | ), 144 | streamToArray(output) 145 | ]); 146 | 147 | c.on('end', () => { 148 | t.deepEqual(output, [1, 2, 10, 11, 2, 3, 20, 21]); 149 | resolve(); 150 | }); 151 | }); 152 | 153 | test.asPromise('fun: combined final', (t, resolve) => { 154 | const output = [], 155 | c = chain([ 156 | fromIterable([1, 2]), 157 | fun( 158 | delay(x => -x), 159 | x => many([x, x * 10]), 160 | function* (x) { 161 | yield x; 162 | yield finalValue(x - 1); 163 | }, 164 | x => -x 165 | ), 166 | streamToArray(output) 167 | ]); 168 | 169 | c.on('end', () => { 170 | t.deepEqual(output, [1, -2, 10, -11, 2, -3, 20, -21]); 171 | resolve(); 172 | }); 173 | }); 174 | 175 | test.asPromise('fun: as fun', (t, resolve) => { 176 | const output = [], 177 | c = chain([ 178 | fromIterable([1, 2]), 179 | fun( 180 | delay(x => -x), 181 | x => many([x, x * 10]), 182 | function* (x) { 183 | yield x; 184 | yield finalValue(x - 1); 185 | }, 186 | x => -x 187 | ), 188 | streamToArray(output) 189 | ]); 190 | 191 | c.on('end', () => { 192 | t.deepEqual(output, [1, -2, 10, -11, 2, -3, 20, -21]); 193 | resolve(); 194 | }); 195 | }); 196 | 197 | test.asPromise('fun: array', (t, resolve) => { 198 | const output = [], 199 | c = chain([fromIterable([1, 2, 3]), [x => x * x, x => 2 * x + 1], streamToArray(output)]); 200 | 201 | c.on('end', () => { 202 | t.deepEqual(output, [3, 9, 19]); 203 | resolve(); 204 | }); 205 | }); 206 | 207 | test.asPromise('fun: embedded arrays', (t, resolve) => { 208 | const output = [], 209 | c = chain([fromIterable([1, 2, 3]), [x => x * x, [x => 2 * x + 1, []]], streamToArray(output)]); 210 | 211 | c.on('end', () => { 212 | t.deepEqual(output, [3, 9, 19]); 213 | resolve(); 214 | }); 215 | }); 216 | -------------------------------------------------------------------------------- /tests/test-gen.mjs: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import test from 'tape-six'; 4 | 5 | import {streamToArray, delay} from './helpers.mjs'; 6 | import chain, {none, finalValue, many, gen} from '../src/index.js'; 7 | import fromIterable from '../src/utils/readableFrom.js'; 8 | 9 | test.asPromise('gen: smoke test', (t, resolve) => { 10 | const output = [], 11 | c = chain([ 12 | fromIterable([1, 2, 3]), 13 | gen( 14 | x => x * x, 15 | x => 2 * x + 1 16 | ), 17 | streamToArray(output) 18 | ]); 19 | 20 | c.on('end', () => { 21 | t.deepEqual(output, [3, 9, 19]); 22 | resolve(); 23 | }); 24 | }); 25 | 26 | test.asPromise('gen: final', (t, resolve) => { 27 | const output = [], 28 | c = chain([ 29 | fromIterable([1, 2, 3]), 30 | gen( 31 | x => x * x, 32 | x => finalValue(x), 33 | x => 2 * x + 1 34 | ), 35 | streamToArray(output) 36 | ]); 37 | 38 | c.on('end', () => { 39 | t.deepEqual(output, [1, 4, 9]); 40 | resolve(); 41 | }); 42 | }); 43 | 44 | test.asPromise('gen: nothing', (t, resolve) => { 45 | const output = [], 46 | c = chain([ 47 | fromIterable([1, 2, 3]), 48 | gen( 49 | x => x * x, 50 | () => none, 51 | x => 2 * x + 1 52 | ), 53 | streamToArray(output) 54 | ]); 55 | 56 | c.on('end', () => { 57 | t.deepEqual(output, []); 58 | resolve(); 59 | }); 60 | }); 61 | 62 | test.asPromise('gen: empty', (t, resolve) => { 63 | const output = [], 64 | c = chain([fromIterable([1, 2, 3]), x => x * x, gen(), streamToArray(output)]); 65 | 66 | c.on('end', () => { 67 | t.deepEqual(output, [1, 4, 9]); 68 | resolve(); 69 | }); 70 | }); 71 | 72 | test.asPromise('gen: async', (t, resolve) => { 73 | const output = [], 74 | c = chain([ 75 | fromIterable([1, 2, 3]), 76 | gen( 77 | delay(x => x * x), 78 | x => 2 * x + 1 79 | ), 80 | streamToArray(output) 81 | ]); 82 | 83 | c.on('end', () => { 84 | t.deepEqual(output, [3, 9, 19]); 85 | resolve(); 86 | }); 87 | }); 88 | 89 | test.asPromise('gen: generator', (t, resolve) => { 90 | const output = [], 91 | c = chain([ 92 | fromIterable([1, 2, 3]), 93 | gen( 94 | x => x * x, 95 | function* (x) { 96 | yield x; 97 | yield x + 1; 98 | yield x + 2; 99 | }, 100 | x => 2 * x + 1 101 | ), 102 | streamToArray(output) 103 | ]); 104 | 105 | c.on('end', () => { 106 | t.deepEqual(output, [3, 5, 7, 9, 11, 13, 19, 21, 23]); 107 | resolve(); 108 | }); 109 | }); 110 | 111 | test.asPromise('gen: many', (t, resolve) => { 112 | const output = [], 113 | c = chain([ 114 | fromIterable([1, 2, 3]), 115 | gen( 116 | x => x * x, 117 | x => many([x, x + 1, x + 2]), 118 | x => 2 * x + 1 119 | ), 120 | streamToArray(output) 121 | ]); 122 | 123 | c.on('end', () => { 124 | t.deepEqual(output, [3, 5, 7, 9, 11, 13, 19, 21, 23]); 125 | resolve(); 126 | }); 127 | }); 128 | 129 | test.asPromise('gen: combined', (t, resolve) => { 130 | const output = [], 131 | c = chain([ 132 | fromIterable([1, 2]), 133 | gen( 134 | delay(x => -x), 135 | x => many([x, x * 10]), 136 | function* (x) { 137 | yield x; 138 | yield x - 1; 139 | }, 140 | x => -x 141 | ), 142 | streamToArray(output) 143 | ]); 144 | 145 | c.on('end', () => { 146 | t.deepEqual(output, [1, 2, 10, 11, 2, 3, 20, 21]); 147 | resolve(); 148 | }); 149 | }); 150 | 151 | test.asPromise('gen: combined final', (t, resolve) => { 152 | const output = [], 153 | c = chain([ 154 | fromIterable([1, 2]), 155 | gen( 156 | delay(x => -x), 157 | x => many([x, x * 10]), 158 | function* (x) { 159 | yield x; 160 | yield finalValue(x - 1); 161 | }, 162 | x => -x 163 | ), 164 | streamToArray(output) 165 | ]); 166 | 167 | c.on('end', () => { 168 | t.deepEqual(output, [1, -2, 10, -11, 2, -3, 20, -21]); 169 | resolve(); 170 | }); 171 | }); 172 | 173 | test.asPromise('gen: iterator', (t, resolve) => { 174 | const output = [], 175 | c = chain([ 176 | fromIterable([1, 2]), 177 | gen( 178 | delay(x => -x), 179 | x => many([x, x * 10]), 180 | function* (x) { 181 | yield x; 182 | yield finalValue(x - 1); 183 | }, 184 | x => -x 185 | ), 186 | streamToArray(output) 187 | ]); 188 | 189 | c.on('end', () => { 190 | t.deepEqual(output, [1, -2, 10, -11, 2, -3, 20, -21]); 191 | resolve(); 192 | }); 193 | }); 194 | 195 | test.asPromise('gen: async iterator', (t, resolve) => { 196 | const output = [], 197 | c = chain([ 198 | fromIterable([1, 2]), 199 | gen( 200 | delay(x => -x), 201 | x => many([x, x * 10]), 202 | async function* (x) { 203 | yield delay(x => x)(x); 204 | yield delay(x => finalValue(x - 1))(x); 205 | }, 206 | x => -x 207 | ), 208 | streamToArray(output) 209 | ]); 210 | 211 | c.on('end', () => { 212 | t.deepEqual(output, [1, -2, 10, -11, 2, -3, 20, -21]); 213 | resolve(); 214 | }); 215 | }); 216 | 217 | test.asPromise('gen: array', (t, resolve) => { 218 | const output = [], 219 | c = chain([fromIterable([1, 2, 3]), [x => x * x, x => 2 * x + 1], streamToArray(output)]); 220 | 221 | c.on('end', () => { 222 | t.deepEqual(output, [3, 9, 19]); 223 | resolve(); 224 | }); 225 | }); 226 | 227 | test.asPromise('gen: embedded arrays', (t, resolve) => { 228 | const output = [], 229 | c = chain([fromIterable([1, 2, 3]), [x => x * x, [x => 2 * x + 1, []]], streamToArray(output)]); 230 | 231 | c.on('end', () => { 232 | t.deepEqual(output, [3, 9, 19]); 233 | resolve(); 234 | }); 235 | }); 236 | -------------------------------------------------------------------------------- /src/index.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import {Duplex, DuplexOptions, Readable, Transform, Writable} from 'node:stream'; 4 | import {TypedDuplex, TypedReadable, TypedTransform, TypedWritable} from './typed-streams'; 5 | 6 | import { 7 | none, 8 | stop, 9 | Stop, 10 | finalSymbol, 11 | finalValue, 12 | final, 13 | isFinalValue, 14 | getFinalValue, 15 | manySymbol, 16 | many, 17 | isMany, 18 | getManyValues, 19 | getFunctionList, 20 | flushSymbol, 21 | flushable, 22 | isFlushable, 23 | fListSymbol, 24 | isFunctionList, 25 | setFunctionList, 26 | clearFunctionList, 27 | toMany, 28 | normalizeMany, 29 | combineMany, 30 | combineManyMut, 31 | type AsFlatList, 32 | type Fn, 33 | type OutputType 34 | } from './defs'; 35 | import gen from './gen'; 36 | import asStream from './asStream'; 37 | 38 | export = chain; 39 | 40 | /** 41 | * Represents a typed duplex stream as a pair of readable and writable streams. 42 | */ 43 | export type DuplexStream = { 44 | readable: ReadableStream; 45 | writable: WritableStream; 46 | }; 47 | 48 | /** 49 | * Options for the chain function, which is based on `DuplexOptions`. 50 | */ 51 | export interface ChainOptions extends DuplexOptions { 52 | /** If `true`, no groupings will be done. Each function will be a separate stream object. */ 53 | noGroupings?: boolean; 54 | /** If `true`, event bindings to the chain stream object will be skipped. */ 55 | skipEvents?: boolean; 56 | } 57 | 58 | /** 59 | * The tuple type for a chain function with one item. 60 | */ 61 | type ChainSteams1 = [Readable | Writable | Duplex | Transform]; 62 | /** 63 | * The tuple type for a chain function with multiple items. 64 | */ 65 | type ChainSteams = [ 66 | Readable | Duplex | Transform, 67 | ...(Duplex | Transform)[], 68 | Writable | Duplex | Transform 69 | ]; 70 | 71 | /** 72 | * Represents the output of the chain function. It is based on `Duplex` with extra properties. 73 | */ 74 | export interface ChainOutput extends Duplex { 75 | /** Internal list of streams. */ 76 | streams: ChainSteams1 | ChainSteams; 77 | /** The first stream, which can be used to feed the chain and to attach event handlers. */ 78 | input: Readable | Writable | Duplex | Transform; 79 | /** The last stream, which can be used to consume results and to attach event handlers. */ 80 | output: Readable | Writable | Duplex | Transform; 81 | } 82 | 83 | /** 84 | * Returns the first argument of a chain, a stream, or a function. 85 | */ 86 | export type Arg0 = F extends TypedTransform 87 | ? W 88 | : F extends TypedDuplex 89 | ? W 90 | : F extends TypedReadable 91 | ? never 92 | : F extends TypedWritable 93 | ? W 94 | : F extends Writable | Transform | Duplex 95 | ? any 96 | : F extends Readable 97 | ? never 98 | : F extends TransformStream 99 | ? W 100 | : F extends DuplexStream 101 | ? W 102 | : F extends WritableStream 103 | ? W 104 | : F extends ReadableStream 105 | ? never 106 | : F extends readonly unknown[] 107 | ? AsFlatList extends readonly [infer F1, ...(readonly unknown[])] 108 | ? Arg0 109 | : AsFlatList extends readonly [] 110 | ? any 111 | : AsFlatList extends readonly (infer F1)[] 112 | ? Arg0 113 | : never 114 | : F extends (...args: readonly any[]) => unknown 115 | ? Parameters[0] 116 | : never; 117 | 118 | /** 119 | * Returns the return type of a chain, a stream, or a function. 120 | */ 121 | export type Ret = F extends TypedTransform 122 | ? R 123 | : F extends TypedDuplex 124 | ? R 125 | : F extends TypedReadable 126 | ? R 127 | : F extends TypedWritable 128 | ? never 129 | : F extends Readable | Transform | Duplex 130 | ? any 131 | : F extends Writable 132 | ? never 133 | : F extends TransformStream 134 | ? R 135 | : F extends DuplexStream 136 | ? R 137 | : F extends ReadableStream 138 | ? R 139 | : F extends WritableStream 140 | ? never 141 | : F extends readonly unknown[] 142 | ? AsFlatList extends readonly [...unknown[], infer F1] 143 | ? Ret 144 | : AsFlatList extends readonly [] 145 | ? Default 146 | : AsFlatList extends readonly (infer F1)[] 147 | ? Ret 148 | : never 149 | : F extends Fn 150 | ? OutputType 151 | : never; 152 | 153 | /** 154 | * Represents an item in the chain function. 155 | * It is used to highlight mismatches between argument types and return types in a list. 156 | */ 157 | export type ChainItem = 158 | F extends TypedTransform 159 | ? I extends W 160 | ? F 161 | : TypedTransform 162 | : F extends TypedDuplex 163 | ? I extends W 164 | ? F 165 | : TypedDuplex 166 | : F extends TypedReadable 167 | ? [I] extends [never] 168 | ? F 169 | : never 170 | : F extends TypedWritable 171 | ? I extends W 172 | ? F 173 | : TypedWritable 174 | : F extends Writable | Transform | Duplex 175 | ? F 176 | : F extends Readable 177 | ? [I] extends [never] 178 | ? F 179 | : never 180 | : F extends TransformStream 181 | ? I extends W 182 | ? F 183 | : TransformStream 184 | : F extends DuplexStream 185 | ? I extends W 186 | ? F 187 | : DuplexStream 188 | : F extends ReadableStream 189 | ? [I] extends [never] 190 | ? F 191 | : never 192 | : F extends WritableStream 193 | ? I extends W 194 | ? F 195 | : WritableStream 196 | : F extends readonly [infer F1, ...infer R] 197 | ? F1 extends (null | undefined) 198 | ? readonly [F1, ...ChainList] 199 | : readonly [ChainItem, ...ChainList, R>] 200 | : F extends readonly unknown[] 201 | ? readonly [ChainItem] 202 | : F extends Fn 203 | ? I extends Arg0 204 | ? F 205 | : (arg: I, ...rest: readonly unknown[]) => ReturnType 206 | : never; 207 | 208 | /** 209 | * Replicates a tuple verifying the types of the list items so arguments match returns. 210 | * The replicated tuple is used to highlight mismatches between list items. 211 | */ 212 | export type ChainList = L extends readonly [infer F1, ...infer R] 213 | ? F1 extends (null | undefined) 214 | ? readonly [F1, ...ChainList] 215 | : readonly [ChainItem, ...ChainList, R>] 216 | : L; 217 | 218 | /** 219 | * Takes a function or an iterable and returns the underlying function. 220 | * @param fn function or iterable 221 | * @returns the underlying function 222 | * @remarks In the case of a function, it returns the argument. For iterables it returns the function associated with `Symbol.iterator` or `Symbol.asyncIterator`. 223 | */ 224 | declare function dataSource( 225 | fn: F 226 | ): F extends AsyncIterable 227 | ? () => AsyncIterator 228 | : F extends Iterable 229 | ? () => Iterator 230 | : F extends Fn 231 | ? F 232 | : never; 233 | 234 | /** 235 | * Creates a stream object out of a list of functions and streams. 236 | * @param fns array of functions, streams, or other arrays 237 | * @returns a duplex stream with additional properties 238 | * @remarks This is the main function of this library. 239 | */ 240 | declare function chain( 241 | ...fns: ChainList, L>, 242 | options?: ChainOptions 243 | ): ChainOutput, Ret>; 244 | 245 | declare function chainUnchecked( 246 | fns: readonly any[], 247 | options?: ChainOptions 248 | ): ChainOutput; 249 | 250 | declare namespace chain { 251 | export { 252 | none, 253 | stop, 254 | Stop, 255 | finalSymbol, 256 | finalValue, 257 | final, 258 | isFinalValue, 259 | getFinalValue, 260 | manySymbol, 261 | many, 262 | isMany, 263 | getManyValues, 264 | getFunctionList, 265 | flushSymbol, 266 | flushable, 267 | isFlushable, 268 | fListSymbol, 269 | isFunctionList, 270 | setFunctionList, 271 | clearFunctionList, 272 | toMany, 273 | normalizeMany, 274 | combineMany, 275 | combineManyMut, 276 | chain, 277 | chainUnchecked, 278 | gen, 279 | asStream, 280 | dataSource 281 | }; 282 | } 283 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | // @ts-self-types="./index.d.ts" 2 | 3 | 'use strict'; 4 | 5 | const {Readable, Writable, Duplex} = require('node:stream'); 6 | const defs = require('./defs'); 7 | const gen = require('./gen'); 8 | const asStream = require('./asStream'); 9 | 10 | // is*NodeStream functions taken from https://github.com/nodejs/node/blob/master/lib/internal/streams/utils.js 11 | const isReadableNodeStream = obj => 12 | obj && 13 | typeof obj.pipe === 'function' && 14 | typeof obj.on === 'function' && 15 | (!obj._writableState || 16 | (typeof obj._readableState === 'object' ? obj._readableState.readable : null) !== false) && // Duplex 17 | (!obj._writableState || obj._readableState); // Writable has .pipe. 18 | 19 | const isWritableNodeStream = obj => 20 | obj && 21 | typeof obj.write === 'function' && 22 | typeof obj.on === 'function' && 23 | (!obj._readableState || 24 | (typeof obj._writableState === 'object' ? obj._writableState.writable : null) !== false); // Duplex 25 | 26 | const isDuplexNodeStream = obj => 27 | obj && 28 | typeof obj.pipe === 'function' && 29 | obj._readableState && 30 | typeof obj.on === 'function' && 31 | typeof obj.write === 'function'; 32 | 33 | const isReadableWebStream = obj => 34 | obj && globalThis.ReadableStream && obj instanceof globalThis.ReadableStream; 35 | 36 | const isWritableWebStream = obj => 37 | obj && globalThis.WritableStream && obj instanceof globalThis.WritableStream; 38 | 39 | const isDuplexWebStream = obj => 40 | obj && 41 | globalThis.ReadableStream && 42 | obj.readable instanceof globalThis.ReadableStream && 43 | globalThis.WritableStream && 44 | obj.writable instanceof globalThis.WritableStream; 45 | 46 | const groupFunctions = (output, fn, index, fns) => { 47 | if ( 48 | isDuplexNodeStream(fn) || 49 | (!index && isReadableNodeStream(fn)) || 50 | (index === fns.length - 1 && isWritableNodeStream(fn)) 51 | ) { 52 | output.push(fn); 53 | return output; 54 | } 55 | if (isDuplexWebStream(fn)) { 56 | output.push(Duplex.fromWeb(fn, {objectMode: true})); 57 | return output; 58 | } 59 | if (!index && isReadableWebStream(fn)) { 60 | output.push(Readable.fromWeb(fn, {objectMode: true})); 61 | return output; 62 | } 63 | if (index === fns.length - 1 && isWritableWebStream(fn)) { 64 | output.push(Writable.fromWeb(fn, {objectMode: true})); 65 | return output; 66 | } 67 | if (typeof fn != 'function') 68 | throw TypeError('Item #' + index + ' is not a proper stream, nor a function.'); 69 | if (!output.length) output.push([]); 70 | const last = output[output.length - 1]; 71 | if (Array.isArray(last)) { 72 | last.push(fn); 73 | } else { 74 | output.push([fn]); 75 | } 76 | return output; 77 | }; 78 | 79 | const produceStreams = item => { 80 | if (Array.isArray(item)) { 81 | if (!item.length) return null; 82 | if (item.length == 1) return item[0] && chain.asStream(item[0]); 83 | return chain.asStream(chain.gen(...item)); 84 | } 85 | return item; 86 | }; 87 | 88 | const wrapFunctions = (fn, index, fns) => { 89 | if ( 90 | isDuplexNodeStream(fn) || 91 | (!index && isReadableNodeStream(fn)) || 92 | (index === fns.length - 1 && isWritableNodeStream(fn)) 93 | ) { 94 | return fn; // an acceptable stream 95 | } 96 | if (isDuplexWebStream(fn)) { 97 | return Duplex.fromWeb(fn, {objectMode: true}); 98 | } 99 | if (!index && isReadableWebStream(fn)) { 100 | return Readable.fromWeb(fn, {objectMode: true}); 101 | } 102 | if (index === fns.length - 1 && isWritableWebStream(fn)) { 103 | return Writable.fromWeb(fn, {objectMode: true}); 104 | } 105 | if (typeof fn == 'function') return chain.asStream(fn); // a function 106 | throw TypeError('Item #' + index + ' is not a proper stream, nor a function.'); 107 | }; 108 | 109 | // default implementation of required stream methods 110 | 111 | const write = (input, chunk, encoding, callback) => { 112 | let error = null; 113 | try { 114 | input.write(chunk, encoding, e => callback(e || error)); 115 | } catch (e) { 116 | error = e; 117 | } 118 | }; 119 | 120 | const final = (input, callback) => { 121 | let error = null; 122 | try { 123 | input.end(null, null, e => callback(e || error)); 124 | } catch (e) { 125 | error = e; 126 | } 127 | }; 128 | 129 | const read = output => { 130 | output.resume(); 131 | }; 132 | 133 | // the chain creator 134 | 135 | const chain = (fns, options) => { 136 | if (!Array.isArray(fns) || !fns.length) { 137 | throw TypeError("Chain's first argument should be a non-empty array."); 138 | } 139 | 140 | fns = fns.flat(Infinity).filter(fn => fn); 141 | 142 | const streams = ( 143 | options && options.noGrouping 144 | ? fns.map(wrapFunctions) 145 | : fns 146 | .map(fn => (defs.isFunctionList(fn) ? defs.getFunctionList(fn) : fn)) 147 | .flat(Infinity) 148 | .reduce(groupFunctions, []) 149 | .map(produceStreams) 150 | ).filter(s => s), 151 | input = streams[0], 152 | output = streams.reduce((output, item) => (output && output.pipe(item)) || item); 153 | 154 | let stream = null; // will be assigned later 155 | 156 | let writeMethod = (chunk, encoding, callback) => write(input, chunk, encoding, callback), 157 | finalMethod = callback => final(input, callback), 158 | readMethod = () => read(output); 159 | 160 | if (!isWritableNodeStream(input)) { 161 | writeMethod = (_1, _2, callback) => callback(null); 162 | finalMethod = callback => callback(null); 163 | input.on('end', () => stream.end()); 164 | } 165 | 166 | if (isReadableNodeStream(output)) { 167 | output.on('data', chunk => !stream.push(chunk) && output.pause()); 168 | output.on('end', () => stream.push(null)); 169 | } else { 170 | readMethod = () => {}; // nop 171 | output.on('finish', () => stream.push(null)); 172 | } 173 | 174 | stream = new Duplex( 175 | Object.assign({writableObjectMode: true, readableObjectMode: true}, options, { 176 | readable: isReadableNodeStream(output), 177 | writable: isWritableNodeStream(input), 178 | write: writeMethod, 179 | final: finalMethod, 180 | read: readMethod 181 | }) 182 | ); 183 | stream.streams = streams; 184 | stream.input = input; 185 | stream.output = output; 186 | 187 | if (!isReadableNodeStream(output)) { 188 | stream.resume(); 189 | } 190 | 191 | // connect events 192 | if (!options || !options.skipEvents) { 193 | streams.forEach(item => item.on('error', error => stream.emit('error', error))); 194 | } 195 | 196 | return stream; 197 | }; 198 | 199 | const dataSource = fn => { 200 | if (typeof fn == 'function') return fn; 201 | if (fn) { 202 | if (typeof fn[Symbol.asyncIterator] == 'function') return fn[Symbol.asyncIterator].bind(fn); 203 | if (typeof fn[Symbol.iterator] == 'function') return fn[Symbol.iterator].bind(fn); 204 | } 205 | throw new TypeError('The argument should be a function or an iterable object.'); 206 | }; 207 | 208 | module.exports = chain; 209 | 210 | // from defs.js 211 | module.exports.none = defs.none; 212 | module.exports.stop = defs.stop; 213 | module.exports.Stop = defs.Stop; 214 | 215 | module.exports.finalSymbol = defs.finalSymbol; 216 | module.exports.finalValue = defs.finalValue; 217 | module.exports.final = defs.final; 218 | module.exports.isFinalValue = defs.isFinalValue; 219 | module.exports.getFinalValue = defs.getFinalValue; 220 | 221 | module.exports.manySymbol = defs.manySymbol; 222 | module.exports.many = defs.many; 223 | module.exports.isMany = defs.isMany; 224 | module.exports.getManyValues = defs.getManyValues; 225 | module.exports.getFunctionList = defs.getFunctionList; 226 | 227 | module.exports.flushSymbol = defs.flushSymbol; 228 | module.exports.flushable = defs.flushable; 229 | module.exports.isFlushable = defs.isFlushable; 230 | 231 | module.exports.fListSymbol = defs.fListSymbol; 232 | module.exports.isFunctionList = defs.isFunctionList; 233 | module.exports.getFunctionList = defs.getFunctionList; 234 | module.exports.setFunctionList = defs.setFunctionList; 235 | module.exports.clearFunctionList = defs.clearFunctionList; 236 | 237 | module.exports.toMany = defs.toMany; 238 | module.exports.normalizeMany = defs.normalizeMany; 239 | module.exports.combineMany = defs.combineMany; 240 | module.exports.combineManyMut = defs.combineManyMut; 241 | 242 | module.exports.chain = chain; // for compatibility with 2.x 243 | module.exports.chainUnchecked = chain; // for TypeScript to bypass type checks 244 | module.exports.gen = gen; 245 | module.exports.asStream = asStream; 246 | 247 | module.exports.dataSource = dataSource; 248 | -------------------------------------------------------------------------------- /src/defs.d.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Special symbol that indicates that a function produced no value. 3 | * Effectively, the rest of the pipeline is not used. 4 | */ 5 | export declare const none: unique symbol; 6 | 7 | /** 8 | * Special symbol that indicates that the pipeline should be stopped. 9 | * Just like {@link none}, it produces no value. 10 | */ 11 | export declare const stop: unique symbol; 12 | 13 | /** Used internally to mark a value as a final value. */ 14 | export declare const finalSymbol: unique symbol; 15 | /** Used internally to mark a value as multiple values. */ 16 | export declare const manySymbol: unique symbol; 17 | /** Used internally to mark a function as capable of being flushed. */ 18 | export declare const flushSymbol: unique symbol; 19 | /** Used internally to mark a function as being derived from a function list. */ 20 | export declare const fListSymbol: unique symbol; 21 | 22 | /** 23 | * An exception that indicates that the pipeline should be stopped. 24 | */ 25 | export declare class Stop extends Error {} 26 | 27 | /** 28 | * Interface for a value that has been marked as a final value. 29 | */ 30 | export interface FinalValue { 31 | [finalSymbol]: 1; 32 | value: T; 33 | } 34 | /** 35 | * Type predicate for `FinalValue`. 36 | * @param o object to test 37 | * @returns `true` if `o` is a `FinalValue` 38 | */ 39 | export declare function isFinalValue(o: object): o is FinalValue; 40 | /** 41 | * Creates a `FinalValue` 42 | * @param value the wrapped value 43 | * @returns a `FinalValue` 44 | */ 45 | export declare function finalValue(value: T): FinalValue; 46 | /** 47 | * Retrieves the value of a `FinalValue` 48 | * @param o a `FinalValue` object 49 | * @returns the wrapped value 50 | */ 51 | export declare function getFinalValue(o: FinalValue): T; 52 | /** 53 | * Alias for {@link finalValue} 54 | */ 55 | export declare const final = finalValue; 56 | 57 | /** 58 | * Interface for a value that has been marked as multiple values. 59 | * It is used to return multiple values from a regular (non-generator) function. 60 | */ 61 | export interface Many { 62 | [manySymbol]: 1; 63 | values: T[]; 64 | } 65 | /** 66 | * Type predicate for `Many`. 67 | * @param o object to test 68 | * @returns `true` if `o` is a `Many` 69 | */ 70 | export declare function isMany(o: unknown): o is Many; 71 | /** 72 | * Creates a `Many` 73 | * @param values the wrapped values 74 | * @returns a `Many` 75 | */ 76 | export declare function many(values: T[]): Many; 77 | /** 78 | * Retrieves the values of a `Many` 79 | * @param o a `Many` object 80 | * @returns the wrapped values 81 | */ 82 | export declare function getManyValues(o: Many): T[]; 83 | 84 | /** 85 | * Interface for a function that can be flushed. 86 | * If it is marked as flushable, it will be called with the special {@link none} value 87 | * when the pipeline is stopped so it can produce the last value. 88 | */ 89 | export interface Flushable { 90 | (value: I, ...rest: any[]): O; 91 | [flushSymbol]: 1; 92 | } 93 | /** 94 | * Type predicate for `Flushable`. 95 | * @param o function to test 96 | * @returns `true` if `o` is a `Flushable` 97 | */ 98 | export declare function isFlushable(o: (value: I, ...rest: any[]) => O): o is Flushable; 99 | /** 100 | * Creates a `Flushable` 101 | * @param write function to be marked as flushable 102 | * @param final an optional function to be called when the pipeline is stopped 103 | * @returns a `Flushable` 104 | * @remarks If `final` is not provided, `write` will be called with {@link none} when the pipeline is stopped 105 | */ 106 | export declare function flushable( 107 | write: (value: I, ...rest: any[]) => O, 108 | final?: () => O 109 | ): Flushable; 110 | 111 | /** 112 | * Interface for a function that can be derived from a function list. 113 | * `chain` can use the list instead of the original function. 114 | */ 115 | export interface FunctionList< 116 | T extends (...args: readonly any[]) => unknown, 117 | I = any, 118 | O = unknown 119 | > { 120 | (value: I, ...rest: any[]): O; 121 | [fListSymbol]: 1; 122 | fList: T[]; 123 | } 124 | /** 125 | * Type predicate for `FunctionList`. 126 | * @param o function to test 127 | * @returns `true` if `o` is a `FunctionList` 128 | */ 129 | export declare function isFunctionList( 130 | o: (value: I, ...rest: readonly any[]) => O 131 | ): o is FunctionList<(...args: readonly any[]) => unknown, I, O>; 132 | /** 133 | * Sets a function list creating a `FunctionList` structure. 134 | * @param o function to be marked as a function list 135 | * @param fns function list 136 | * @returns `o` as a `FunctionList` 137 | */ 138 | export declare function setFunctionList< 139 | T extends (...args: readonly any[]) => unknown, 140 | F extends (...args: readonly any[]) => unknown 141 | >( 142 | o: F, 143 | fns: T[] 144 | ): F extends (value: infer I, ...rest: any[]) => infer O ? FunctionList : never; 145 | /** 146 | * Retrieves the function list of a `FunctionList` 147 | * @param o a `FunctionList` object 148 | * @returns the function list 149 | */ 150 | export declare function getFunctionList< 151 | T extends (...args: readonly any[]) => unknown, 152 | I = any, 153 | O = unknown 154 | >(o: FunctionList): T[]; 155 | /** 156 | * Clears the function list from a `FunctionList` 157 | * @param o a `FunctionList` object 158 | * @returns `o` as a `FunctionList` 159 | */ 160 | export declare function clearFunctionList< 161 | T extends (...args: readonly any[]) => unknown, 162 | I = any, 163 | O = unknown 164 | >(o: FunctionList): (value: I, ...rest: any[]) => O; 165 | 166 | /** 167 | * Convert a value to `Many`. 168 | * @param value the value to convert 169 | * @returns a `Many` containing the value 170 | * @remarks `Many` is used to return multiple values from a regular (non-generator) function. 171 | */ 172 | export declare function toMany(value: readonly Many): Many; 173 | export declare function toMany(value: typeof none): Many; 174 | export declare function toMany(value: readonly T): Many; 175 | /** 176 | * Normalize a value by unbundling it if it is a `Many`. 177 | * @param value the value to normalize 178 | * @returns the normalized value 179 | */ 180 | export declare function normalizeMany(value: readonly unknown): unknown; 181 | /** 182 | * Combine two values into a `Many`. 183 | * @param a the first value 184 | * @param b the second value 185 | * @returns a `Many` containing both values 186 | */ 187 | export declare function combineMany(a: typeof none, b: typeof none): Many; 188 | export declare function combineMany(a: typeof none, b: readonly Many): Many; 189 | export declare function combineMany(a: readonly Many, b: typeof none): Many; 190 | export declare function combineMany(a: typeof none, b: readonly T): Many; 191 | export declare function combineMany(a: readonly T, b: typeof none): Many; 192 | export declare function combineMany(a: readonly Many, b: readonly Many): Many; 193 | export declare function combineMany(a: readonly T, b: readonly Many): Many; 194 | export declare function combineMany(a: readonly Many, b: readonly U): Many; 195 | export declare function combineMany(a: readonly T, b: readonly U): Many; 196 | /** 197 | * Combine two values into a `Many` mutably. 198 | * @param a the first value 199 | * @param b the second value 200 | * @returns a `Many` containing both values 201 | * @remarks if `a` or `b` are `Many`, they can be modified in-place 202 | */ 203 | export declare function combineManyMut(a: typeof none, b: typeof none): Many; 204 | export declare function combineManyMut(a: typeof none, b: readonly Many): Many; 205 | export declare function combineManyMut(a: readonly Many, b: typeof none): Many; 206 | export declare function combineManyMut(a: typeof none, b: readonly T): Many; 207 | export declare function combineManyMut(a: readonly T, b: typeof none): Many; 208 | export declare function combineManyMut(a: readonly Many, b: readonly Many): Many; 209 | export declare function combineManyMut(a: readonly T, b: readonly Many): Many; 210 | export declare function combineManyMut(a: readonly Many, b: readonly U): Many; 211 | export declare function combineManyMut(a: readonly T, b: readonly U): Many; 212 | 213 | // generic utilities: unpacking types 214 | 215 | /** 216 | * Generic utility for getting the return type of a function including async and generators. 217 | */ 218 | export type UnpackReturnType unknown> = 219 | ReturnType extends Promise 220 | ? Awaited> 221 | : ReturnType extends AsyncGenerator 222 | ? O 223 | : ReturnType extends Generator 224 | ? O 225 | : ReturnType; 226 | 227 | /** 228 | * `stream-chain`-specific utility for getting the type from functions used in a function list. 229 | */ 230 | export type UnpackType = T extends Many 231 | ? U 232 | : T extends FinalValue 233 | ? U 234 | : Exclude; 235 | 236 | /** 237 | * Unpacking the return type of a function as a combination of {@link UnpackType} and {@link UnpackReturnType}. 238 | */ 239 | export type OutputType = UnpackType>; 240 | 241 | // generic utilities: working with tuples 242 | 243 | /** 244 | * Returns the first element of a tuple or `never`. 245 | */ 246 | export type First = L extends readonly [ 247 | infer T, 248 | ...(readonly unknown[]) 249 | ] 250 | ? T 251 | : never; 252 | /** 253 | * Returns the last element of a tuple or `never`. 254 | */ 255 | export type Last = L extends readonly [ 256 | ...(readonly unknown[]), 257 | infer T 258 | ] 259 | ? T 260 | : never; 261 | /** 262 | * Flattens a tuple of tuples recursively returning a flat tuple. 263 | */ 264 | export type Flatten = L extends readonly [infer T, ...infer R] 265 | ? T extends readonly unknown[] 266 | ? readonly [...Flatten, ...Flatten] 267 | : readonly [T, ...Flatten] 268 | : L; 269 | /** 270 | * Filters a tuple removing all elements of a specified type returning a filtered tuple. 271 | */ 272 | export type Filter = L extends readonly [infer T, ...infer R] 273 | ? T extends X 274 | ? Filter 275 | : readonly [T, ...Filter] 276 | : L; 277 | /** 278 | * Flattens and filters a tuple. See {@link Flatten} and {@link Filter}. 279 | */ 280 | export type AsFlatList = Filter, null | undefined>; 281 | 282 | // generic utilities: working with functions 283 | 284 | /** 285 | * A generic one-argument function. Used internally. 286 | */ 287 | export type Fn = (arg: any, ...args: readonly unknown[]) => unknown; 288 | 289 | /** 290 | * Returns the first argument of a function or a function list or `never`. 291 | */ 292 | export type Arg0 = F extends readonly unknown[] 293 | ? AsFlatList extends readonly [infer F1, ...(readonly unknown[])] 294 | ? Arg0 295 | : AsFlatList extends readonly [] 296 | ? any 297 | : AsFlatList extends readonly (infer F1)[] 298 | ? Arg0 299 | : never 300 | : F extends (...args: readonly any[]) => unknown 301 | ? Parameters[0] 302 | : never; 303 | 304 | /** 305 | * Returns the unpacked return type of a function or a function list or `never`. 306 | */ 307 | export type Ret = F extends readonly unknown[] 308 | ? AsFlatList extends readonly [...unknown[], infer F1] 309 | ? Ret 310 | : AsFlatList extends readonly [] 311 | ? Default 312 | : AsFlatList extends readonly (infer F1)[] 313 | ? Ret 314 | : never 315 | : F extends Fn 316 | ? OutputType 317 | : never; 318 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "include": ["src/**/*", "ts-check/**/*"], 3 | "compilerOptions": { 4 | /* Visit https://aka.ms/tsconfig to read more about this file */ 5 | 6 | /* Projects */ 7 | // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ 8 | // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ 9 | // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ 10 | // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ 11 | // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ 12 | // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ 13 | 14 | /* Language and Environment */ 15 | "target": "ES2022", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ 16 | // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ 17 | // "jsx": "preserve", /* Specify what JSX code is generated. */ 18 | // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ 19 | // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ 20 | // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ 21 | // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ 22 | // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ 23 | // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ 24 | // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ 25 | // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ 26 | // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ 27 | 28 | /* Modules */ 29 | "module": "Node16", /* Specify what module code is generated. */ 30 | // "rootDir": "./", /* Specify the root folder within your source files. */ 31 | "moduleResolution": "Node16", /* Specify how TypeScript looks up a file from a given module specifier. */ 32 | // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ 33 | // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ 34 | // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ 35 | // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ 36 | // "types": [], /* Specify type package names to be included without being referenced in a source file. */ 37 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 38 | // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ 39 | // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ 40 | // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ 41 | // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ 42 | // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ 43 | // "resolveJsonModule": true, /* Enable importing .json files. */ 44 | // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ 45 | // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ 46 | 47 | /* JavaScript Support */ 48 | // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ 49 | // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ 50 | // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ 51 | 52 | /* Emit */ 53 | // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ 54 | // "declarationMap": true, /* Create sourcemaps for d.ts files. */ 55 | // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ 56 | // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ 57 | // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ 58 | // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ 59 | // "outDir": "./", /* Specify an output folder for all emitted files. */ 60 | // "removeComments": true, /* Disable emitting comments. */ 61 | "noEmit": true, /* Disable emitting files from a compilation. */ 62 | // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ 63 | // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ 64 | // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ 65 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 66 | // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ 67 | // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ 68 | // "newLine": "crlf", /* Set the newline character for emitting files. */ 69 | // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ 70 | // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ 71 | // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ 72 | // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ 73 | // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ 74 | 75 | /* Interop Constraints */ 76 | // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ 77 | // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ 78 | // "isolatedDeclarations": true, /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */ 79 | // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ 80 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ 81 | // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ 82 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 83 | 84 | /* Type Checking */ 85 | "strict": true, /* Enable all strict type-checking options. */ 86 | // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ 87 | // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ 88 | // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ 89 | // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ 90 | // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ 91 | // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ 92 | // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ 93 | // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ 94 | // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ 95 | // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ 96 | // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ 97 | // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ 98 | // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ 99 | // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ 100 | // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ 101 | // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ 102 | // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ 103 | // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ 104 | 105 | /* Completeness */ 106 | // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ 107 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # stream-chain [![NPM version][npm-img]][npm-url] 2 | 3 | [npm-img]: https://img.shields.io/npm/v/stream-chain.svg 4 | [npm-url]: https://npmjs.org/package/stream-chain 5 | 6 | `stream-chain` creates a chain of streams out of regular functions, asynchronous functions, generator functions, existing Node streams, and Web streams, while properly handling [backpressure](https://nodejs.org/en/learn/modules/backpressuring-in-streams). The resulting chain is represented as a [Duplex](https://nodejs.org/api/stream.html#stream_class_stream_duplex) stream, which can be combined with other streams the usual way. It eliminates a boilerplate helping to concentrate on functionality without losing the performance especially make it easy to build object mode data processing pipelines. 7 | 8 | Originally `stream-chain` was used internally with [stream-fork](https://www.npmjs.com/package/stream-fork) and [stream-json](https://www.npmjs.com/package/stream-json) to create flexible data processing pipelines. 9 | 10 | `stream-chain` is a lightweight, no-dependencies micro-package with TS typings. It is distributed under New BSD license. 11 | 12 | ## Intro 13 | 14 | ```js 15 | import chain from 'stream-chain'; 16 | // or: const chain = require('stream-chain'); 17 | 18 | import fs from 'node:fs'; 19 | import zlib from 'node:zlib'; 20 | import {Transform} from 'node:stream'; 21 | 22 | // this chain object will work on a stream of numbers 23 | const pipeline = chain([ 24 | // transforms a value 25 | x => x * x, 26 | 27 | // returns several values 28 | x => chain.many([x - 1, x, x + 1]), 29 | 30 | // waits for an asynchronous operation 31 | async x => await getTotalFromDatabaseByKey(x), 32 | 33 | // returns multiple values with a generator 34 | function* (x) { 35 | for (let i = x; i > 0; --i) { 36 | yield i; 37 | } 38 | return 0; 39 | }, 40 | 41 | // filters out even values 42 | x => x % 2 ? x : null, 43 | 44 | // uses an arbitrary transform stream 45 | new Transform({ 46 | objectMode: true, 47 | transform(x, _, callback) { 48 | callback(null, x + 1); 49 | } 50 | }), 51 | 52 | // transform to strings 53 | x => '' + x, 54 | 55 | // compress 56 | zlib.createGzip() 57 | ]); 58 | 59 | // the chain object is a regular stream 60 | // it can be used with normal stream methods 61 | 62 | // log errors 63 | pipeline.on('error', error => console.log(error)); 64 | 65 | // use the chain object, and save the result to a file 66 | dataSource.pipe(pipeline).pipe(fs.createWriteStream('output.txt.gz')); 67 | ``` 68 | 69 | Making processing pipelines appears to be easy: just chain functions one after another, and we are done. Real life pipelines filter objects out and/or produce more objects out of a few ones. On top of that we have to deal with asynchronous operations, while processing or producing data: networking, databases, files, user responses, and so on. Unequal number of values per stage, and unequal throughput of stages introduced problems like [backpressure](https://nodejs.org/en/learn/modules/backpressuring-in-streams), which requires algorithms implemented by [streams](https://nodejs.org/api/stream.html). 70 | 71 | While a lot of API improvements were made to make streams easy to use, in reality, a lot of boilerplate is required when creating a pipeline. `stream-chain` eliminates most of it. 72 | 73 | ## Installation 74 | 75 | ```bash 76 | npm i --save stream-chain 77 | # or: yarn add stream-chain 78 | ``` 79 | 80 | ## Documentation 81 | 82 | All documentation can be found in the [wiki](https://github.com/uhop/stream-chain/wiki). It document in details the main function and various utilities and helpers that can simplify stream programming. Additionally it includes a support for JSONL (line-separated JSON files). 83 | 84 | An object that is returned by `chain()` is based on [Duplex](https://nodejs.org/api/stream.html#stream_class_stream_duplex). It chains its dependents in a single pipeline optionally binding `error` events. 85 | 86 | Many details about this package can be discovered by looking at test files located in `tests/` and in the source code (`src/`). 87 | 88 | ### `chain(fns[, options])` 89 | 90 | The factory function accepts the following arguments: 91 | 92 | * `fns` is an array of functions, arrays or stream instances. 93 | * If a value is a function, it is a candidate for a [Transform](https://nodejs.org/api/stream.html#stream_class_stream_transform) stream (see below for more details), which calls this function with two parameters: `chunk` (an object), and an optional `encoding`. See [Node's documentation](https://nodejs.org/api/stream.html#stream_transform_transform_chunk_encoding_callback) for more details on those parameters. 94 | * If it is a regular function, it can return: 95 | * Regular value: 96 | * If it is `undefined` or `null`, no value shall be passed. 97 | * Otherwise, the value will be passed to the next stream. 98 | 99 | ```js 100 | // produces no values: 101 | x => null 102 | x => undefined 103 | // produces one value: 104 | x => x 105 | ``` 106 | 107 | * Special value: 108 | * If it is an instance of [Promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise) or "thenable" (an object with a method called `then()`), it will be waited for. Its result should be a regular value. 109 | 110 | ```js 111 | // delays by 0.5s: 112 | x => new Promise( 113 | resolve => setTimeout(() => resolve(x), 500)) 114 | ``` 115 | 116 | * If it is an instance of a generator or "nextable" (an object with a method called `next()`), it will be iterated according to the [generator](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Generator) protocol. The results should be regular values. 117 | 118 | ```js 119 | // produces multiple values: 120 | class Nextable { 121 | constructor(x) { 122 | this.x = x; 123 | this.i = -1; 124 | } 125 | next() { 126 | return { 127 | done: this.i <= 1, 128 | value: this.x + this.i++ 129 | }; 130 | } 131 | } 132 | x => new Nextable(x) 133 | ``` 134 | 135 | `next()` can return a `Promise` according to the [asynchronous generator](https://zaiste.net/nodejs_10_asynchronous_iteration_async_generators/) protocol. 136 | * Any thrown exception will be caught and passed to a callback function effectively generating an error event. 137 | 138 | ```js 139 | // fails 140 | x => { throw new Error('Bad!'); } 141 | ``` 142 | 143 | * If it is an asynchronous function, it can return a regular value. 144 | * In essence, it is covered under "special values" as a function that returns a promise. 145 | 146 | ```js 147 | // delays by 0.5s: 148 | async x => { 149 | await new Promise(resolve => setTimeout(() => resolve(), 500)); 150 | return x; 151 | } 152 | ``` 153 | 154 | * If it is a generator function, each yield should produce a regular value. 155 | * In essence, it is covered under "special values" as a function that returns a generator object. 156 | 157 | ```js 158 | // produces multiple values: 159 | function* (x) { 160 | for (let i = -1; i <= 1; ++i) { 161 | if (i) yield x + i; 162 | } 163 | return x; 164 | } 165 | ``` 166 | 167 | * If it is an asynchronous generator function, each yield should produce a regular value. 168 | * In essence, it is covered under "special values" as a function that returns a generator object. 169 | 170 | ```js 171 | // produces multiple values: 172 | async function* (x) { 173 | for (let i = -1; i <= 1; ++i) { 174 | if (i) { 175 | await new Promise(resolve => setTimeout(() => resolve(), 50)); 176 | yield x + i; 177 | } 178 | } 179 | return x; 180 | } 181 | ``` 182 | 183 | * If a value is an array, its items are assumed to be functions, streams or other such arrays. The array is flattened, all individual items are included in a chain sequentially. 184 | * It is a provision to create lightweight bundles from pipeline items. 185 | * If a value is a valid stream, it is included as is in the pipeline. 186 | * [Transform](https://nodejs.org/api/stream.html#stream_class_stream_transform). 187 | * [Duplex](https://nodejs.org/api/stream.html#stream_class_stream_duplex). 188 | * The very first stream can be [Readable](https://nodejs.org/api/stream.html#stream_class_stream_readable). 189 | * In this case the pipeline ignores all possible writes to the front, and ends when the first stream ends. 190 | * The very last stream can be [Writable](https://nodejs.org/api/stream.html#stream_class_stream_writable). 191 | * In this case the pipeline does not produce any output, and finishes when the last stream finishes. 192 | * Because `'data'` event is not used in this case, the instance resumes itself automatically. Read about it in Node's documentation: 193 | * [Two reading modes](https://nodejs.org/api/stream.html#two-reading-modes). 194 | * [Three states](https://nodejs.org/api/stream.html#three-states). 195 | * [readable.resume()](https://nodejs.org/api/stream.html#stream_readable_resume). 196 | * *(Since 3.1.0)* If a value is a web stream object (like `ReadableStream` or `WritableStream`), it is adapted to a corresponding Node stream and included in the pipeline. 197 | * Note that the support of web streams is still experimental in Node. 198 | * `options` is an optional object detailed in the [Node's documentation](https://nodejs.org/api/stream.html#stream_new_stream_duplex_options). 199 | * The default options is this object: 200 | 201 | ```js 202 | {writableObjectMode: true, readableObjectMode: true} 203 | ``` 204 | 205 | If `options` is specified it is copied over the default options. 206 | * Always make sure that `writableObjectMode` is the same as the corresponding object mode of the first stream, and `readableObjectMode` is the same as the corresponding object mode of the last stream. 207 | * Eventually both these modes can be deduced, but Node does not define the standard way to determine it, so currently it cannot be done reliably. 208 | * Additionally the following custom properties are recognized: 209 | * `skipEvents` is an optional boolean flag. If it is falsy (the default), `'error'` events from all streams are forwarded to the created instance. If it is truthy, no event forwarding is made. A user can always do so externally or in a constructor of derived classes. 210 | * `noGrouping` is an optional boolean flag. If it is falsy (the default), all subsequent functions are grouped together using the `gen()` utility for improved performance. If it is specified and truthy, all functions will be wrapped as streams individually. This mode is compatible with how the 2.x version works. 211 | 212 | An instance can be used to attach handlers for stream events. 213 | 214 | ```js 215 | const pipeline = chain([x => x * x, x => [x - 1, x, x + 1]]); 216 | pipeline.on('error', error => console.error(error)); 217 | dataSource.pipe(pipeline); 218 | ``` 219 | 220 | ## License 221 | 222 | BSD-3-Clause 223 | 224 | ## Release History 225 | 226 | * 3.4.0 *Added `Many`-related helpers and `chainUnchecked()` for TS.* 227 | * 3.3.2 *Technical release: updated deps, more tests.* 228 | * 3.3.1 *Minor enhancement: more flexible split on lines.* 229 | * 3.3.0 *Added a way to ignore JSON parsing errors silently.* 230 | * 3.2.0 *Added TS typings and `clearFunctionList()`.* 231 | * 3.1.0 *Added a seamless support for web streams.* 232 | * 3.0.1 *First release of 3.0. See [wiki](https://github.com/uhop/stream-chain/wiki) for details.* 233 | * 3.0.0 *New major version. Unreleased.* 234 | * 2.2.5 *Relaxed the definition of a stream (thx [Rich Hodgkins](https://github.com/rhodgkins)).* 235 | * 2.2.4 *Bugfix: wrong `const`-ness in the async generator branch (thx [Patrick Pang](https://github.com/patrickpang)).* 236 | * 2.2.3 *Technical release. No need to upgrade.* 237 | * 2.2.2 *Technical release. No need to upgrade.* 238 | * 2.2.1 *Technical release: new symbols namespace, explicit license (thx [Keen Yee Liau](https://github.com/kyliau)), added Greenkeeper.* 239 | * 2.2.0 *Added utilities: `take`, `takeWhile`, `skip`, `skipWhile`, `fold`, `scan`, `Reduce`, `comp`.* 240 | * 2.1.0 *Added simple transducers, dropped Node 6.* 241 | * 2.0.3 *Added TypeScript typings and the badge.* 242 | * 2.0.2 *Workaround for Node 6: use `'finish'` event instead of `_final()`.* 243 | * 2.0.1 *Improved documentation.* 244 | * 2.0.0 *Upgraded to use Duplex instead of EventEmitter as the base.* 245 | * 1.0.3 *Improved documentation.* 246 | * 1.0.2 *Better README.* 247 | * 1.0.1 *Fixed the README.* 248 | * 1.0.0 *The initial release.* 249 | --------------------------------------------------------------------------------