├── .gitignore ├── media └── logo.png ├── .prettierrc.mjs ├── .changeset ├── config.json └── README.md ├── src ├── index.ts ├── handler.ts ├── atom.ts ├── stream │ ├── index.ts │ ├── consumption.ts │ ├── base.ts │ ├── higher-order.ts │ └── transforms.ts └── util.ts ├── .github └── workflows │ ├── test-prs.yml │ └── release.yml ├── tsconfig.json ├── eslint.config.js ├── package.json ├── test ├── errors.test.ts ├── benchmarks │ └── index.bench.ts ├── creation.test.ts ├── higher-order.test.ts ├── consumption.test.ts └── transforms.test.ts ├── CHANGELOG.md └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | docs 4 | -------------------------------------------------------------------------------- /media/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clear/windpipe/main/media/logo.png -------------------------------------------------------------------------------- /.prettierrc.mjs: -------------------------------------------------------------------------------- 1 | export default { 2 | useTabs: false, 3 | singleQuote: false, 4 | trailingComma: "all", 5 | printWidth: 100, 6 | semi: true, 7 | tabWidth: 4, 8 | }; 9 | -------------------------------------------------------------------------------- /.changeset/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://unpkg.com/@changesets/config@3.0.0/schema.json", 3 | "changelog": "@changesets/cli/changelog", 4 | "commit": false, 5 | "fixed": [], 6 | "linked": [], 7 | "access": "restricted", 8 | "baseBranch": "main", 9 | "updateInternalDependencies": "patch", 10 | "ignore": [] 11 | } 12 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { Stream } from "./stream"; 2 | 3 | // Export all useful types for atoms 4 | export type { 5 | Atom, 6 | AtomOk, 7 | AtomError, 8 | AtomException, 9 | AtomUnknown, 10 | VALUE, 11 | ERROR, 12 | EXCEPTION, 13 | UNKNOWN, 14 | MaybeAtom, 15 | } from "./atom"; 16 | 17 | // Re-export useful utility types 18 | export type { MaybePromise, Truthy, CallbackOrStream, NodeCallback } from "./util"; 19 | 20 | export default Stream; 21 | -------------------------------------------------------------------------------- /.github/workflows/test-prs.yml: -------------------------------------------------------------------------------- 1 | name: Test Pull Requests 2 | 3 | on: 4 | pull_request: 5 | branches: [ "main" ] 6 | push: 7 | branches: 8 | - main 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v3 16 | - name: Use Node.js 20.x 17 | uses: actions/setup-node@v3 18 | with: 19 | node-version: 20.x 20 | cache: 'npm' 21 | - run: npm ci 22 | - run: npm run build 23 | - run: npm test 24 | 25 | -------------------------------------------------------------------------------- /.changeset/README.md: -------------------------------------------------------------------------------- 1 | # Changesets 2 | 3 | Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works 4 | with multi-package repos, or single-package repos to help you version and publish your code. You can 5 | find the full documentation for it [in our repository](https://github.com/changesets/changesets) 6 | 7 | We have a quick list of common questions to get you started engaging with this project in 8 | [our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md) 9 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Base Options: */ 4 | "esModuleInterop": true, 5 | "skipLibCheck": true, 6 | "target": "es2022", 7 | "verbatimModuleSyntax": true, 8 | "allowJs": true, 9 | "resolveJsonModule": true, 10 | "moduleDetection": "force", 11 | /* Strictness */ 12 | "strict": true, 13 | "noUncheckedIndexedAccess": true, 14 | /* If NOT transpiling with TypeScript: */ 15 | "moduleResolution": "Bundler", 16 | "module": "ESNext", 17 | "noEmit": true, 18 | /* If your code doesn't run in the DOM: */ 19 | "lib": [ 20 | "es2022" 21 | ] 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | import globals from "globals"; 2 | import pluginJs from "@eslint/js"; 3 | import tseslint from "typescript-eslint"; 4 | import eslintPluginPrettier from "eslint-plugin-prettier/recommended"; 5 | 6 | export default [ 7 | { 8 | ignores: ["dist/*", "docs/*", ".github/*", ".changeset/*"], 9 | }, 10 | 11 | // Use node globals 12 | { languageOptions: { globals: globals.node } }, 13 | 14 | // Use recommended JS config 15 | pluginJs.configs.recommended, 16 | 17 | // Use recommended TS config 18 | ...tseslint.configs.recommended, 19 | 20 | // Allow unused variables that start with _ 21 | { 22 | rules: { 23 | "no-unused-vars": "off", 24 | "@typescript-eslint/no-unused-vars": [ 25 | "warn", 26 | { 27 | argsIgnorePattern: "^_", 28 | varsIgnorePattern: "^_", 29 | caughtErrorsIgnorePattern: "^_", 30 | }, 31 | ], 32 | }, 33 | }, 34 | 35 | // Delegate to prettier as last plugin 36 | eslintPluginPrettier, 37 | ]; 38 | -------------------------------------------------------------------------------- /src/handler.ts: -------------------------------------------------------------------------------- 1 | import { 2 | normalise, 3 | ok, 4 | exception, 5 | isException, 6 | type Atom, 7 | type AtomOk, 8 | type AtomException, 9 | type MaybeAtom, 10 | } from "./atom"; 11 | import type { MaybePromise } from "./util"; 12 | 13 | /** 14 | * Given some value, will either await it if it's a promise, or return the value un-modified. This 15 | * is an async function so that the result can be `await`ed regardless of whether the value is 16 | * actually a promise or not. 17 | */ 18 | async function normalisePromise(value: MaybePromise): Promise { 19 | if (value instanceof Promise) { 20 | return await value; 21 | } else { 22 | return value; 23 | } 24 | } 25 | 26 | /** 27 | * Run the given handler, then the returned value will be normalised as an atom and returned. If an 28 | * unhandled error is thrown during the handler, then it will be caught and returned as an `exception` 29 | * atom. 30 | */ 31 | export async function handler( 32 | handler: () => MaybePromise>, 33 | trace: string[], 34 | ): Promise> { 35 | const result = await run(handler, trace); 36 | 37 | if (isException(result)) { 38 | return result; 39 | } 40 | 41 | return normalise(result.value); 42 | } 43 | 44 | /** 45 | * Run some callback. If it completes successfully, the value will be returned as `AtomOk`. If an 46 | * error is thrown, it will be caught and returned as an `AtomException`. `AtomError` will never be 47 | * produced from this helper. 48 | */ 49 | export async function run( 50 | cb: () => MaybePromise, 51 | trace: string[], 52 | ): Promise | AtomException> { 53 | try { 54 | return ok(await normalisePromise(cb())) as AtomOk; 55 | } catch (e) { 56 | return exception(e, trace) as AtomException; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | concurrency: ${{ github.workflow }}-${{ github.ref }} 9 | 10 | permissions: 11 | contents: write 12 | pull-requests: write 13 | pages: write 14 | id-token: write 15 | 16 | jobs: 17 | release: 18 | name: Release 19 | runs-on: ubuntu-latest 20 | steps: 21 | - name: Checkout Repo 22 | uses: actions/checkout@v4 23 | 24 | - name: Setup Node.js 20.x 25 | uses: actions/setup-node@v4 26 | with: 27 | node-version: 20.x 28 | 29 | - name: Install Dependencies 30 | run: npm ci 31 | 32 | - name: Create release pull request or publish to npm 33 | id: changesets 34 | uses: changesets/action@v1 35 | with: 36 | publish: npm run ci:release 37 | env: 38 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 39 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 40 | 41 | generate-docs: 42 | needs: release 43 | 44 | name: Generate and deploy documentation 45 | 46 | environment: 47 | name: github-pages 48 | url: ${{ steps.deployment.outputs.page_url }} 49 | 50 | runs-on: ubuntu-latest 51 | 52 | steps: 53 | - name: Checkout Repo 54 | uses: actions/checkout@v4 55 | 56 | - name: Setup Node.js 20.x 57 | uses: actions/setup-node@v4 58 | with: 59 | node-version: 20.x 60 | 61 | - name: Install Dependencies 62 | run: npm ci 63 | 64 | - name: Generate Docs 65 | run: npm run doc 66 | 67 | - name: Setup pages 68 | uses: actions/configure-pages@v4 69 | 70 | - name: Upload docs 71 | uses: actions/upload-pages-artifact@v3 72 | with: 73 | path: ./docs 74 | 75 | - name: Deploy to Github Pages 76 | id: deployment 77 | uses: actions/deploy-pages@v4 78 | -------------------------------------------------------------------------------- /src/atom.ts: -------------------------------------------------------------------------------- 1 | export const VALUE = Symbol.for("VALUE"); 2 | export const ERROR = Symbol.for("ERROR"); 3 | export const EXCEPTION = Symbol.for("EXCEPTION"); 4 | /** @deprecated use `EXCEPTION` instead */ 5 | export const UNKNOWN = EXCEPTION; 6 | 7 | export type AtomOk = { type: typeof VALUE; value: T }; 8 | export type AtomError = { type: typeof ERROR; value: E }; 9 | export type AtomException = { type: typeof EXCEPTION; value: unknown; trace: Array }; 10 | /** @deprecated use `AtomException` instead */ 11 | export type AtomUnknown = AtomException; 12 | 13 | export type Atom = AtomOk | AtomError | AtomException; 14 | export type MaybeAtom = T | Atom; 15 | 16 | export const ok = (value: T): Atom => ({ type: VALUE, value }); 17 | export const error = (error: E): Atom => ({ type: ERROR, value: error }); 18 | export const exception = (error: unknown, trace: Array): Atom => ({ 19 | type: EXCEPTION, 20 | value: error, 21 | trace: [...trace], 22 | }); 23 | /** @deprecated use `exception` instead */ 24 | export const unknown = exception; 25 | 26 | export const isOk = (atom: Atom): atom is AtomOk => atom.type === VALUE; 27 | export const isError = (atom: Atom): atom is AtomError => atom.type === ERROR; 28 | export const isException = (atom: Atom): atom is AtomException => 29 | atom.type === EXCEPTION; 30 | /** @deprecated use `isUnknown` instead */ 31 | export const isUnknown = isException; 32 | 33 | /** 34 | * Given some value (which may or may not be an atom), convert it into an atom. If it is not an 35 | * atom, then it will be turned into an `ok` atom. 36 | */ 37 | export function normalise(value: MaybeAtom): Atom { 38 | if ( 39 | value !== null && 40 | typeof value === "object" && 41 | "type" in value && 42 | (isOk(value) || isError(value) || isException(value)) 43 | ) { 44 | return value; 45 | } else { 46 | return ok(value); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "windpipe", 3 | "version": "0.13.1", 4 | "description": "Highland but better", 5 | "type": "module", 6 | "scripts": { 7 | "lint": "tsc && eslint .", 8 | "format": "eslint --fix .", 9 | "build": "tsup ./src/index.ts", 10 | "doc": "typedoc ./src --media ./media --plugin typedoc-plugin-extras --favicon ./media/logo.png --footerLastModified true --plugin typedoc-material-theme --themeColor '#03284e' --plugin typedoc-plugin-rename-defaults", 11 | "test": "vitest", 12 | "ci:release": "npm run build && changeset publish" 13 | }, 14 | "keywords": [], 15 | "files": [ 16 | "dist", 17 | "README.md" 18 | ], 19 | "main": "./dist/index.cjs", 20 | "types": "./dist/index.d.cts", 21 | "exports": { 22 | ".": { 23 | "import": { 24 | "types": "./dist/index.d.ts", 25 | "default": "./dist/index.js" 26 | }, 27 | "require": { 28 | "types": "./dist/index.d.cts", 29 | "default": "./dist/index.cjs" 30 | } 31 | } 32 | }, 33 | "author": { 34 | "name": "Tom Anderson", 35 | "email": "tom@ando.gq", 36 | "url": "https://ando.gq" 37 | }, 38 | "repository": { 39 | "type": "git", 40 | "url": "git+https://github.com/andogq/windpipe.git" 41 | }, 42 | "license": "ISC", 43 | "devDependencies": { 44 | "@changesets/cli": "^2.27.1", 45 | "@eslint/js": "^9.0.0", 46 | "@types/highland": "^2.13.0", 47 | "@types/node": "^20.10.7", 48 | "eslint": "^8.57.0", 49 | "eslint-config-prettier": "^9.1.0", 50 | "eslint-plugin-prettier": "^5.1.3", 51 | "globals": "^15.0.0", 52 | "highland": "^2.13.5", 53 | "npm-run-all": "^4.1.5", 54 | "prettier": "3.2.5", 55 | "tsup": "^8.0.1", 56 | "typedoc": "^0.25.7", 57 | "typedoc-material-theme": "^1.0.2", 58 | "typedoc-plugin-extras": "^3.0.0", 59 | "typedoc-plugin-rename-defaults": "^0.7.0", 60 | "typescript": "^5.3.3", 61 | "typescript-eslint": "^7.6.0", 62 | "vitest": "^1.6.0" 63 | }, 64 | "publishConfig": { 65 | "access": "public" 66 | }, 67 | "tsup": { 68 | "format": [ 69 | "esm", 70 | "cjs" 71 | ], 72 | "splitting": true, 73 | "cjsInterop": true, 74 | "dts": { 75 | "footer": "export = Stream" 76 | } 77 | }, 78 | "packageManager": "pnpm@8.14.0+sha1.bb42032ff80dba5f9245bc1b03470d2fa0b7fb2f" 79 | } 80 | -------------------------------------------------------------------------------- /src/stream/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ok, 3 | error, 4 | exception, 5 | isOk, 6 | isError, 7 | isException, 8 | type Atom, 9 | type AtomOk, 10 | type AtomError, 11 | type AtomException, 12 | } from "../atom"; 13 | import { HigherOrderStream } from "./higher-order"; 14 | 15 | export type { StreamEnd } from "./base"; 16 | export { WindpipeConsumptionError } from "./consumption"; 17 | 18 | /** 19 | * @template T - Type of the 'values' on the stream. 20 | * @template E - Type of the 'errors' on the stream. 21 | */ 22 | export class Stream extends HigherOrderStream { 23 | // Re-export atom utilities for convenience 24 | /** 25 | * Create an `ok` atom with the provided value. 26 | * 27 | * @group Atom 28 | */ 29 | static ok(value: T): Atom { 30 | return ok(value); 31 | } 32 | 33 | /** 34 | * Create an `error` atom with the provided value. 35 | * 36 | * @group Atom 37 | */ 38 | static error(value: E): Atom { 39 | return error(value); 40 | } 41 | 42 | /** 43 | * Create an `exception` atom with the provided value. 44 | * 45 | * @group Atom 46 | */ 47 | static exception(value: unknown, trace: string[]): Atom { 48 | return exception(value, trace); 49 | } 50 | 51 | /** 52 | * @group Atom 53 | * @deprecated use `exception` instead 54 | */ 55 | static unknown(value: unknown, trace: string[]): Atom { 56 | return this.exception(value, trace); 57 | } 58 | 59 | /** 60 | * Verify if the provided atom is of the `ok` variant. 61 | * 62 | * @group Atom 63 | */ 64 | static isOk(atom: Atom): atom is AtomOk { 65 | return isOk(atom); 66 | } 67 | 68 | /** 69 | * Verify if the provided atom is of the `error` variant. 70 | * 71 | * @group Atom 72 | */ 73 | static isError(atom: Atom): atom is AtomError { 74 | return isError(atom); 75 | } 76 | 77 | /** 78 | * Verify if the provided atom is of the `exception` variant. 79 | * 80 | * @group Atom 81 | */ 82 | static isException(atom: Atom): atom is AtomException { 83 | return isException(atom); 84 | } 85 | 86 | /** 87 | * @group Atom 88 | * @deprecated use `isException` instead 89 | */ 90 | static isUnknown(atom: Atom): atom is AtomException { 91 | return this.isException(atom); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /test/errors.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, test } from "vitest"; 2 | import $ from "../src"; 3 | 4 | describe.concurrent("error handling", () => { 5 | test("throw in map", async ({ expect }) => { 6 | expect.assertions(1); 7 | 8 | const s = $.from([1, 2, 3]).map((n) => { 9 | if (n === 2) { 10 | // Unhandled error 11 | throw new Error("bad number"); 12 | } else { 13 | return n; 14 | } 15 | }); 16 | 17 | expect(await s.toArray({ atoms: true })).toEqual([ 18 | $.ok(1), 19 | $.exception(new Error("bad number"), ["map"]), 20 | $.ok(3), 21 | ]); 22 | }); 23 | 24 | test("promise rejection in map", async ({ expect }) => { 25 | expect.assertions(1); 26 | 27 | async function process(n: number) { 28 | if (n === 2) { 29 | throw new Error("bad number"); 30 | } else { 31 | return n; 32 | } 33 | } 34 | 35 | const s = $.from([1, 2, 3]).map(process); 36 | 37 | expect(await s.toArray({ atoms: true })).toEqual([ 38 | $.ok(1), 39 | $.exception(new Error("bad number"), ["map"]), 40 | $.ok(3), 41 | ]); 42 | }); 43 | 44 | test("track multiple transforms", async ({ expect }) => { 45 | expect.assertions(1); 46 | 47 | const s = $.from([1, 2, 3, 4, 5]) 48 | .map((n) => { 49 | if (n === 2) { 50 | // Unhandled error 51 | throw new Error("bad number"); 52 | } else { 53 | return n; 54 | } 55 | }) 56 | .filter((n) => n % 2 === 0); 57 | 58 | expect(await s.toArray({ atoms: true })).toEqual([ 59 | $.exception(new Error("bad number"), ["map"]), 60 | $.ok(4), 61 | ]); 62 | }); 63 | 64 | test("error thrown in later transform", async ({ expect }) => { 65 | expect.assertions(1); 66 | 67 | const s = $.from([1, 2, 3, 4, 5]) 68 | .filter((n) => n > 1) 69 | .map((n) => { 70 | if (n % 2 === 1) { 71 | return n * 10; 72 | } else { 73 | return n; 74 | } 75 | }) 76 | .map((n) => { 77 | if (n === 2) { 78 | // Unhandled error 79 | throw new Error("bad number"); 80 | } else { 81 | return n; 82 | } 83 | }) 84 | .filter((n) => n % 2 === 0); 85 | 86 | expect(await s.toArray({ atoms: true })).toEqual([ 87 | $.exception(new Error("bad number"), ["filter", "map", "map"]), 88 | $.ok(30), 89 | $.ok(4), 90 | $.ok(50), 91 | ]); 92 | }); 93 | }); 94 | -------------------------------------------------------------------------------- /src/util.ts: -------------------------------------------------------------------------------- 1 | import Stream, { type Atom } from "."; 2 | 3 | /** 4 | * Maybe it's a promise. Maybe it's not. Who's to say. 5 | */ 6 | export type MaybePromise = Promise | T; 7 | 8 | /** 9 | * The truthy representation of some type. Will ensure that the type is not null/undefined, and 10 | * isn't false, or an empty string. 11 | */ 12 | export type Truthy = NonNullable>; 13 | 14 | /** 15 | * Type that may be a callback that resolves to a stream, or just a stream. 16 | */ 17 | export type CallbackOrStream = (() => Stream) | Stream; 18 | 19 | /** 20 | * Completely exhausts the provided async iterator. 21 | */ 22 | export async function exhaust(iterable: AsyncIterable) { 23 | const it = iterable[Symbol.asyncIterator](); 24 | 25 | // eslint-disable-next-line no-constant-condition 26 | while (true) { 27 | const result = await it.next(); 28 | if (result.done) { 29 | break; 30 | } 31 | } 32 | } 33 | 34 | export type NodeCallback = (err: E | null, value?: T) => void; 35 | 36 | /** 37 | * Creates a `next` function and associated promise to promise-ify a node style callback. The 38 | * `next` function must be passed as the callback to a function, and the resulting error or value 39 | * will be emitted from the promise. The promise will always resolve. 40 | * 41 | * The error value of the callback (first parameter) will be emitted as an `Error` atom from the 42 | * promise, whilst the value of the callback (second parameter) will be emitted as an `Ok` atom on 43 | * the promise. 44 | */ 45 | export function createNodeCallback(): [Promise>, NodeCallback] { 46 | // Resolve function to be hoisted out of the promise 47 | let resolve: (atom: Atom) => void; 48 | 49 | // Create the prom 50 | const promise = new Promise>((res) => { 51 | resolve = res; 52 | }); 53 | 54 | // Create the next callback 55 | const next: NodeCallback = (err, value) => { 56 | if (err) { 57 | resolve(Stream.error(err)); 58 | } else if (value) { 59 | resolve(Stream.ok(value)); 60 | } 61 | }; 62 | 63 | // Return a tuple of the promise and next function 64 | return [promise, next]; 65 | } 66 | 67 | export type Signal = Promise & { done: () => void }; 68 | 69 | /** 70 | * Create a new 'signal', which is just a Promise that has the `resolve` method exposed on 71 | * the promise itself, allowing the Promise to be resolved outside of the callback by 72 | * calling `Promise.done()`. 73 | */ 74 | export function newSignal(): Signal { 75 | let done: () => void; 76 | 77 | // @ts-expect-error building signal object 78 | const signal: Signal = new Promise((resolve) => { 79 | done = resolve; 80 | }); 81 | 82 | // @ts-expect-error done assigned in promise initialiser 83 | signal.done = done; 84 | 85 | return signal; 86 | } 87 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # windpipe 2 | 3 | ## 0.13.1 4 | 5 | ### Patch Changes 6 | 7 | - f1f6f1f: Fix .merge() failing on slow streams 8 | 9 | ## 0.13.0 10 | 11 | ### Minor Changes 12 | 13 | - c6e43cd: Implement Stream.merge() 14 | 15 | ## 0.12.0 16 | 17 | ### Minor Changes 18 | 19 | - 9ef31c7: allow different error types when existing stream has `never` error 20 | - 5d20101: create `single` consumption method 21 | 22 | ### Patch Changes 23 | 24 | - 180b70d: fix: export for WindpipeConsumptionError 25 | - eb8bdae: re-arrange exports in package.json 26 | 27 | ## 0.11.1 28 | 29 | ### Patch Changes 30 | 31 | - 39221db: Fix json serialisation issue when stream contains undefined values 32 | 33 | ## 0.11.0 34 | 35 | ### Minor Changes 36 | 37 | - 035cf6a: add `fromPusher` stream creation method 38 | 39 | ## 0.10.0 40 | 41 | ### Minor Changes 42 | 43 | - 7c566d2: create `onFirst` and `onLast` operators 44 | 45 | ## 0.9.10 46 | 47 | ### Patch Changes 48 | 49 | - de16dab: fix batching with timeout and yield-remaining 50 | 51 | ## 0.9.9 52 | 53 | ### Patch Changes 54 | 55 | - 8d3b6be: fix: inconsistency with batch yielding 56 | 57 | ## 0.9.8 58 | 59 | ### Patch Changes 60 | 61 | - 2c8238d: fix: buffered map continuing on stream end 62 | 63 | ## 0.9.7 64 | 65 | ### Patch Changes 66 | 67 | - 70fe273: feat: reject error atom option for `toArray` 68 | 69 | ## 0.9.6 70 | 71 | ### Patch Changes 72 | 73 | - 7300858: rename `unknown` to `exception`, and deprecate all calls to `unknown` 74 | 75 | ## 0.9.5 76 | 77 | ### Patch Changes 78 | 79 | - a87c801: fix: incorrect splicing when generating batch 80 | 81 | ## 0.9.4 82 | 83 | ### Patch Changes 84 | 85 | - 2370529: feat: allow batching by bucket 86 | 87 | ## 0.9.3 88 | 89 | ### Patch Changes 90 | 91 | - d9dcc4e: fix: add timeout with `n` option for buffer 92 | 93 | ## 0.9.2 94 | 95 | ### Patch Changes 96 | 97 | - b293d9c: `batch` operator 98 | - 784adb4: fix: continue emitting stream items after encountering non-raw item on raw stream 99 | - 87515b3: fix: emit known and unknown errors onto node stream 100 | 101 | ## 0.9.1 102 | 103 | ### Patch Changes 104 | 105 | - e050124: create `bufferedMap` operator 106 | 107 | ## 0.9.0 108 | 109 | ### Minor Changes 110 | 111 | - 8db09c4: Implement `.toReadable()` method for streams 112 | 113 | ### Patch Changes 114 | 115 | - b8a8ed7: Fix creating streams from arrays with nullish values 116 | 117 | ## 0.8.2 118 | 119 | ### Patch Changes 120 | 121 | - 01dff15: clone array in `fromArray` to prevent mutating original array 122 | 123 | ## 0.8.1 124 | 125 | ### Patch Changes 126 | 127 | - 52f03a1: fix broken types 128 | 129 | ## 0.8.0 130 | 131 | ### Minor Changes 132 | 133 | - ad86792: Add .collect() method to streams 134 | - 3ce4ff3: Implement `fromCallback` for stream creation 135 | - 909d5a1: Adds the `cachedFlatMap` operator 136 | 137 | ### Patch Changes 138 | 139 | - af01d2f: catch unhandled errors in `fromNext` stream creation 140 | - 022efea: Improve exported API and generated docs 141 | 142 | ## 0.7.0 143 | 144 | ### Minor Changes 145 | 146 | - 10e211e: Implement .flatten() method on streams 147 | 148 | ## 0.6.0 149 | 150 | ### Minor Changes 151 | 152 | - 31a0db7: alter `flat*` APIs to simplify handlers 153 | - ce64206: fix export for CJS 154 | 155 | ### Patch Changes 156 | 157 | - e9ea819: add `exhaust` stream consumer 158 | 159 | ## 0.5.1 160 | 161 | ### Patch Changes 162 | 163 | - 56147df: fix incorrect stream type for `flatTap` 164 | 165 | ## 0.5.0 166 | 167 | ### Minor Changes 168 | 169 | - 27191f4: fix default exports for cjs 170 | 171 | ## 0.4.0 172 | 173 | ### Minor Changes 174 | 175 | - e55d490: create new `ofError` and `ofUnknown` static methods for creating a stream 176 | - e55d490: alter exported API 177 | - 341ef76: add `flatTap` 178 | 179 | ## 0.3.1 180 | 181 | ### Patch Changes 182 | 183 | - edd7aad: add Atom types to export 184 | 185 | ## 0.3.0 186 | 187 | ### Minor Changes 188 | 189 | - 0aebf68: add `$.ok`, `$.error`, and `$.unknown` methods, to simplify creation of single atom streams. 190 | 191 | restrict items exported from `./atom.ts`, as not everything was required for the public API. 192 | 193 | ## 0.2.0 194 | 195 | ### Minor Changes 196 | 197 | - 22723f5: add `drop` stream transform 198 | -------------------------------------------------------------------------------- /test/benchmarks/index.bench.ts: -------------------------------------------------------------------------------- 1 | import { describe, bench } from "vitest"; 2 | import Stream from "../../src"; 3 | import Highland from "highland"; 4 | import { error, ok } from "../../src/atom"; 5 | 6 | const SAMPLE_SIZE = 10; 7 | const ARRAY = new Array(SAMPLE_SIZE).fill(undefined).map((_, i) => i); 8 | 9 | describe("stream creation from next function", () => { 10 | bench("windpipe", async () => { 11 | let i = 0; 12 | await Stream.from(async () => { 13 | if (i < SAMPLE_SIZE) { 14 | return i++; 15 | } else { 16 | return Stream.StreamEnd; 17 | } 18 | }).toArray(); 19 | }); 20 | 21 | bench("highland", () => { 22 | return new Promise((resolve) => { 23 | let i = 0; 24 | 25 | Highland((push, next) => { 26 | if (i < SAMPLE_SIZE) { 27 | push(null, i++); 28 | next(); 29 | } else { 30 | push(null, Highland.nil); 31 | } 32 | }).toArray(() => resolve()); 33 | }); 34 | }); 35 | }); 36 | 37 | describe("simple transform operations", () => { 38 | bench("windpipe", async () => { 39 | await Stream.from(ARRAY) 40 | .map((n) => n + 100) 41 | .toArray(); 42 | }); 43 | 44 | bench("highland", () => { 45 | return new Promise((resolve) => { 46 | Highland(ARRAY) 47 | .map((n) => n + 100) 48 | .toArray(() => resolve()); 49 | }); 50 | }); 51 | }); 52 | 53 | describe("sample data operations", () => { 54 | bench("windpipe", async () => { 55 | await Stream.from< 56 | { 57 | name: string; 58 | id: number; 59 | permissions: { read: boolean; write: boolean }; 60 | balance: number; 61 | }, 62 | string 63 | >([ 64 | { 65 | name: "test user 1", 66 | id: 1, 67 | permissions: { 68 | read: true, 69 | write: true, 70 | }, 71 | balance: 100, 72 | }, 73 | { 74 | name: "test user 2", 75 | id: 2, 76 | permissions: { 77 | read: true, 78 | write: false, 79 | }, 80 | balance: -100, 81 | }, 82 | { 83 | name: "test user 3", 84 | id: 3, 85 | permissions: { 86 | read: false, 87 | write: false, 88 | }, 89 | balance: 83, 90 | }, 91 | { 92 | name: "test user 4", 93 | id: 4, 94 | permissions: { 95 | read: true, 96 | write: false, 97 | }, 98 | balance: 79, 99 | }, 100 | { 101 | name: "test user 5", 102 | id: 5, 103 | permissions: { 104 | read: true, 105 | write: false, 106 | }, 107 | balance: 51, 108 | }, 109 | ]) 110 | .map((person) => { 111 | if (person.balance < 0) { 112 | return error("invalid balance"); 113 | } else { 114 | return ok(person); 115 | } 116 | }) 117 | .filter((person) => { 118 | return person.permissions.read; 119 | }) 120 | .map((person) => { 121 | return { 122 | id: person.id, 123 | balance: person.balance, 124 | }; 125 | }) 126 | .map(({ id, balance }) => `#${id}: $${balance}.00`) 127 | .toArray(); 128 | }); 129 | 130 | bench("highland", () => { 131 | return new Promise((resolve) => { 132 | Highland([ 133 | { 134 | name: "test user 1", 135 | id: 1, 136 | permissions: { 137 | read: true, 138 | write: true, 139 | }, 140 | balance: 100, 141 | }, 142 | { 143 | name: "test user 2", 144 | id: 2, 145 | permissions: { 146 | read: true, 147 | write: false, 148 | }, 149 | balance: -100, 150 | }, 151 | { 152 | name: "test user 3", 153 | id: 3, 154 | permissions: { 155 | read: false, 156 | write: false, 157 | }, 158 | balance: 83, 159 | }, 160 | { 161 | name: "test user 4", 162 | id: 4, 163 | permissions: { 164 | read: true, 165 | write: false, 166 | }, 167 | balance: 79, 168 | }, 169 | { 170 | name: "test user 5", 171 | id: 5, 172 | permissions: { 173 | read: true, 174 | write: false, 175 | }, 176 | balance: 51, 177 | }, 178 | ]) 179 | .map((person) => { 180 | if (person.balance < 0) { 181 | throw "invalid balance"; 182 | } else { 183 | return person; 184 | } 185 | }) 186 | .filter((person) => { 187 | return person.permissions.read; 188 | }) 189 | .map((person) => { 190 | return { 191 | id: person.id, 192 | balance: person.balance, 193 | }; 194 | }) 195 | .map(({ id, balance }) => `#${id}: $${balance}.00`) 196 | .errors(() => {}) 197 | .toArray(() => resolve()); 198 | }); 199 | }); 200 | }); 201 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | Logo 3 |

4 | 5 |

6 | TypeScript streams influenced by Rust, an experimental successor to Highland. 7 |

8 | 9 | # Features 10 | 11 | - Strict typing of stream values 12 | 13 | - Built-in error handling for stream operaitons 14 | 15 | - Many common stream operations (`map`, `tap`, `flat_map`, etc.) 16 | 17 | - Interopability with other async primitives: 18 | 19 | - Promises 20 | 21 | - Iterators (async and standard) 22 | 23 | - Node streams (WIP) 24 | 25 | - Stream kind (single emitter, multi-emitter, fallible, etc) in type system (WIP) 26 | 27 | - Control flow operators (`if` and `case`, WIP) 28 | 29 | - Stream-relative stack traces (nested anonymous functions in stack traces no more!) 30 | 31 | # Examples 32 | 33 | ## Simple 34 | 35 | ```ts 36 | const values = await Stream.of(10) // Create a stream with a single value 37 | .map((value) => value * 2) // Double each value in the stream 38 | .tap((value) => console.log(`The doubled value is: ${value}`)) // Perform some side effect on each value 39 | .flat_map((value) => [value, -1 * value]) // Use each value to produce multiple new values 40 | .toArray(); // Consume the stream into a promise that will emit an array 41 | 42 | console.log(values); // [20, -20] 43 | ``` 44 | 45 | ## Error Handling 46 | 47 | ```ts 48 | const s = Stream.from([1, 2, 5, 0]) 49 | .map((value) => { 50 | if (value === 0) { 51 | // Invalid value, produce an error 52 | return err({ msg: "can't divide by zero"}); 53 | } 54 | 55 | return 10 / value; 56 | }) 57 | .map_err((err) => ({ 58 | // We have to shout at the user, change the error message 59 | loud_msg: err.msg.toUpperCase() 60 | })); 61 | 62 | 63 | while (true) { 64 | const next = await s.next(); 65 | 66 | if (is_end(next)) { 67 | // End of the stream reached! 68 | break; 69 | } else if (is_ok(next)) { 70 | // Successful value! 71 | console.log(next.value); 72 | } else if (is_err(next)) { 73 | // An error was encountered 74 | console.error("encountered an error:", next.value); 75 | } 76 | } 77 | 78 | /* Outputs: 79 | (log) 10 80 | (log) 5 81 | (log) 2 82 | (error) { loud_msg: "encountered an error: CAN'T DIVIDE BY ZERO" } 83 | */ 84 | ``` 85 | 86 | ## Control Flow 87 | 88 | In an effort to make common patterns like applying `if` statements in a `flat_map`, some basic 89 | control flow operations are available directly on the stream! 90 | 91 | ```ts 92 | // Using stream control flow: 93 | Stream.from([83, 18398, 915, 618]) 94 | .if((value) => value > 750, (value) => process_large_transaction(value)) 95 | .else_if((value) => value < 100, (value) => process_small_transaction(value)) 96 | .else((value) => process_transaction(value)) 97 | 98 | // Using regular `flat_map` with native control flow 99 | Stream.from([83, 18398, 915, 618]) 100 | .flat_map((value) => { 101 | if (value > 750) { 102 | return process_large_transaction(value); 103 | } else if (value < 100) { 104 | return process_small_transaction(value); 105 | } else { 106 | return process_transaction(value); 107 | } 108 | }); 109 | ``` 110 | 111 | # Error Handling 112 | 113 | Error handling is a crucial component to every application, however languages like JavaScript and 114 | TypeScript make it exceptionally easy to omit error handling, producing unrelaiable applications, 115 | and increasing debugging time due to there being no documentation within the source of where an 116 | error may be produced. 117 | 118 | Windpipe attempts to solve this by including errors as part of the core stream type: 119 | `Stream`, where `T` is the type of the 'success' value of the stream, and `E` is the type of 120 | the 'application error'. For example, if a stream was being used as part of a financial system for 121 | processing transactions then `T` could be `number` (representing the current value of the 122 | transaction) and `E` could be an instance of `TransactionError` which could include fields like 123 | `error_no` and `message`. As stream operations are applied to the value in the stream, each step 124 | may emit `T` onwards (indicating that all is well, the stream can continue), or `E` to immediately 125 | terminate that value in the stream and produce an error. Including the type of the error in the 126 | stream makes it obvious to the consumer that a) an error may be produced, and b) the shape of the 127 | error so that there aren't any runtime gymnastics to work out what this error value actually is. 128 | 129 | Windpipe also includes a third variant that isn't encoded in the type, aptly called `unknown`. This 130 | is meant to handle the many possible unhandled errors that may be thrown within an application that 131 | the developer mightn't have explicitly handled. This attempts to address the prevalent issue with 132 | JavaScript where it's impossible to know what errors may be thrown. 133 | 134 | The result of this is that the developer has a choice of three things to consume from each 135 | advancement of the stream: 136 | 137 | - `T`: The value we would expect to return from the stream 138 | 139 | - `E`: Some application error that we have explicitly defined and will have types for 140 | 141 | - `any`: An unknown error that was thrown somewhere, which we could try recover from by turning it 142 | into a value or an application error 143 | 144 | With an error type encoded in our type, we can do all the same fun things we can do with the 145 | stream values, such as mapping errors into a different error type. For example, if a login page 146 | uses a HTTP helper that produces `Stream`, `.map_err` can be used to convert 147 | `HttpError` into a `LoginError`. 148 | 149 | # Windpipe vs x 150 | 151 | Windpipe is an attempt to plug a few holes left by other solutions. 152 | 153 | ## Windpipe vs [Highland](https://github.com/caolan/highland) 154 | 155 | Highland is what originally spurred me to create Windpipe. Functionally, it's relatively similar 156 | however its type support leaves a lot to be desired. The `@types/*` definitions for the package are 157 | incomplete, out of date, or incorrect, and the library itself is unmaintained. Finally, Highland 158 | has no support for typed application errors, it only emits as either a 'value' or an 'error'. 159 | 160 | I am intending for Windpipe to be mostly compatible with Highland (particularly for the basic 161 | operators), ideally with some kind of adapter to convert one stream type into another, for partial 162 | adoption. 163 | 164 | ## Windpipe vs Native Promises 165 | 166 | Compared to Windpipe, native promises have three main downfalls: 167 | 168 | - Promises are eagerly executed, meaning that they will immediately try to resolve their value when 169 | they are created, rather than when they are called. 170 | 171 | - Promises may only emit a single value or a single error, emitting a mix or multiple of either is 172 | not possible without other (cumbersome) APIs such as async iterators. 173 | 174 | - Only the 'success' value of a promise is typed, the error value is completely untyped and may be 175 | anything. 176 | 177 | Windpipe should be able to completely support any use case that promises may be used for, in 178 | addition with many features that promises can never support. 179 | 180 | -------------------------------------------------------------------------------- /src/stream/consumption.ts: -------------------------------------------------------------------------------- 1 | import { Readable } from "node:stream"; 2 | import { isOk, type Atom } from "../atom"; 3 | import { StreamBase } from "./base"; 4 | import { exhaust } from "../util"; 5 | 6 | export class WindpipeConsumptionError extends Error { 7 | static noItems() { 8 | return new WindpipeConsumptionError("no items found whilst consuming stream"); 9 | } 10 | } 11 | 12 | export class StreamConsumption extends StreamBase { 13 | /** 14 | * Create an iterator that will emit each atom in the stream. 15 | * 16 | * @group Consumption 17 | */ 18 | [Symbol.asyncIterator](): AsyncIterator> { 19 | return this.stream[Symbol.asyncIterator](); 20 | } 21 | 22 | /** 23 | * Completely exhaust the stream, driving it to completion. This is particularly useful when 24 | * side effects of the stream are desired, but the actual values of the stream are not needed. 25 | */ 26 | exhaust(): Promise { 27 | return exhaust(this); 28 | } 29 | 30 | /** 31 | * Create an async iterator that will emit each value in the stream. 32 | * 33 | * @group Consumption 34 | */ 35 | values(): AsyncIterableIterator { 36 | const it = this[Symbol.asyncIterator](); 37 | 38 | async function next() { 39 | const { value, done } = await it.next(); 40 | 41 | if (done) { 42 | return { value, done: true }; 43 | } else if (isOk(value)) { 44 | return { value: value.value }; 45 | } else { 46 | return await next(); 47 | } 48 | } 49 | 50 | return { 51 | [Symbol.asyncIterator](): AsyncIterableIterator { 52 | // WARN: This feels weird, however it follows what the types require 53 | return { 54 | [Symbol.asyncIterator]: this[Symbol.asyncIterator], 55 | next, 56 | }; 57 | }, 58 | next, 59 | }; 60 | } 61 | 62 | /** 63 | * Pull the stream once and remove the first item. This will not consume the rest of the 64 | * stream. 65 | * 66 | * @note This method can only be called once on a given stream. 67 | */ 68 | single(options: { atom: true; optional: true }): Promise | undefined>; 69 | single(options: { atom: true; optional?: false }): Promise>; 70 | single(options: { atom?: false; optional: true }): Promise; 71 | single(options?: { atom?: false; optional?: false }): Promise; 72 | async single({ 73 | atom = false, 74 | optional = false, 75 | }: { 76 | atom?: boolean; 77 | optional?: boolean; 78 | } = {}): Promise | undefined> { 79 | const it = this[Symbol.asyncIterator](); 80 | 81 | const { value, done } = await it.next(); 82 | 83 | if (done) { 84 | if (optional) { 85 | // Fine to return undefined 86 | return undefined; 87 | } 88 | 89 | throw WindpipeConsumptionError.noItems(); 90 | } 91 | 92 | if (atom) { 93 | return value; 94 | } 95 | 96 | if (isOk(value)) { 97 | return value.value; 98 | } 99 | 100 | throw value.value; 101 | } 102 | 103 | /** 104 | * Iterate through each atom in the stream, and return them as a single array. 105 | * 106 | * @param options.atoms - Return every atom on the stream. 107 | * @param options.reject - If an error or exception is encountered, reject the promise with it. 108 | * 109 | * @group Consumption 110 | */ 111 | async toArray(options?: { atoms?: false; reject?: boolean }): Promise; 112 | async toArray(options?: { atoms: true }): Promise[]>; 113 | async toArray(options?: { atoms?: boolean; reject?: boolean }): Promise<(Atom | T)[]> { 114 | const array: (Atom | T)[] = []; 115 | 116 | for await (const atom of this) { 117 | if (options?.atoms) { 118 | array.push(atom); 119 | } else if (isOk(atom)) { 120 | array.push(atom.value); 121 | } else if (options?.reject) { 122 | throw atom.value; 123 | } 124 | } 125 | 126 | return array; 127 | } 128 | 129 | /** 130 | * Serialise the stream, and produce a Node stream with the serialised result. 131 | * 132 | * @param options.single - Whether to emit an array for multiple items, or only a single item. 133 | * @param options.atoms - By default, only `ok` values are serialised, however enabling this 134 | * will serialise all values. 135 | * 136 | * @note this will skip `undefined` values as they cannot be serialised. 137 | * 138 | * @see {@link Stream#toReadable} if serialisation is not required 139 | * @group Consumption 140 | */ 141 | serialise(options?: { single?: boolean; atoms?: boolean }): Readable { 142 | // Set up a new readable stream that does nothing 143 | const s = new Readable({ 144 | read() {}, 145 | }); 146 | 147 | // Spin off asynchronously so that the stream can be immediately returned 148 | (async () => { 149 | let sentItems = 0; 150 | 151 | if (options?.single !== true) { 152 | s.push("["); 153 | } 154 | 155 | for await (const atom of this) { 156 | // Determine whether non-ok values should be filtered out 157 | if (options?.atoms !== true && !isOk(atom)) { 158 | continue; 159 | } 160 | 161 | // Skip undefined values (they cannot be serialised into JSON) 162 | if (atom.value === undefined) { 163 | continue; 164 | } 165 | 166 | if (sentItems > 0) { 167 | if (options?.single) { 168 | // Monitor for multiple values being sent when only one is desired 169 | console.warn( 170 | "indicated that stream would serialise to a single value, however multiple were emitted (ignoring)", 171 | ); 172 | break; 173 | } else { 174 | // Comma seperate multiple values 175 | s.push(","); 176 | } 177 | } 178 | 179 | s.push(JSON.stringify(options?.atoms ? atom : atom.value)); 180 | 181 | sentItems += 1; 182 | } 183 | 184 | if (options?.single !== true) { 185 | s.push("]"); 186 | } 187 | 188 | // End the stream 189 | s.push(null); 190 | })(); 191 | 192 | return s; 193 | } 194 | 195 | /** 196 | * Produce a readable node stream with the values from the stream. 197 | * 198 | * @param kind - What kind of readable stream to produce. When "raw" only strings and buffers can be emitted on the stream. Use "object" to preserve 199 | * objects in the readable stream. Note that object values are not serialised, they are emitted as objects. 200 | * @param options - Options for configuring how atoms are output on the stream 201 | * 202 | * @see {@link Stream#serialize} if the stream values should be serialized to json 203 | * @group Consumption 204 | */ 205 | toReadable(kind: "raw" | "object", options?: { atoms?: boolean }): Readable; 206 | 207 | /** 208 | * Produce a readable node stream with the raw values from the stream 209 | * @note the stream must only contain atoms of type `string` or `Buffer`. If not, a 210 | * stream error will be emitted. 211 | * 212 | * @param options.single - Whether to emit only the first atom 213 | * 214 | * @see {@link Stream#serialize} if the stream values should be serialized to json 215 | * @group Consumption 216 | */ 217 | toReadable(kind: "raw"): Readable; 218 | 219 | /** 220 | * Produce a readable node stream in object mode with the values from the stream 221 | * 222 | * @param options.atoms - By default, only `ok` values are emitted, however enabling this 223 | * will emit all values. 224 | * 225 | * @note When not using `options.atoms`, any `null` atom values will be skipped when piping to the readable stream 226 | * @see {@link Stream#serialize} if the stream values should be serialized to json 227 | * @group Consumption 228 | */ 229 | toReadable(kind: "object", options?: { atoms?: boolean }): Readable; 230 | 231 | toReadable( 232 | kind: "raw" | "object", 233 | options: { single?: boolean; atoms?: boolean } = {}, 234 | ): Readable { 235 | // Set up a new readable stream that does nothing 236 | const s = new Readable({ 237 | read() {}, 238 | objectMode: kind === "object", 239 | }); 240 | 241 | // Spin off asynchronously so that the stream can be immediately returned 242 | (async () => { 243 | for await (const atom of this) { 244 | // Determine whether non-ok values should be filtered out 245 | if (options?.atoms !== true && !isOk(atom)) { 246 | s.emit("error", atom.value); 247 | continue; 248 | } 249 | 250 | // monitor for non raw values when not using object mode 251 | if ( 252 | kind === "raw" && 253 | !(typeof atom.value === "string" || atom.value instanceof Buffer) 254 | ) { 255 | const message = `Stream indicated it would emit raw values but emitted a '${typeof atom.value}' object`; 256 | console.error(message); 257 | s.emit("error", new Error(message)); 258 | continue; 259 | } 260 | 261 | // Show a warning if any atom value is null 262 | if (!options?.atoms && atom.value === null) { 263 | console.warn( 264 | "Stream attempted to emit a `null` value in object mode which would have ended the stream early. (Skipping)", 265 | ); 266 | continue; 267 | } 268 | 269 | // Emit atom or atom value 270 | s.push(options?.atoms ? atom : atom.value); 271 | } 272 | 273 | // End the stream 274 | s.push(null); 275 | })(); 276 | 277 | return s; 278 | } 279 | } 280 | -------------------------------------------------------------------------------- /test/creation.test.ts: -------------------------------------------------------------------------------- 1 | import { afterEach, beforeEach, describe, test, vi } from "vitest"; 2 | import $, { type NodeCallback } from "../src"; 3 | import { Readable } from "stream"; 4 | 5 | describe("stream creation", () => { 6 | describe.concurrent("from promise", () => { 7 | test("resolving promise to emit value", async ({ expect }) => { 8 | expect.assertions(1); 9 | 10 | const s = $.fromPromise(Promise.resolve(10)); 11 | 12 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(10)]); 13 | }); 14 | }); 15 | 16 | describe.concurrent("from iterator", () => { 17 | test("multi-value generator", async ({ expect }) => { 18 | expect.assertions(1); 19 | 20 | const s = $.fromIterator( 21 | (function* () { 22 | yield 1; 23 | yield 2; 24 | yield 3; 25 | })(), 26 | ); 27 | 28 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3)]); 29 | }); 30 | 31 | test("multi-value async generator", async ({ expect }) => { 32 | expect.assertions(1); 33 | 34 | const s = $.fromIterator( 35 | (async function* () { 36 | yield 1; 37 | yield 2; 38 | yield 3; 39 | })(), 40 | ); 41 | 42 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3)]); 43 | }); 44 | }); 45 | 46 | describe.concurrent("from iterable", () => { 47 | test("array iterable", async ({ expect }) => { 48 | expect.assertions(1); 49 | 50 | const s = $.fromIterable([1, 2, 3]); 51 | 52 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3)]); 53 | }); 54 | 55 | test("readable stream", async ({ expect }) => { 56 | expect.assertions(1); 57 | 58 | const s = $.fromIterable(Readable.from([1, 2, 3])); 59 | 60 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3)]); 61 | }); 62 | }); 63 | 64 | describe.concurrent("from array", () => { 65 | test("simple array", async ({ expect }) => { 66 | expect.assertions(1); 67 | 68 | const s = $.fromArray([1, 2, 3]); 69 | 70 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3)]); 71 | }); 72 | 73 | test("array with nullish values", async ({ expect }) => { 74 | expect.assertions(1); 75 | 76 | const s = $.fromArray([1, null, undefined]); 77 | 78 | expect(await s.toArray({ atoms: true })).toEqual([ 79 | $.ok(1), 80 | $.ok(null), 81 | $.ok(undefined), 82 | ]); 83 | }); 84 | 85 | test("don't modify original array", async ({ expect }) => { 86 | expect.assertions(2); 87 | 88 | const array = [1, 2, 3]; 89 | const s = $.fromArray(array); 90 | 91 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3)]); 92 | expect(array).toHaveLength(3); 93 | }); 94 | }); 95 | 96 | describe.concurrent("from callback", () => { 97 | /** 98 | * Sample function that accepts a node-style callback. 99 | * 100 | * @param success - Whether the method should succeed or fail. 101 | * @param cb - Node-style callback to pass error or value to. 102 | */ 103 | function someNodeCallback(success: boolean, cb: NodeCallback) { 104 | if (success) { 105 | cb(null, 123); 106 | } else { 107 | cb("an error"); 108 | } 109 | } 110 | 111 | test("value returned from callback", async ({ expect }) => { 112 | expect.assertions(1); 113 | 114 | const s = $.fromCallback((next) => { 115 | someNodeCallback(true, next); 116 | }); 117 | 118 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(123)]); 119 | }); 120 | 121 | test("error returned from callback", async ({ expect }) => { 122 | expect.assertions(1); 123 | 124 | const s = $.fromCallback((next) => { 125 | someNodeCallback(false, next); 126 | }); 127 | 128 | expect(await s.toArray({ atoms: true })).toEqual([$.error("an error")]); 129 | }); 130 | }); 131 | 132 | describe.concurrent("from next function", () => { 133 | test("simple count up", async ({ expect }) => { 134 | expect.assertions(1); 135 | 136 | let i = 0; 137 | const s = $.fromNext(async () => { 138 | if (i < 4) { 139 | return i++; 140 | } else { 141 | return $.StreamEnd; 142 | } 143 | }); 144 | 145 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(0), $.ok(1), $.ok(2), $.ok(3)]); 146 | }); 147 | 148 | test("next atoms produces atoms", async ({ expect }) => { 149 | expect.assertions(1); 150 | 151 | const atoms = [ 152 | $.ok(0), 153 | $.error("some error"), 154 | $.ok(1), 155 | $.exception("unknown error", []), 156 | ]; 157 | const s = $.fromNext(async () => { 158 | if (atoms.length > 0) { 159 | return atoms.shift(); 160 | } else { 161 | return $.StreamEnd; 162 | } 163 | }); 164 | 165 | expect(await s.toArray({ atoms: true })).toEqual([ 166 | $.ok(0), 167 | $.error("some error"), 168 | $.ok(1), 169 | $.exception("unknown error", []), 170 | ]); 171 | }); 172 | 173 | test("next catches unhandled errors", async ({ expect }) => { 174 | expect.assertions(1); 175 | 176 | let i = 0; 177 | const s = $.fromNext(async () => { 178 | i += 1; 179 | 180 | if (i === 1) { 181 | throw "some error"; 182 | } 183 | 184 | if (i == 2) { 185 | return i; 186 | } 187 | 188 | return $.StreamEnd; 189 | }); 190 | 191 | expect(await s.toArray({ atoms: true })).toEqual([ 192 | $.exception("some error", []), 193 | $.ok(2), 194 | ]); 195 | }); 196 | }); 197 | 198 | describe("fromPusher", () => { 199 | beforeEach(() => { 200 | vi.useFakeTimers(); 201 | }); 202 | afterEach(() => { 203 | vi.useRealTimers(); 204 | }); 205 | 206 | test("push single value", async ({ expect }) => { 207 | expect.assertions(1); 208 | 209 | const { stream, push, done } = $.fromPusher(); 210 | 211 | // Push a value, then immediately complete 212 | push(1); 213 | done(); 214 | 215 | expect(await stream.toArray({ atoms: true })).toEqual([$.ok(1)]); 216 | }); 217 | 218 | test("push single value, delayed done", async ({ expect }) => { 219 | expect.assertions(1); 220 | 221 | const { stream, push, done } = $.fromPusher(); 222 | 223 | const streamPromise = stream.toArray({ atoms: true }); 224 | 225 | // Push a value 226 | push(1); 227 | 228 | // Complete at a later point 229 | setImmediate(() => done()); 230 | 231 | await vi.runAllTimersAsync(); 232 | 233 | expect(await streamPromise).toEqual([$.ok(1)]); 234 | }); 235 | 236 | test("push single value, not done", async ({ expect }) => { 237 | expect.assertions(2); 238 | 239 | const { stream, push } = $.fromPusher(); 240 | 241 | // Push a value. 242 | push(1); 243 | 244 | const spy = vi.fn(); 245 | 246 | // Ensure the stream never completes. 247 | stream 248 | .tap(spy) 249 | .exhaust() 250 | .then(() => expect.fail("promise must not resolve")); 251 | 252 | const WAIT_TIME = 10_000; 253 | 254 | // Block the test until the microtask queue is empty, to make sure there's nothing 255 | // else coming down the stream. 256 | setTimeout(() => { 257 | expect(spy).toBeCalledTimes(1); 258 | expect(spy).toBeCalledWith(1); 259 | }, WAIT_TIME); 260 | 261 | await vi.advanceTimersByTimeAsync(WAIT_TIME); 262 | }); 263 | 264 | test("multiple values", async ({ expect }) => { 265 | expect.assertions(1); 266 | 267 | const { stream, push, done } = $.fromPusher(); 268 | 269 | push(1); 270 | push(2); 271 | push(3); 272 | push(4); 273 | done(); 274 | 275 | expect(await stream.toArray({ atoms: true })).toEqual([ 276 | $.ok(1), 277 | $.ok(2), 278 | $.ok(3), 279 | $.ok(4), 280 | ]); 281 | }); 282 | 283 | test("multiple atoms", async ({ expect }) => { 284 | expect.assertions(1); 285 | 286 | const { stream, push, done } = $.fromPusher(); 287 | 288 | push($.ok(1)); 289 | push($.ok(2)); 290 | push($.error(3)); 291 | push($.exception(4, [])); 292 | done(); 293 | 294 | expect(await stream.toArray({ atoms: true })).toEqual([ 295 | $.ok(1), 296 | $.ok(2), 297 | $.error(3), 298 | $.exception(4, []), 299 | ]); 300 | }); 301 | 302 | test("no items pushed", async ({ expect }) => { 303 | expect.assertions(1); 304 | 305 | const { stream, done } = $.fromPusher(); 306 | 307 | done(); 308 | 309 | expect(await stream.toArray({ atoms: true })).toEqual([]); 310 | }); 311 | 312 | test("push items with delay", async ({ expect }) => { 313 | expect.assertions(9); 314 | 315 | const spy = vi.fn(); 316 | 317 | const { stream, push, done } = $.fromPusher(); 318 | 319 | const streamPromise = stream.map(spy).exhaust(); 320 | 321 | // Some synchronous values 322 | push($.ok(1)); 323 | push($.ok(2)); 324 | 325 | await vi.runAllTimersAsync(); 326 | 327 | // Some timeout values 328 | setTimeout(() => push($.ok(3)), 1000); 329 | setTimeout(() => push($.ok(4)), 2000); 330 | setTimeout(() => push($.ok(5)), 3000); 331 | 332 | // Finish the stream 333 | setTimeout(() => done(), 4000); 334 | 335 | // Initial assertions 336 | expect(spy).toHaveBeenCalledTimes(2); 337 | expect(spy).toHaveBeenNthCalledWith(1, 1); 338 | expect(spy).toHaveBeenNthCalledWith(2, 2); 339 | 340 | // Async assertions 341 | await vi.advanceTimersByTimeAsync(1000); 342 | expect(spy).toHaveBeenCalledTimes(3); 343 | expect(spy).toHaveBeenNthCalledWith(3, 3); 344 | 345 | await vi.advanceTimersByTimeAsync(1000); 346 | expect(spy).toHaveBeenCalledTimes(4); 347 | expect(spy).toHaveBeenNthCalledWith(4, 4); 348 | 349 | await vi.advanceTimersByTimeAsync(1000); 350 | expect(spy).toHaveBeenCalledTimes(5); 351 | expect(spy).toHaveBeenNthCalledWith(5, 5); 352 | 353 | // Run everything else thorugh 354 | await vi.runAllTimersAsync(); 355 | 356 | await streamPromise; 357 | }); 358 | }); 359 | }); 360 | -------------------------------------------------------------------------------- /test/higher-order.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, test, vi } from "vitest"; 2 | import $ from "../src"; 3 | 4 | describe.concurrent("higher order streams", () => { 5 | describe.concurrent("flat map", () => { 6 | test("returning stream", async ({ expect }) => { 7 | expect.assertions(1); 8 | 9 | const s = $.from([1, 2, 3]).flatMap((n) => $.from(new Array(n).fill(n))); 10 | 11 | expect(await s.toArray({ atoms: true })).toEqual([ 12 | $.ok(1), 13 | $.ok(2), 14 | $.ok(2), 15 | $.ok(3), 16 | $.ok(3), 17 | $.ok(3), 18 | ]); 19 | }); 20 | 21 | test("errors already in stream", async ({ expect }) => { 22 | expect.assertions(1); 23 | 24 | const s = $.from([ 25 | $.ok(1), 26 | $.error("known error"), 27 | $.ok(2), 28 | $.exception("bad error", []), 29 | $.ok(3), 30 | ]).flatMap((n) => $.from(new Array(n).fill(n))); 31 | 32 | expect(await s.toArray({ atoms: true })).toEqual([ 33 | $.ok(1), 34 | $.error("known error"), 35 | $.ok(2), 36 | $.ok(2), 37 | $.exception("bad error", []), 38 | $.ok(3), 39 | $.ok(3), 40 | $.ok(3), 41 | ]); 42 | }); 43 | }); 44 | 45 | describe.concurrent("flat tap", () => { 46 | test("simple stream", async ({ expect }) => { 47 | expect.assertions(3); 48 | 49 | const subCallback = vi.fn(); 50 | const callback = vi.fn().mockImplementation((n) => $.of(n * n).tap(subCallback)); 51 | const s = $.from([1, 2, 3, 4]).flatTap(callback); 52 | 53 | // Ensure that the flat tap doesn't alter the emitted stream items 54 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3), $.ok(4)]); 55 | 56 | // Ensure that the flatTap implementation is called once for each item in the stream 57 | expect(callback).toBeCalledTimes(4); 58 | 59 | // Ensure that the stream returned from flatTap is fully executed 60 | expect(subCallback).toBeCalledTimes(4); 61 | }); 62 | 63 | test("simple stream", async ({ expect }) => { 64 | expect.assertions(2); 65 | 66 | const callback = vi.fn().mockImplementation((n) => $.of(n * n)); 67 | const s = $.from([1, 2, 3, 4]).flatTap(callback); 68 | 69 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3), $.ok(4)]); 70 | expect(callback).toBeCalledTimes(4); 71 | }); 72 | }); 73 | 74 | describe.concurrent("otherwise", () => { 75 | test("empty stream", async ({ expect }) => { 76 | expect.assertions(1); 77 | 78 | const s = $.from([]).otherwise($.from([1, 2])); 79 | 80 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2)]); 81 | }); 82 | 83 | test("non-empty stream", async ({ expect }) => { 84 | expect.assertions(1); 85 | 86 | const s = $.from([1]).otherwise($.from([2, 3])); 87 | 88 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1)]); 89 | }); 90 | 91 | test("empty stream with otherwise function", async ({ expect }) => { 92 | expect.assertions(2); 93 | 94 | const otherwise = vi.fn().mockReturnValue($.from([1, 2])); 95 | 96 | const s = $.from([]).otherwise(otherwise); 97 | 98 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2)]); 99 | expect(otherwise).toHaveBeenCalledOnce(); 100 | }); 101 | 102 | test("non-empty stream with otherwise function", async ({ expect }) => { 103 | expect.assertions(2); 104 | 105 | const otherwise = vi.fn().mockReturnValue($.from([2, 3])); 106 | 107 | const s = $.from([1]).otherwise(otherwise); 108 | 109 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1)]); 110 | expect(otherwise).not.toHaveBeenCalled(); 111 | }); 112 | 113 | test("stream with known error", async ({ expect }) => { 114 | expect.assertions(1); 115 | 116 | const s = $.from([$.error("some error")]).otherwise($.from([1])); 117 | 118 | expect(await s.toArray({ atoms: true })).toEqual([$.error("some error")]); 119 | }); 120 | 121 | test("stream with unknown error", async ({ expect }) => { 122 | expect.assertions(1); 123 | 124 | const s = $.from([$.exception("some error", [])]).otherwise($.from([1])); 125 | 126 | expect(await s.toArray({ atoms: true })).toEqual([$.exception("some error", [])]); 127 | }); 128 | 129 | test("stream with never error", async ({ expect }) => { 130 | expect.assertions(1); 131 | 132 | const s = $.from([]).otherwise($.ofError("some error")); 133 | expect(await s.toArray({ atoms: true })).toEqual([$.error("some error")]); 134 | }); 135 | }); 136 | 137 | describe.concurrent("cachedFlatMap", () => { 138 | test("lookup non-repeating strings returning single atom", async ({ expect }) => { 139 | expect.assertions(2); 140 | 141 | const lookup = vi.fn((param: string) => $.of(param)); 142 | 143 | const s = $.from(["a", "b", "c"]).cachedFlatMap(lookup, (v) => v); 144 | 145 | expect(await s.toArray({ atoms: true })).toEqual([$.ok("a"), $.ok("b"), $.ok("c")]); 146 | expect(lookup).toBeCalledTimes(3); 147 | }); 148 | 149 | test("lookup repeating strings returning single atom", async ({ expect }) => { 150 | expect.assertions(2); 151 | 152 | const lookup = vi.fn((param: string) => $.of(param)); 153 | 154 | const s = $.from(["a", "b", "c", "a", "a"]).cachedFlatMap(lookup, (v) => v); 155 | 156 | expect(await s.toArray({ atoms: true })).toEqual([ 157 | $.ok("a"), 158 | $.ok("b"), 159 | $.ok("c"), 160 | $.ok("a"), 161 | $.ok("a"), 162 | ]); 163 | expect(lookup).toBeCalledTimes(3); 164 | }); 165 | 166 | test("lookup repeating numbers returning multiple atoms", async ({ expect }) => { 167 | expect.assertions(2); 168 | 169 | const lookup = vi.fn((n: number) => $.fromArray([n, n * 2, n * 4])); 170 | 171 | const s = $.from([1, 100, 200, 1, 10]).cachedFlatMap(lookup, (v) => v); 172 | 173 | expect(await s.toArray({ atoms: true })).toEqual([ 174 | $.ok(1), 175 | $.ok(2), 176 | $.ok(4), 177 | $.ok(100), 178 | $.ok(200), 179 | $.ok(400), 180 | $.ok(200), 181 | $.ok(400), 182 | $.ok(800), 183 | $.ok(1), 184 | $.ok(2), 185 | $.ok(4), 186 | $.ok(10), 187 | $.ok(20), 188 | $.ok(40), 189 | ]); 190 | expect(lookup).toBeCalledTimes(4); 191 | }); 192 | 193 | test("lookup repeating numbers returning multiple atoms", async ({ expect }) => { 194 | expect.assertions(2); 195 | 196 | const oneHundredDividedBy = vi.fn((n: number) => { 197 | if (n === 0) { 198 | throw "Cannot divide by zero!"; 199 | } 200 | 201 | return $.of(100 / n); 202 | }); 203 | 204 | const s = $.from([5, 0, 50, 5, 5]).cachedFlatMap(oneHundredDividedBy, (v) => v); 205 | 206 | expect(await s.toArray({ atoms: true })).toEqual([ 207 | $.ok(20), 208 | $.exception("Cannot divide by zero!", ["cachedFlatMap"]), 209 | $.ok(2), 210 | $.ok(20), 211 | $.ok(20), 212 | ]); 213 | expect(oneHundredDividedBy).toBeCalledTimes(3); 214 | }); 215 | 216 | test("lookup repeating numbers, including an error, returning multiple atoms", async ({ 217 | expect, 218 | }) => { 219 | expect.assertions(2); 220 | 221 | const lookup = vi.fn((n: number) => $.of(n)); 222 | 223 | const s = $.from([ 224 | $.ok(1), 225 | $.ok(2), 226 | $.error("oh no!"), 227 | $.ok(2), 228 | $.ok(1), 229 | ]).cachedFlatMap(lookup, (v) => v); 230 | 231 | expect(await s.toArray({ atoms: true })).toEqual([ 232 | $.ok(1), 233 | $.ok(2), 234 | $.error("oh no!"), 235 | $.ok(2), 236 | $.ok(1), 237 | ]); 238 | expect(lookup).toBeCalledTimes(2); 239 | }); 240 | }); 241 | 242 | describe.concurrent("flatten", () => { 243 | test("simple nested stream", async ({ expect }) => { 244 | expect.assertions(1); 245 | 246 | const s = $.from([$.from([1, 2]), $.from([3, 4])]).flatten(); 247 | 248 | // We should get all values in order 249 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3), $.ok(4)]); 250 | }); 251 | 252 | test("no effect on already flat stream", async ({ expect }) => { 253 | expect.assertions(1); 254 | 255 | const s = $.from([1, 2, 3, 4]).flatten(); 256 | 257 | // We should get all values in order 258 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3), $.ok(4)]); 259 | }); 260 | 261 | test("correctly flattens mixed depth stream", async ({ expect }) => { 262 | expect.assertions(1); 263 | 264 | const s = $.from([1, 2, $.from([3, 4])]).flatten(); 265 | 266 | // We should get all values in order 267 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3), $.ok(4)]); 268 | }); 269 | 270 | test("maintains errors from flattened stream", async ({ expect }) => { 271 | expect.assertions(1); 272 | 273 | const s = $.from([$.ok($.from([1, 2])), $.error("oh no")]).flatten(); 274 | 275 | // We should get all values in order 276 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.error("oh no")]); 277 | }); 278 | 279 | test("flattening an empty stream", async ({ expect }) => { 280 | expect.assertions(1); 281 | 282 | const s = $.from([]).flatten(); 283 | 284 | expect(await s.toArray({ atoms: true })).toEqual([]); 285 | }); 286 | }); 287 | 288 | describe.concurrent("merge", () => { 289 | test("interleaves atoms", async ({ expect }) => { 290 | expect.assertions(1); 291 | 292 | const fast = $.from([1, 2]); 293 | const slow = $.from([3, 4]).delay(100); 294 | const s = $.from([slow, fast]).merge(); 295 | 296 | // We should expect to see fast then slow despite the order they are emitted from the outer stream 297 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3), $.ok(4)]); 298 | }); 299 | 300 | test("interleaves many atoms, we get them all", async ({ expect }) => { 301 | expect.assertions(1); 302 | 303 | // Create a bunch of streams with different delays 304 | const LEN = 10; 305 | const STREAM_COUNT = 5; 306 | const streams = $.fromArray([10, 20, 30, 50, 15]).map((delay) => 307 | $.fromArray(Array.from({ length: LEN }, (_) => 1)).delay(delay), 308 | ); 309 | 310 | // Merge them and get all the atoms 311 | const atoms = await streams.merge().toArray({ atoms: true }); 312 | 313 | // We should expect to see fast then slow despite the order they are emitted from the outer stream 314 | expect(atoms.length).toEqual(LEN * STREAM_COUNT); 315 | }); 316 | 317 | test("interleaves many atoms from slow outer, we get them all", async ({ expect }) => { 318 | expect.assertions(1); 319 | 320 | // Create a bunch of streams with different delays 321 | const LEN = 10; 322 | const STREAM_COUNT = 5; 323 | const streams = $.fromArray([10, 20, 30, 50, 15]) 324 | .map((delay) => $.fromArray(Array.from({ length: LEN }, (_) => 1)).delay(delay)) 325 | .delay(100); 326 | 327 | // Merge them and get all the atoms 328 | const atoms = await streams.merge().toArray({ atoms: true }); 329 | 330 | // We should expect to see fast then slow despite the order they are emitted from the outer stream 331 | expect(atoms.length).toEqual(LEN * STREAM_COUNT); 332 | }); 333 | 334 | test("no effect on already flat stream", async ({ expect }) => { 335 | expect.assertions(1); 336 | 337 | const s = $.from([1, 2, 3, 4]).merge(); 338 | 339 | // We should get all values in order 340 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(1), $.ok(2), $.ok(3), $.ok(4)]); 341 | }); 342 | }); 343 | }); 344 | -------------------------------------------------------------------------------- /src/stream/base.ts: -------------------------------------------------------------------------------- 1 | import { normalise, type Atom, type MaybeAtom, error, exception } from "../atom"; 2 | import { Stream, WindpipeConsumptionError } from "."; 3 | import { Readable, Writable } from "stream"; 4 | import { createNodeCallback, newSignal, type NodeCallback } from "../util"; 5 | 6 | /** 7 | * Unique type to represent the stream end marker. 8 | */ 9 | export type StreamEnd = typeof StreamBase.StreamEnd; 10 | 11 | export class StreamBase { 12 | protected stream: Readable; 13 | protected stackTrace: string[] = []; 14 | protected traceComplete: boolean = false; 15 | 16 | /** 17 | * Marker for the end of a stream. 18 | */ 19 | static StreamEnd = Symbol.for("STREAM_END"); 20 | 21 | static WindpipeConsumptionError = WindpipeConsumptionError; 22 | 23 | constructor(stream: Readable) { 24 | this.stream = stream; 25 | } 26 | 27 | /** 28 | * Add a layer to the trace object. Returns a copy of the current trace. 29 | */ 30 | protected trace(trace: string) { 31 | if (!this.traceComplete) { 32 | this.stackTrace.push(trace); 33 | this.traceComplete = true; 34 | } 35 | 36 | return this.getTrace(); 37 | } 38 | 39 | /** 40 | * Capture the current trace. Creates a clone of the trace to prevent it being modified. 41 | */ 42 | protected getTrace(): string[] { 43 | return [...this.stackTrace]; 44 | } 45 | 46 | /** 47 | * Create a stream from some kind of stream-like value. This can be an iterable, a promise that 48 | * resolves to some value, or even another readable stream. 49 | * 50 | * @group Creation 51 | */ 52 | static from( 53 | value: 54 | | Promise> 55 | | Iterator> 56 | | AsyncIterator> 57 | | Iterable> 58 | | AsyncIterable> 59 | | Array> 60 | | (() => Promise>), 61 | ): Stream { 62 | if (Array.isArray(value)) { 63 | // Likely an array 64 | return StreamBase.fromArray(value); 65 | } 66 | 67 | if (value instanceof Promise) { 68 | // Likely a promise 69 | return StreamBase.fromPromise(value); 70 | } 71 | 72 | if (Symbol.iterator in value || Symbol.asyncIterator in value) { 73 | // Likely an iterable 74 | return StreamBase.fromIterable(value); 75 | } 76 | 77 | if ("next" in value && typeof value.next === "function") { 78 | // Likely an iterator 79 | return StreamBase.fromIterator(value); 80 | } 81 | 82 | if (typeof value === "function") { 83 | // Likely a `next` function 84 | return StreamBase.fromNext(value); 85 | } 86 | 87 | throw new TypeError("expected a promise, (async) iterator, or (async) iterable"); 88 | } 89 | 90 | /** 91 | * Create a stream from a node-style callback. A node-compatible callback function will be 92 | * passed as the first parameter to the callback of this function. 93 | * 94 | * The first parameter provided to the callback (the `error`) will be emitted as an `Error` 95 | * atom, whilst the second parameter (the `value`) will be emitted as an `Ok` atom. 96 | * 97 | * @example 98 | * $.fromCallback((next) => someAsyncMethod(paramA, paramB, next)); 99 | * 100 | * @group Creation 101 | */ 102 | static fromCallback(cb: (next: NodeCallback) => void): Stream { 103 | // Set up a next function 104 | const [promise, next] = createNodeCallback(); 105 | 106 | // Run the callback 107 | cb(next); 108 | 109 | return StreamBase.fromPromise(promise); 110 | } 111 | 112 | /** 113 | * Create a stream from a promise. The promise will be `await`ed, and the resulting value only 114 | * ever emitted once. 115 | * 116 | * @param promise - The promise to create the stream from. 117 | * 118 | * @group Creation 119 | */ 120 | static fromPromise(promise: Promise>): Stream { 121 | let awaited = false; 122 | 123 | return Stream.fromNext(async () => { 124 | if (!awaited) { 125 | awaited = true; 126 | 127 | return normalise(await promise); 128 | } else { 129 | return StreamBase.StreamEnd; 130 | } 131 | }); 132 | } 133 | 134 | /** 135 | * Create a stream from an iterator. 136 | * 137 | * @param iterator - The iterator that will produce values, which may be an async iterator. 138 | * 139 | * @group Creation 140 | */ 141 | static fromIterator( 142 | iterator: Iterator> | AsyncIterator>, 143 | ): Stream { 144 | return Stream.fromNext(async () => { 145 | const result = iterator.next(); 146 | const { value, done } = result instanceof Promise ? await result : result; 147 | 148 | if (done) { 149 | return StreamBase.StreamEnd; 150 | } else { 151 | return normalise(value); 152 | } 153 | }); 154 | } 155 | 156 | /** 157 | * Create a stream from an iterable. 158 | * 159 | * @param iterable - The iterable that will produce an iterator, which may be an async 160 | * iterator. 161 | * 162 | * @group Creation 163 | */ 164 | static fromIterable( 165 | iterable: Iterable> | AsyncIterable>, 166 | ): Stream { 167 | if (Symbol.iterator in iterable) { 168 | return StreamBase.fromIterator(iterable[Symbol.iterator]()); 169 | } else { 170 | return StreamBase.fromIterator(iterable[Symbol.asyncIterator]()); 171 | } 172 | } 173 | 174 | /** 175 | * Create a stream from an array. 176 | * 177 | * @note The array will be shallow cloned internally, so that the original array won't be 178 | * impacted. 179 | * 180 | * @param array - The array that values will be emitted from. 181 | * 182 | * @group Creation 183 | */ 184 | static fromArray(array: MaybeAtom[]): Stream { 185 | // Clone the array so that shifting elements doesn't impact the original array. 186 | array = [...array]; 187 | 188 | return Stream.fromNext(async () => { 189 | if (array.length === 0) return StreamBase.StreamEnd; 190 | return array.shift()!; 191 | }); 192 | } 193 | 194 | /** 195 | * Create a new stream with the provided atom producer. 196 | * 197 | * @param next - A callback method to produce the next atom. If no atom is available, then 198 | * `StreamEnd` must be returned. 199 | * 200 | * @group Creation 201 | */ 202 | static fromNext(next: () => Promise | StreamEnd>): Stream { 203 | return new Stream( 204 | new Readable({ 205 | objectMode: true, 206 | async read() { 207 | try { 208 | const value = await next(); 209 | 210 | // Promise returned as normal 211 | if (value === StreamBase.StreamEnd) { 212 | this.push(null); 213 | } else { 214 | // @ts-expect-error - The previous `if` statement doesn't cause TS to 215 | // type-narrow out `symbol` 216 | this.push(normalise(value)); 217 | } 218 | } catch (e) { 219 | // Promise was rejected, add as an exception 220 | this.push(exception(e, [])); 221 | } 222 | }, 223 | }), 224 | ); 225 | } 226 | 227 | /** 228 | * Create a new stream, and use the provided `push` and `done` methods to add values to it, and 229 | * complete the stream. 230 | * 231 | * - `push`: Adds the provided value to the stream. 232 | * - `done`: Indicatest that the stream is done, meaning that any future calls to `push` or 233 | * `done` will be ignored. 234 | * 235 | * @group Creation 236 | */ 237 | static fromPusher(): { 238 | stream: Stream; 239 | push: (value: MaybeAtom) => void; 240 | done: () => void; 241 | } { 242 | // Queue of atoms waiting to be pushed. 243 | const queue: MaybeAtom[] = []; 244 | 245 | // Flag to indicate when the `done` method is called. 246 | let done = false; 247 | 248 | // Signal to indicate when some action has taken place. 249 | let signal = newSignal(); 250 | 251 | async function next(retry = 10) { 252 | // If there's something waiting in the queue, immediately produce it. 253 | if (queue.length > 0) { 254 | return queue.shift()!; 255 | } 256 | 257 | // If the stream is complete, immediately return. 258 | if (done) { 259 | return Stream.StreamEnd; 260 | } 261 | 262 | // Prepare a new signal, and wait for it. 263 | signal = newSignal(); 264 | await signal; 265 | 266 | // Protection incase something goes whack with the signal, shouldn't ever be 267 | // encountered. 268 | if (retry === 0) { 269 | console.warn("[windpipe] recursion limit hit whilst waiting for pushed value"); 270 | 271 | return Stream.StreamEnd; 272 | } 273 | 274 | // Recurse and try again. 275 | return next(retry - 1); 276 | } 277 | 278 | return { 279 | stream: Stream.fromNext(next), 280 | push: (value: MaybeAtom) => { 281 | if (done) { 282 | console.error("[windpipe] cannot push after stream is complete"); 283 | return; 284 | } 285 | 286 | queue.push(value); 287 | signal.done(); 288 | }, 289 | done: () => { 290 | done = true; 291 | signal.done(); 292 | }, 293 | }; 294 | } 295 | 296 | /** 297 | * Create a new stream containing a single value. Unless an atom is provided, it will be 298 | * converted to an `ok` atom. 299 | * 300 | * @group Creation 301 | */ 302 | static of(value: MaybeAtom): Stream { 303 | let consumed = false; 304 | return Stream.fromNext(async () => { 305 | if (!consumed) { 306 | consumed = true; 307 | return value; 308 | } else { 309 | return StreamBase.StreamEnd; 310 | } 311 | }); 312 | } 313 | 314 | /** 315 | * Create a new stream containing a single error atom. 316 | * 317 | * @group Creation 318 | */ 319 | static ofError(value: E): Stream { 320 | return this.of(error(value)); 321 | } 322 | 323 | /** 324 | * Create a new stream containing a single exception atom. 325 | * 326 | * @group Creation 327 | */ 328 | static ofException(value: unknown): Stream { 329 | return this.of(exception(value, [])); 330 | } 331 | 332 | /** 333 | * @group Creation 334 | * @deprecated use `ofException` instead 335 | */ 336 | static ofUnknown(value: unknown): Stream { 337 | return this.ofException(value); 338 | } 339 | 340 | /** 341 | * Create a stream and corresponding writable Node stream, where any writes to the writable 342 | * Node stream will be emitted on the returned stream. 343 | */ 344 | static writable(): { stream: Stream; writable: Writable } { 345 | const buffer: (Atom | StreamEnd)[] = []; 346 | const queue: ((value: Atom | StreamEnd) => void)[] = []; 347 | 348 | function enqueue(value: Atom | StreamEnd) { 349 | if (queue.length > 0) { 350 | queue.shift()?.(value); 351 | } else { 352 | buffer.push(value); 353 | } 354 | } 355 | 356 | function dequeue(): Promise | StreamEnd> { 357 | return new Promise((resolve) => { 358 | if (buffer.length > 0) { 359 | resolve(buffer.shift() as Atom | StreamEnd); 360 | } else { 361 | queue.push(resolve); 362 | } 363 | }); 364 | } 365 | 366 | // The writable stream that will receive the transformed value. 367 | const writable = new Writable({ 368 | objectMode: true, 369 | async write(value, _encoding, callback) { 370 | enqueue(value); 371 | 372 | callback(); 373 | }, 374 | async final(callback) { 375 | // Emit a `StreamEnd` to close the stream 376 | enqueue(StreamBase.StreamEnd); 377 | 378 | callback(); 379 | }, 380 | }); 381 | 382 | return { 383 | stream: Stream.fromNext(dequeue), 384 | writable, 385 | }; 386 | } 387 | } 388 | -------------------------------------------------------------------------------- /test/consumption.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, test, vi } from "vitest"; 2 | import $ from "../src"; 3 | import { Readable } from "node:stream"; 4 | 5 | describe.concurrent("stream consumption", () => { 6 | describe.concurrent("serialise", () => { 7 | test("simple number values", async ({ expect }) => { 8 | expect.assertions(1); 9 | 10 | const jsonStream = $.from([1, 2, 3]).serialise(); 11 | const json = await streamToString(jsonStream); 12 | expect(json).toEqual("[1,2,3]"); 13 | }); 14 | 15 | test("null values", async ({ expect }) => { 16 | expect.assertions(1); 17 | 18 | const jsonStream = $.from([1, null, 3]).serialise(); 19 | const json = await streamToString(jsonStream); 20 | expect(json).toEqual("[1,null,3]"); 21 | }); 22 | 23 | test("skip undefined values", async ({ expect }) => { 24 | expect.assertions(1); 25 | 26 | const jsonStream = $.from([1, undefined, 3]).serialise(); 27 | const json = await streamToString(jsonStream); 28 | expect(json).toEqual("[1,3]"); 29 | }); 30 | }); 31 | 32 | describe.concurrent("single", () => { 33 | describe("single ok atom", () => { 34 | test("with no params", async ({ expect }) => { 35 | expect.assertions(1); 36 | 37 | const s = $.of($.ok(1)); 38 | 39 | expect(s.single()).resolves.toEqual(1); 40 | }); 41 | 42 | test("with optional false", async ({ expect }) => { 43 | expect.assertions(1); 44 | 45 | const s = $.of($.ok(1)); 46 | 47 | expect(s.single({ optional: false })).resolves.toEqual(1); 48 | }); 49 | 50 | test("with optional true", async ({ expect }) => { 51 | expect.assertions(1); 52 | 53 | const s = $.of($.ok(1)); 54 | 55 | expect(s.single({ optional: true })).resolves.toEqual(1); 56 | }); 57 | 58 | test("with atom false", async ({ expect }) => { 59 | expect.assertions(1); 60 | 61 | const s = $.of($.ok(1)); 62 | 63 | expect(s.single({ atom: false })).resolves.toEqual(1); 64 | }); 65 | 66 | test("with atom true", async ({ expect }) => { 67 | expect.assertions(1); 68 | 69 | const s = $.of($.ok(1)); 70 | 71 | expect(s.single({ atom: true })).resolves.toEqual($.ok(1)); 72 | }); 73 | }); 74 | 75 | describe("single error atom", () => { 76 | test("with no params", async ({ expect }) => { 77 | expect.assertions(1); 78 | 79 | const s = $.of($.error(1)); 80 | 81 | expect(s.single()).rejects.toEqual(1); 82 | }); 83 | 84 | test("with optional false", async ({ expect }) => { 85 | expect.assertions(1); 86 | 87 | const s = $.of($.error(1)); 88 | 89 | expect(s.single({ optional: false })).rejects.toEqual(1); 90 | }); 91 | 92 | test("with optional true", async ({ expect }) => { 93 | expect.assertions(1); 94 | 95 | const s = $.of($.error(1)); 96 | 97 | expect(s.single({ optional: true })).rejects.toEqual(1); 98 | }); 99 | 100 | test("with atom false", async ({ expect }) => { 101 | expect.assertions(1); 102 | 103 | const s = $.of($.error(1)); 104 | 105 | expect(s.single({ atom: false })).rejects.toEqual(1); 106 | }); 107 | 108 | test("with atom true", async ({ expect }) => { 109 | expect.assertions(1); 110 | 111 | const s = $.of($.error(1)); 112 | 113 | expect(s.single({ atom: true })).resolves.toEqual($.error(1)); 114 | }); 115 | }); 116 | 117 | describe("empty stream", () => { 118 | test("with no params", async ({ expect }) => { 119 | expect.assertions(1); 120 | 121 | const s = $.from([]); 122 | 123 | expect(s.single()).rejects.toThrow($.WindpipeConsumptionError); 124 | }); 125 | 126 | test("with optional false", async ({ expect }) => { 127 | expect.assertions(1); 128 | 129 | const s = $.from([]); 130 | 131 | expect(s.single({ optional: false })).rejects.toThrow($.WindpipeConsumptionError); 132 | }); 133 | 134 | test("with optional true", async ({ expect }) => { 135 | expect.assertions(1); 136 | 137 | const s = $.from([]); 138 | 139 | expect(s.single({ optional: true })).resolves.toEqual(undefined); 140 | }); 141 | 142 | test("with atom false", async ({ expect }) => { 143 | expect.assertions(1); 144 | 145 | const s = $.from([]); 146 | 147 | expect(s.single({ atom: false })).rejects.toThrow($.WindpipeConsumptionError); 148 | }); 149 | 150 | test("with atom true", async ({ expect }) => { 151 | expect.assertions(1); 152 | 153 | const s = $.from([]); 154 | 155 | expect(s.single({ atom: true })).rejects.toThrow($.WindpipeConsumptionError); 156 | }); 157 | }); 158 | 159 | test("single pull", async ({ expect }) => { 160 | expect.assertions(2); 161 | 162 | const fn = vi.fn().mockReturnValue(Promise.resolve(1)); 163 | 164 | const s = $.fromNext(fn); 165 | 166 | expect(s.single()).resolves.toEqual(1); 167 | expect(fn).toBeCalledTimes(1); 168 | }); 169 | }); 170 | 171 | describe.concurrent("toArray", () => { 172 | test("values", async ({ expect }) => { 173 | expect.assertions(1); 174 | 175 | const array = await $.from([1, 2, 3]).toArray(); 176 | 177 | expect(array).toEqual([1, 2, 3]); 178 | }); 179 | 180 | test("values with errors on stream", async ({ expect }) => { 181 | expect.assertions(1); 182 | 183 | const array = await $.from([ 184 | 1, 185 | $.error("known"), 186 | 2, 187 | 3, 188 | $.exception("$.error", []), 189 | ]).toArray(); 190 | 191 | expect(array).toEqual([1, 2, 3]); 192 | }); 193 | 194 | test("values with no items on stream", async ({ expect }) => { 195 | expect.assertions(1); 196 | 197 | const array = await $.from([]).toArray(); 198 | 199 | expect(array).toEqual([]); 200 | }); 201 | 202 | test("atoms", async ({ expect }) => { 203 | expect.assertions(1); 204 | 205 | const array = await $.from([1, 2, 3]).toArray({ atoms: true }); 206 | 207 | expect(array).toEqual([$.ok(1), $.ok(2), $.ok(3)]); 208 | }); 209 | 210 | test("atoms with errors on stream", async ({ expect }) => { 211 | expect.assertions(1); 212 | 213 | const array = await $.from([ 214 | 1, 215 | $.error("known"), 216 | 2, 217 | 3, 218 | $.exception("$.error", []), 219 | ]).toArray({ 220 | atoms: true, 221 | }); 222 | 223 | expect(array).toEqual([ 224 | $.ok(1), 225 | $.error("known"), 226 | $.ok(2), 227 | $.ok(3), 228 | $.exception("$.error", []), 229 | ]); 230 | }); 231 | 232 | test("atoms with no items on stream", async ({ expect }) => { 233 | expect.assertions(1); 234 | 235 | const array = await $.from([]).toArray({ atoms: true }); 236 | 237 | expect(array).toEqual([]); 238 | }); 239 | 240 | test("reject when error on stream", async ({ expect }) => { 241 | expect.assertions(1); 242 | 243 | const arrayPromise = $.from([$.ok(1), $.error("some error")]).toArray({ reject: true }); 244 | 245 | expect(arrayPromise).rejects.toThrow("some error"); 246 | }); 247 | }); 248 | 249 | describe.concurrent("toReadable", () => { 250 | test("object values", async ({ expect }) => { 251 | expect.assertions(2); 252 | 253 | const stream = $.from([1, 2, 3]).toReadable("object"); 254 | expect(stream).to.be.instanceof(Readable); 255 | 256 | const values = await promisifyStream(stream); 257 | expect(values).to.deep.equal([1, 2, 3]); 258 | }); 259 | 260 | test("object atoms", async ({ expect }) => { 261 | expect.assertions(2); 262 | 263 | const stream = $.from([$.ok(1), $.ok(2), $.error(3)]).toReadable("object", { 264 | atoms: true, 265 | }); 266 | expect(stream).to.be.instanceof(Readable); 267 | 268 | const values = await promisifyStream(stream); 269 | expect(values).to.deep.equal([$.ok(1), $.ok(2), $.error(3)]); 270 | }); 271 | 272 | test("null in object stream", async ({ expect }) => { 273 | expect.assertions(2); 274 | 275 | const stream = $.from([1, null, 2, 3]).toReadable("object"); 276 | expect(stream).to.be.instanceof(Readable); 277 | const values = await promisifyStream(stream); 278 | expect(values).to.deep.equal([1, 2, 3]); 279 | }); 280 | 281 | test("raw values", async ({ expect }) => { 282 | expect.assertions(2); 283 | 284 | const stream = $.from(["hello", " ", "world"]).toReadable("raw"); 285 | 286 | expect(stream).to.be.instanceof(Readable); 287 | const values = await promisifyStream(stream); 288 | expect(values.join("")).to.equal("hello world"); 289 | }); 290 | 291 | test("error when using object in raw stream", async ({ expect }) => { 292 | expect.assertions(1); 293 | 294 | // Creating the stream wont panic 295 | const stream = $.from([1]).toReadable("raw"); 296 | 297 | // But reading it will emit an error so this should reject 298 | const streamPromise = promisifyStream(stream); 299 | expect(streamPromise).rejects.toBeTruthy(); 300 | }); 301 | 302 | test("continue emitting items after non-raw item in raw stream", async ({ expect }) => { 303 | expect.assertions(2); 304 | 305 | const stream = $.from([1, "valid"]).toReadable("raw"); 306 | 307 | const { data, errors } = await emptyStream(stream); 308 | 309 | expect(data).toHaveLength(1); 310 | expect(errors).toHaveLength(1); 311 | }); 312 | 313 | test("propagate known errors into raw readable stream", async ({ expect }) => { 314 | expect.assertions(1); 315 | 316 | const stream = promisifyStream( 317 | $.from([$.ok("a"), $.ok("b"), $.error("some error"), $.ok("c")]).toReadable("raw"), 318 | ); 319 | 320 | expect(stream).rejects.toEqual("some error"); 321 | }); 322 | 323 | test("propagate known errors into object readable stream", async ({ expect }) => { 324 | expect.assertions(1); 325 | 326 | const stream = promisifyStream( 327 | $.from([$.ok("a"), $.ok("b"), $.error("some error"), $.ok("c")]).toReadable( 328 | "object", 329 | ), 330 | ); 331 | 332 | expect(stream).rejects.toEqual("some error"); 333 | }); 334 | 335 | test("propagate multiple errors into readable stream", async ({ expect }) => { 336 | expect.assertions(2); 337 | 338 | const stream = $.from([ 339 | $.ok("a"), 340 | $.error("an error"), 341 | $.ok("b"), 342 | $.error("some error"), 343 | $.ok("c"), 344 | ]).toReadable("object"); 345 | 346 | // Monitor the stream 347 | const { data, errors } = await emptyStream(stream); 348 | 349 | expect(errors).toEqual(["an error", "some error"]); 350 | expect(data).toEqual(["a", "b", "c"]); 351 | }); 352 | 353 | test("propagate known and unknown errors", async ({ expect }) => { 354 | expect.assertions(2); 355 | 356 | const stream = $.from([ 357 | $.ok("a"), 358 | $.error("an error"), 359 | $.ok("b"), 360 | $.exception("unknown error", []), 361 | $.ok("c"), 362 | ]).toReadable("object"); 363 | 364 | // Monitor the stream 365 | const { data, errors } = await emptyStream(stream); 366 | 367 | expect(errors).toEqual(["an error", "unknown error"]); 368 | expect(data).toEqual(["a", "b", "c"]); 369 | }); 370 | }); 371 | }); 372 | 373 | function promisifyStream(stream: Readable): Promise { 374 | const data: unknown[] = []; 375 | 376 | return new Promise((resolve, reject) => { 377 | stream.on("data", (value) => data.push(value)); 378 | stream.on("error", reject); 379 | stream.on("end", () => resolve(data)); 380 | }); 381 | } 382 | 383 | async function streamToString(stream: Readable): Promise { 384 | const textDecoder = new TextDecoder(); 385 | const chunks = await stream.toArray(); 386 | return chunks.map((chunk) => textDecoder.decode(chunk)).join(""); 387 | } 388 | 389 | async function emptyStream(stream: Readable): Promise<{ data: unknown[]; errors: unknown[] }> { 390 | const errors: unknown[] = []; 391 | const data: unknown[] = []; 392 | 393 | await new Promise((resolve) => { 394 | stream.on("data", (d) => data.push(d)); 395 | stream.on("error", (e: string) => errors.push(e)); 396 | stream.on("end", () => resolve()); 397 | }); 398 | 399 | return { data, errors }; 400 | } 401 | -------------------------------------------------------------------------------- /src/stream/higher-order.ts: -------------------------------------------------------------------------------- 1 | import { Stream } from "."; 2 | import { isError, isOk, type Atom, isException } from "../atom"; 3 | import { run } from "../handler"; 4 | import { type CallbackOrStream, type MaybePromise, exhaust } from "../util"; 5 | import { StreamTransforms } from "./transforms"; 6 | 7 | function accept(value: T): { accept: T } { 8 | return { accept: value }; 9 | } 10 | 11 | function reject(value: T): { reject: T } { 12 | return { reject: value }; 13 | } 14 | 15 | type FilterResult = { accept: A } | { reject: R }; 16 | 17 | type IfNever = [T] extends [never] ? A : B; 18 | 19 | export class HigherOrderStream extends StreamTransforms { 20 | /** 21 | * Base implementation of `flat*` operations. In general, all of these methods will filter over 22 | * the type of atom, run some stream-producing callback with it, and then produce a new 23 | * generator to expand into the stream. 24 | * 25 | * @template A - The accepted type from the filter. Allows for type narrowing from `Atom` 26 | * into `AtomOk`, etc. 27 | * @template U - The `ok` type of the produced stream. 28 | * @template F - The `error` type of the produced stream. 29 | * @template CT - The `ok` type of the stream produced from the callback. 30 | * @template CE - The `error` type of the stream produced from the callback. 31 | */ 32 | private flatOp( 33 | filter: (atom: Atom) => FilterResult>, 34 | cb: (atom: A) => MaybePromise>, 35 | process: (atom: Atom, stream: Stream) => AsyncGenerator>, 36 | ): Stream { 37 | const trace = this.trace("flatOp"); 38 | 39 | return this.consume(async function* (it) { 40 | for await (const atom of it) { 41 | const result = filter(atom); 42 | 43 | if ("reject" in result) { 44 | yield result.reject; 45 | continue; 46 | } 47 | 48 | // Run the flat map handler 49 | const streamAtom = await run(() => cb(result.accept), trace); 50 | 51 | // If an error was emitted whilst initialising the new stream, return it 52 | if (!isOk(streamAtom)) { 53 | yield streamAtom; 54 | continue; 55 | } 56 | 57 | // Otherwise, consume the iterator 58 | yield* process(atom, streamAtom.value); 59 | } 60 | }); 61 | } 62 | 63 | /** 64 | * Internal helper for implementing the other `flatMap` methods. 65 | */ 66 | private flatMapAtom( 67 | filter: (atom: Atom) => FilterResult>, 68 | cb: (atom: A) => MaybePromise>, 69 | ): Stream { 70 | this.trace("flatMapAll"); 71 | 72 | return this.flatOp(filter, cb, async function* (_atom, stream) { 73 | yield* stream; 74 | }); 75 | } 76 | 77 | /** 78 | * Map over each value in the stream, produce a stream from it, and flatten all the value 79 | * streams together 80 | * 81 | * @group Higher Order 82 | */ 83 | flatMap(cb: (value: T) => MaybePromise>): Stream { 84 | this.trace("flatMap"); 85 | 86 | return this.flatMapAtom( 87 | (atom) => (isOk(atom) ? accept(atom) : reject(atom)), 88 | (atom) => { 89 | return cb(atom.value); 90 | }, 91 | ); 92 | } 93 | 94 | /** 95 | * Map over each error in the stream, produce a stream from it, and flatten all the value 96 | * streams together. 97 | * 98 | * @group Higher Order 99 | */ 100 | flatMapError(cb: (value: E) => MaybePromise>): Stream { 101 | this.trace("flatMapError"); 102 | 103 | return this.flatMapAtom( 104 | (atom) => (isError(atom) ? accept(atom) : reject(atom)), 105 | (atom) => { 106 | return cb(atom.value); 107 | }, 108 | ); 109 | } 110 | 111 | /** 112 | * Maps over each exception in the stream, producing a new stream from it, and flatten all 113 | * the value streams together. 114 | * 115 | * @group Higher Order 116 | */ 117 | flatMapException( 118 | cb: (value: unknown, trace: string[]) => MaybePromise>, 119 | ): Stream { 120 | const trace = this.trace("flatMapException"); 121 | 122 | return this.flatMapAtom( 123 | (atom) => (isException(atom) ? accept(atom) : reject(atom)), 124 | (atom) => { 125 | return cb(atom.value, trace); 126 | }, 127 | ); 128 | } 129 | 130 | /** 131 | * @group Higher Order 132 | * @deprecated use `flatMapException` instead 133 | */ 134 | flatMapUnknown( 135 | cb: (value: unknown, trace: string[]) => MaybePromise>, 136 | ): Stream { 137 | return this.flatMapException(cb); 138 | } 139 | 140 | /** 141 | * Map over each value in the stream, produce a stream from it, cache the resultant stream 142 | * and flatten all the value streams together 143 | * 144 | * @group Higher Order 145 | */ 146 | cachedFlatMap( 147 | cb: (value: T) => MaybePromise>, 148 | keyFn: (value: T) => string | number | symbol, 149 | ): Stream { 150 | const trace = this.trace("cachedFlatMap"); 151 | 152 | return this.consume(async function* (it) { 153 | const cache = new Map[]>(); 154 | 155 | for await (const atom of it) { 156 | if (!isOk(atom)) { 157 | yield atom; 158 | continue; 159 | } 160 | 161 | const key = keyFn(atom.value); 162 | const cachedValues = cache.get(key); 163 | 164 | if (cachedValues !== undefined) { 165 | yield* cachedValues; 166 | continue; 167 | } 168 | 169 | // Run the flat map handler 170 | const streamAtom = await run(() => cb(atom.value), trace); 171 | 172 | // If an error was emitted whilst initialising the new stream, return it 173 | if (!isOk(streamAtom)) { 174 | yield streamAtom; 175 | continue; 176 | } 177 | 178 | // Otherwise, consume the iterator 179 | const values = await streamAtom.value.toArray({ atoms: true }); 180 | 181 | cache.set(key, values); 182 | 183 | yield* values; 184 | } 185 | }); 186 | } 187 | 188 | /** 189 | * Produce a new stream from the stream that has any nested streams flattened 190 | * 191 | * @note Any atoms that are not nested streams are emitted as-is 192 | * @group Higher Order 193 | */ 194 | flatten(): T extends Stream ? Stream : Stream { 195 | this.trace("flatten"); 196 | 197 | return this.consume(async function* (it) { 198 | for await (const atom of it) { 199 | // Yield errors/unkowns directly 200 | if (!isOk(atom)) { 201 | yield atom; 202 | continue; 203 | } 204 | 205 | // Yield each atom within nested streams 206 | if (atom.value instanceof Stream) { 207 | yield* atom.value; 208 | } else { 209 | yield atom; 210 | } 211 | } 212 | }) as T extends Stream ? Stream : Stream; 213 | } 214 | 215 | /** 216 | * Produce a new stream from the stream that has any nested streams merged together, emitting their 217 | * atoms as they are emitted. 218 | * 219 | * @note Any atoms that are not nested streams are emitted as-is 220 | * @group Higher Order 221 | */ 222 | merge(): T extends Stream ? Stream : Stream { 223 | this.trace("merge"); 224 | 225 | return this.consume(async function* (it) { 226 | // Get an iterator for the stream of streams 227 | const outer = it[Symbol.asyncIterator](); 228 | 229 | // Keep track of the pending value from the outer stream 230 | let outerPending = null; 231 | 232 | // Are we overall done with consuming the outer stream? 233 | let outerExhausted = false; 234 | 235 | // Keep a map from inner iterators to their pending next() promise 236 | // we can race them to get the next value but we only remove them once resolved so we dont drop any values 237 | const innerPending = new Map< 238 | AsyncIterator>, 239 | Promise>> 240 | >(); 241 | 242 | // While we either have outer atoms to pull yet or we are waiting for inner streams to exhaust 243 | // keep looping and racing them 244 | while (!outerExhausted || innerPending.size > 0) { 245 | // We could race either a new nested stream from outer or a new atom from an inner 246 | type OuterResult = IteratorResult> & { type: "outer" }; 247 | type InnerResult = IteratorResult> & { 248 | type: "inner"; 249 | iterator: AsyncIterator>; 250 | }; 251 | type Source = Promise; 252 | const sources: Array = []; 253 | 254 | // Is there still nested streams in outer? 255 | if (!outerExhausted) { 256 | // Make sure we have something to await 257 | if (outerPending === null) { 258 | outerPending = outer.next(); 259 | } 260 | 261 | // and use it as a source 262 | sources.push(outerPending.then((r) => ({ ...r, type: "outer" }))); 263 | } 264 | 265 | // Are there active inner streams still? 266 | // For each, we want to race their pending value 267 | for (const [iterator, nextPromise] of innerPending) { 268 | sources.push(nextPromise.then((r) => ({ ...r, iterator, type: "inner" }))); 269 | } 270 | 271 | // Anything left to wait on? 272 | // (NOTE: unlikely to trigger since this is effectively the "while" condition) 273 | if (sources.length === 0) { 274 | break; 275 | } 276 | 277 | // Okay now race all the sources 278 | const winner = await Promise.race(sources); 279 | // Is this an inner result? We add an `iterator` key to those 280 | if (winner.type === "inner") { 281 | if (winner.done) { 282 | // In practice, seems like no values are included on these results, so ignoring that 283 | innerPending.delete(winner.iterator); 284 | } else { 285 | // Yield the value and immediately call .next() to get the next one 286 | yield winner.value; 287 | innerPending.set(winner.iterator, winner.iterator.next()); 288 | } 289 | 290 | continue; 291 | } 292 | 293 | // Is this an outer result? 294 | if (winner.type === "outer") { 295 | // Unset the pending value 296 | outerPending = null; 297 | 298 | // Handle the result 299 | if (winner.done) { 300 | outerExhausted = true; 301 | } else { 302 | const atom = winner.value; 303 | if (!isOk(atom)) { 304 | // If an error, just emit it 305 | yield atom; 306 | } else if (atom.value instanceof Stream) { 307 | // if a value and its a stream, start tracking it 308 | const iter = atom.value[Symbol.asyncIterator](); 309 | innerPending.set(iter, iter.next()); 310 | } else { 311 | // If a value and not a stream, just emit it 312 | yield atom; 313 | } 314 | } 315 | continue; 316 | } 317 | 318 | throw new Error("invalid result. unreachable"); 319 | } 320 | }) as T extends Stream ? Stream : Stream; 321 | } 322 | 323 | /** 324 | * Base implementation of the `flatTap` operations. 325 | */ 326 | private flatTapAtom( 327 | filter: (atom: Atom) => FilterResult>, 328 | cb: (atom: A) => MaybePromise>, 329 | ): Stream { 330 | this.trace("flatTapAtom"); 331 | 332 | return this.flatOp(filter, cb, async function* (atom, stream) { 333 | await exhaust(stream); 334 | 335 | yield atom; 336 | }); 337 | } 338 | 339 | /** 340 | * Produce a stream for each value in the stream. The resulting stream will be emptied, 341 | * however the resulting values will just be dropped. This is analogous to an asynchronous 342 | * side effect. 343 | * 344 | * @group Higher Order 345 | */ 346 | flatTap(cb: (value: T) => MaybePromise>): Stream { 347 | this.trace("flatTap"); 348 | 349 | return this.flatTapAtom( 350 | (atom) => (isOk(atom) ? accept(atom) : reject(atom)), 351 | (atom) => cb(atom.value), 352 | ); 353 | } 354 | 355 | /** 356 | * Emit items from provided stream if this stream is completely empty. 357 | * 358 | * @note If there are any errors or exceptions on the stream, then the new stream won't be 359 | * consumed. 360 | * 361 | * @group Higher Order 362 | */ 363 | otherwise>(cbOrStream: CallbackOrStream): Stream { 364 | return this.consume(async function* (it) { 365 | // Count the items being emitted from the iterator 366 | let count = 0; 367 | for await (const atom of it) { 368 | count += 1; 369 | yield atom as Atom; 370 | } 371 | 372 | // If nothing was emitted, then create the stream and emit it 373 | if (count === 0) { 374 | if (typeof cbOrStream === "function") { 375 | yield* cbOrStream(); 376 | } else { 377 | yield* cbOrStream; 378 | } 379 | } 380 | }); 381 | } 382 | 383 | /** 384 | * Consume the entire stream, and completely replace it with a new stream. This will remove 385 | * any errors and exceptions currently on the stream. 386 | * 387 | * Equivalent to: 388 | * 389 | * ``` 390 | * stream 391 | * .filter(() => false) 392 | * .otherwise(newStream); 393 | * ``` 394 | * ` 395 | * 396 | * @group Higher Order 397 | */ 398 | replaceWith(cbOrStream: CallbackOrStream): Stream { 399 | return this.consume(async function* (it) { 400 | // Consume all the items in the stream 401 | await exhaust(it); 402 | 403 | // Replace with the user stream 404 | if (typeof cbOrStream === "function") { 405 | yield* cbOrStream(); 406 | } else { 407 | yield* cbOrStream; 408 | } 409 | }); 410 | } 411 | } 412 | -------------------------------------------------------------------------------- /src/stream/transforms.ts: -------------------------------------------------------------------------------- 1 | import { Stream } from "."; 2 | import { isOk, isException, type MaybeAtom, type Atom, isError, exception, ok } from "../atom"; 3 | import { handler } from "../handler"; 4 | import { StreamConsumption } from "./consumption"; 5 | import { Readable } from "stream"; 6 | import util from "node:util"; 7 | import { type Truthy, type MaybePromise, newSignal } from "../util"; 8 | 9 | export class StreamTransforms extends StreamConsumption { 10 | /** 11 | * Consume the stream atoms, emitting new atoms from the generator. 12 | * 13 | * @group Transform 14 | */ 15 | consume( 16 | generator: (it: AsyncIterable>) => AsyncGenerator>, 17 | ): Stream { 18 | const trace = this.trace("consume"); 19 | 20 | const stream = new Stream(Readable.from(generator(this.stream))); 21 | stream.stackTrace = trace; 22 | 23 | return stream; 24 | } 25 | 26 | /** 27 | * Collect the values of the stream atoms into an array then return a stream which emits that array 28 | * 29 | * @note non-ok atoms are emitted as-is, the collected array is always emitted last 30 | * @note empty streams will emit an empty array 31 | * @group Transform 32 | */ 33 | collect(): Stream { 34 | this.trace("collect"); 35 | 36 | return this.consume(async function* (it) { 37 | const values: T[] = []; 38 | for await (const atom of it) { 39 | if (isOk(atom)) { 40 | values.push(atom.value); 41 | } else { 42 | yield atom; 43 | } 44 | } 45 | yield ok(values); 46 | }); 47 | } 48 | 49 | /** 50 | * Map over each value in the stream. 51 | * 52 | * @group Transform 53 | */ 54 | map(cb: (value: T) => MaybePromise>): Stream { 55 | const trace = this.trace("map"); 56 | 57 | return this.consume(async function* (it) { 58 | for await (const atom of it) { 59 | if (isOk(atom)) { 60 | yield await handler(() => cb(atom.value), trace); 61 | } else { 62 | yield atom; 63 | } 64 | } 65 | }); 66 | } 67 | 68 | /** 69 | * Map over each error in the stream. 70 | * 71 | * @group Transform 72 | */ 73 | mapError(cb: (error: E) => MaybePromise>): Stream { 74 | const trace = this.trace("mapError"); 75 | 76 | return this.consume(async function* (it) { 77 | for await (const atom of it) { 78 | if (isError(atom)) { 79 | yield await handler(() => cb(atom.value), trace); 80 | } else { 81 | yield atom; 82 | } 83 | } 84 | }); 85 | } 86 | 87 | /** 88 | * Map over each exception in the stream. 89 | * 90 | * @group Transform 91 | */ 92 | mapException(cb: (error: unknown) => MaybePromise>): Stream { 93 | const trace = this.trace("mapException"); 94 | 95 | return this.consume(async function* (it) { 96 | for await (const atom of it) { 97 | if (isException(atom)) { 98 | yield await handler(() => cb(atom.value), trace); 99 | } else { 100 | yield atom; 101 | } 102 | } 103 | }); 104 | } 105 | 106 | /** 107 | * @group Transform 108 | * @deprecated use `mapException` instead 109 | */ 110 | mapUnknown(cb: (error: unknown) => MaybePromise>): Stream { 111 | return this.mapException(cb); 112 | } 113 | 114 | /** 115 | * Run a callback for each value in the stream, ideal for side effects on stream items. 116 | * 117 | * @group Transform 118 | */ 119 | tap(cb: (value: T) => unknown): Stream { 120 | this.trace("tap"); 121 | 122 | return this.map((value) => { 123 | try { 124 | cb(value); 125 | } catch (e) { 126 | console.error("Error thrown in tap operation:", e); 127 | } 128 | 129 | return value; 130 | }); 131 | } 132 | 133 | /** 134 | * Inspect every atom that is emitted through the stream. 135 | * 136 | * @group Transform 137 | */ 138 | inspect(): Stream { 139 | this.trace("inspect"); 140 | 141 | return this.consume(async function* (it) { 142 | for await (const atom of it) { 143 | console.log(util.inspect(atom, false, Infinity, true)); 144 | 145 | yield atom; 146 | } 147 | }); 148 | } 149 | 150 | /** 151 | * Filter over each value in the stream. 152 | * 153 | * @group Transform 154 | */ 155 | filter(condition: (value: T) => MaybePromise): Stream { 156 | const trace = this.trace("filter"); 157 | 158 | return this.consume(async function* (it) { 159 | for await (const atom of it) { 160 | // Re-emit any existing errors onto the stream 161 | if (!isOk(atom)) { 162 | yield atom; 163 | } 164 | 165 | // Run the filter condition 166 | const filter = await handler(() => condition(atom.value as T), trace); 167 | 168 | if (isOk(filter) && filter.value) { 169 | yield atom; 170 | } else if (!isOk(filter)) { 171 | // Non-value returned from the filter 172 | const error: Error & { detail?: unknown } = new Error( 173 | "non-ok value returned from filter condition", 174 | ); 175 | error.detail = filter; 176 | yield exception(error, trace); 177 | } 178 | } 179 | }); 180 | } 181 | 182 | /** 183 | * Remove falsey values from the stream. 184 | * 185 | * This is equivalent to doing `.filter((value) => value)`. 186 | * 187 | * @group Transform 188 | */ 189 | compact(): Stream, E> { 190 | this.trace("compact"); 191 | 192 | return this.filter((value) => { 193 | if (value) { 194 | return true; 195 | } else { 196 | return false; 197 | } 198 | }) as Stream, E>; 199 | } 200 | 201 | /** 202 | * Operate on each item in the stream, reducing it into a single value. The resulting value is 203 | * returned in its own stream. 204 | * 205 | * @group Transforms 206 | */ 207 | reduce(cb: (memo: U, value: T) => MaybePromise>, memo: U): Stream { 208 | const trace = this.trace("reduce"); 209 | 210 | return this.consume(async function* (it) { 211 | for await (const atom of it) { 212 | if (isOk(atom)) { 213 | // Run the reducer 214 | const value = await handler(() => cb(memo, atom.value), trace); 215 | 216 | if (isOk(value)) { 217 | memo = value.value; 218 | } else { 219 | // Reducer produced a non-ok atom, emit it and continue reducing 220 | yield value; 221 | } 222 | } else { 223 | yield atom as Atom; 224 | } 225 | } 226 | 227 | yield ok(memo); 228 | }); 229 | } 230 | 231 | /** 232 | * Return a stream containing the first `n` values. If `options.atoms` is `true`, then the 233 | * first `n` atoms rather than values will be emitted. 234 | * 235 | * @param options.atoms - If enabled, first `n` atoms will be counted, otherwise values. 236 | * 237 | * @group Transform 238 | */ 239 | take(n: number, options?: { atoms?: boolean }): Stream { 240 | this.trace("take"); 241 | 242 | return this.consume(async function* (it) { 243 | if (n <= 0) { 244 | return; 245 | } 246 | 247 | let i = 0; 248 | 249 | for await (const atom of it) { 250 | if (isOk(atom) || options?.atoms === true) { 251 | yield atom; 252 | } 253 | 254 | i++; 255 | 256 | if (i >= n) { 257 | break; 258 | } 259 | } 260 | }); 261 | } 262 | 263 | /** 264 | * Drop the first `n` items from the stream. 265 | * 266 | * @group Transform 267 | */ 268 | drop(n: number, options?: { atoms?: boolean }): Stream { 269 | this.trace("drop"); 270 | 271 | return this.consume(async function* (it) { 272 | let i = 0; 273 | 274 | for await (const atom of it) { 275 | // Skip this atom if only values are desired 276 | if (!options?.atoms && !isOk(atom)) { 277 | continue; 278 | } 279 | 280 | // Only yield if we're beyond the first n items 281 | if (i >= n) { 282 | yield atom; 283 | } 284 | 285 | i++; 286 | } 287 | }); 288 | } 289 | 290 | /** 291 | * Delay emitting each value on the stream by `ms`. 292 | * 293 | * @group Transform 294 | */ 295 | delay(ms: number): Stream { 296 | this.trace("delay"); 297 | 298 | return this.consume(async function* (it) { 299 | for await (const atom of it) { 300 | await new Promise((resolve) => setTimeout(resolve, ms)); 301 | 302 | yield atom; 303 | } 304 | }); 305 | } 306 | 307 | /** 308 | * Map over each value in the stream, and apply some callback to it. This differs from `map` as 309 | * the upstream is continually pulled, regardless if the downstream is ready for it and whether 310 | * the current iteration has complete. 311 | * 312 | * # Example 313 | * 314 | * Approximate timelines are shown below. It isn't 100% correct for what's actually happening, 315 | * but gives the right idea. 316 | * 317 | * - `p`: producer (upstream) 318 | * - `m`: map operation (asynchronous) 319 | * 320 | * ## Map 321 | * 322 | * ``` 323 | * 0ms 10ms 20ms 30ms 324 | * |---------|---------|---------|---------| 325 | * |<-- p -->|<-- m -->| | | 326 | * | | |<-- p -->|<-- m -->| 327 | * ``` 328 | * 329 | * ## Buffered Map 330 | * 331 | * ``` 332 | * 0ms 10ms 20ms 30ms 333 | * |---------|---------|---------|---------| 334 | * |<-- p -->|<-- m -->| | | 335 | * | |<-- p -->|<-- m -->| | 336 | * | | |<-- p -->|<-- m -->| 337 | * ``` 338 | * 339 | * @group Transform 340 | */ 341 | bufferedMap( 342 | cb: (value: T) => MaybePromise>, 343 | options?: { delay?: number; maxBufferSize?: number; maxBufferPeriod?: number }, 344 | ): Stream { 345 | const trace = this.trace("bufferedMap"); 346 | 347 | let itemReadySignal = newSignal(); 348 | let bufferPulledSignal = newSignal(); 349 | let lastPull = Date.now(); 350 | let end = false; 351 | 352 | // Buffer all of the pending map results 353 | const buffer: Promise>[] = []; 354 | 355 | // Continually fill up the buffer 356 | (async () => { 357 | // Continually pull items from the stream 358 | for await (const atom of this) { 359 | if (isOk(atom)) { 360 | // Pass the value through the callback 361 | buffer.push(handler(() => cb(atom.value), trace)); 362 | } else { 363 | // Add the atom directly to the buffer 364 | buffer.push(Promise.resolve(atom)); 365 | } 366 | 367 | // Trigger and rotate the signal, so that any pending stream consumers can continue 368 | itemReadySignal.done(); 369 | itemReadySignal = newSignal(); 370 | 371 | // Optionally delay re-polling, to prevent spamming the upstream 372 | if (options?.delay) { 373 | await new Promise((resolve) => setTimeout(resolve, options.delay)); 374 | } 375 | 376 | // Optionally halt if the buffer is full or if there hasn't been a pull in a while 377 | while ( 378 | (options?.maxBufferSize && buffer.length >= options?.maxBufferSize) || 379 | (options?.maxBufferPeriod && Date.now() - lastPull >= options?.maxBufferPeriod) 380 | ) { 381 | await bufferPulledSignal; 382 | } 383 | } 384 | 385 | // Once the async iterator is exhausted, indicate that there will be no more items 386 | end = true; 387 | itemReadySignal.done(); 388 | })(); 389 | 390 | // Create the resulting stream by pulling a value from the buffer whenever one is requested 391 | return Stream.fromNext(async () => { 392 | let value: Promise> | undefined = undefined; 393 | 394 | while (value === undefined) { 395 | if (buffer.length === 0) { 396 | if (end) { 397 | return Stream.StreamEnd; 398 | } 399 | 400 | await itemReadySignal; 401 | } 402 | 403 | value = buffer.shift(); 404 | } 405 | 406 | const v = await value; 407 | 408 | lastPull = Date.now(); 409 | bufferPulledSignal.done(); 410 | bufferPulledSignal = newSignal(); 411 | 412 | return v; 413 | }); 414 | } 415 | 416 | /** 417 | * Batch items together on the stream and emit them as an array. The batch can be over some 418 | * size, or can be over a period of time. 419 | * 420 | * @param options.n - Batch up to `n` items (activates batching by count) 421 | * @param options.yeildRemaining - If the end of the stream is encountered and there are less 422 | * than `n` items buffered, yield them anyway 423 | * @param options.timeout - Batch for `timeout` milliseconds (activates batching by timeout) 424 | * @param options.yieldEmpty - If `timeout` is reached and no items have been emitted on the 425 | * stream, still emit an empty array. 426 | */ 427 | batch(options: { 428 | n?: number; 429 | timeout?: number; 430 | yieldRemaining?: boolean; 431 | yieldEmpty?: boolean; 432 | byBucket?: (value: T) => string | number; 433 | }): Stream { 434 | return this.consume(async function* (it) { 435 | const atoms = it[Symbol.asyncIterator](); 436 | 437 | /** 438 | * Create a promise that will resolve after the specified timeout period. If no timeout 439 | * is provided then it will never resolve. 440 | */ 441 | function newHeartbeat() { 442 | return new Promise<"timeout">((resolve) => { 443 | if ("timeout" in options) { 444 | setTimeout(() => { 445 | resolve("timeout" as const); 446 | }, options.timeout); 447 | } 448 | }); 449 | } 450 | 451 | // Define the promises outside of the loop, so we can re-use them without loosing the 452 | // previous instance. 453 | let nextAtom = atoms.next(); 454 | let heartbeat = newHeartbeat(); 455 | 456 | const batches: Record = {}; 457 | let totalBatchSize = 0; 458 | 459 | const batchFilter = options?.byBucket ?? (() => "default"); 460 | 461 | let end = false; 462 | 463 | while (!end) { 464 | if (totalBatchSize > 1000) { 465 | // Producer may not be I/O bound, yield to the event loop to give other things a 466 | // chance to run. 467 | await new Promise((resolve) => { 468 | setTimeout(resolve, 0); 469 | }); 470 | } 471 | 472 | // See if an atom is ready, or if we time out 473 | const result = await Promise.race([heartbeat, nextAtom]); 474 | 475 | if (typeof result === "object" && "value" in result) { 476 | // Atom was ready, process it (TypeScript types are incorrect) 477 | const atom = result.value as Atom | undefined; 478 | 479 | if (atom) { 480 | if (isOk(atom)) { 481 | const key = batchFilter(atom.value); 482 | 483 | let batch = batches[key]; 484 | 485 | // Add the bucket if it doesn't exist 486 | if (!batch) { 487 | batch = batches[key] = []; 488 | } 489 | 490 | batch.push(atom.value); 491 | totalBatchSize += 1; 492 | 493 | // Batch was modified, see if it's ready to emit 494 | if (options?.n && batch.length >= options.n) { 495 | yield ok(batch.splice(0, options.n)); 496 | 497 | totalBatchSize -= options.n; 498 | } 499 | } else { 500 | // Immediately yield any errors 501 | yield atom; 502 | } 503 | } 504 | 505 | // Set up the next promise 506 | nextAtom = atoms.next(); 507 | 508 | // Iterator is complete, stop the loop 509 | if (result.done) { 510 | end = true; 511 | } 512 | } 513 | 514 | if (result === "timeout" && "timeout" in options) { 515 | // Work out which batches are ready 516 | const ready = Object.values(batches).filter( 517 | (batch) => batch.length >= (options?.n ?? 1) || options?.yieldRemaining, 518 | ); 519 | 520 | if (ready.reduce((total, batch) => total + batch.length, 0) === 0) { 521 | if (options?.yieldEmpty) { 522 | // Only yield an empty batch if there are absolutely no items ready to 523 | // be yielded and if the configuration allows it 524 | yield ok([]); 525 | } 526 | } else { 527 | for (const batch of ready) { 528 | const items = batch.splice(0, options?.n ?? batch.length); 529 | yield ok(items); 530 | totalBatchSize -= items.length; 531 | } 532 | } 533 | } 534 | 535 | if (result === "timeout") { 536 | heartbeat = newHeartbeat(); 537 | } 538 | } 539 | 540 | if ("timeout" in options && (totalBatchSize > 0 || options.yieldEmpty)) { 541 | // Wait for heartbeat to finish 542 | await heartbeat; 543 | 544 | if (totalBatchSize > 0) { 545 | // Yield the rest of the batches 546 | for (const batch of Object.values(batches)) { 547 | if (batch.length === 0) { 548 | continue; 549 | } 550 | 551 | yield ok(batch); 552 | } 553 | } else if (options.yieldEmpty) { 554 | // Yield the final empty batch 555 | yield ok([]); 556 | } 557 | } else if (options?.n && totalBatchSize > 0) { 558 | for (const batch of Object.values(batches)) { 559 | while (batch.length >= options.n) { 560 | yield ok(batch.splice(0, options.n)); 561 | } 562 | 563 | if (batch.length > 0 && options.yieldRemaining) { 564 | yield ok(batch); 565 | } 566 | } 567 | } 568 | }); 569 | } 570 | 571 | /** 572 | * Run the provided callback after the first atom passes through the stream. 573 | * 574 | * @param callback - Callback to run. 575 | * @param [options = {}] 576 | * @param [options.atom = true] - Run on any atom (true by default). Will only run after first 577 | * `ok` atom if false. 578 | */ 579 | onFirst(callback: () => void, options: { atom?: boolean } = {}): Stream { 580 | this.trace("onFirst"); 581 | 582 | return this.consume(async function* (it) { 583 | let first = false; 584 | 585 | for await (const atom of it) { 586 | yield atom; 587 | 588 | if (!first && (options?.atom !== false || Stream.isOk(atom))) { 589 | callback(); 590 | first = true; 591 | } 592 | } 593 | }); 594 | } 595 | 596 | /** 597 | * Run the provided callback after the last atom passes through the stream. 598 | * 599 | * @param callback - Callback to run. 600 | * @param [options = {}] 601 | * @param [options.atom = true] - Run on any atom (true by default). Will only run after last 602 | * `ok` atom if false. 603 | */ 604 | onLast(callback: () => void, options: { atom?: boolean } = {}): Stream { 605 | this.trace("onLast"); 606 | 607 | return this.consume(async function* (it) { 608 | let emitted = false; 609 | 610 | for await (const atom of it) { 611 | yield atom; 612 | 613 | if (options?.atom !== false || Stream.isOk(atom)) { 614 | emitted = true; 615 | } 616 | } 617 | 618 | if (emitted) { 619 | callback(); 620 | } 621 | }); 622 | } 623 | } 624 | -------------------------------------------------------------------------------- /test/transforms.test.ts: -------------------------------------------------------------------------------- 1 | import { afterEach, beforeEach, describe, test, vi, type ExpectStatic } from "vitest"; 2 | import $ from "../src"; 3 | 4 | describe("stream transforms", () => { 5 | describe("map", () => { 6 | test("synchronous value", async ({ expect }) => { 7 | expect.assertions(1); 8 | 9 | const s = $.from([1, 2, 3]).map((n) => n * 10); 10 | 11 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(10), $.ok(20), $.ok(30)]); 12 | }); 13 | 14 | test("synchronous atom", async ({ expect }) => { 15 | expect.assertions(1); 16 | 17 | const s = $.from([1, 2, 3]).map((n) => { 18 | if (n === 2) { 19 | return $.error("number 2"); 20 | } else { 21 | return $.ok(n); 22 | } 23 | }); 24 | 25 | expect(await s.toArray({ atoms: true })).toEqual([ 26 | $.ok(1), 27 | $.error("number 2"), 28 | $.ok(3), 29 | ]); 30 | }); 31 | 32 | test("synchronous mix", async ({ expect }) => { 33 | expect.assertions(1); 34 | 35 | const s = $.from([1, 2, 3]).map((n) => { 36 | if (n === 2) { 37 | return $.error("number 2"); 38 | } else { 39 | return n; 40 | } 41 | }); 42 | 43 | expect(await s.toArray({ atoms: true })).toEqual([ 44 | $.ok(1), 45 | $.error("number 2"), 46 | $.ok(3), 47 | ]); 48 | }); 49 | 50 | test("asynchronous value", async ({ expect }) => { 51 | expect.assertions(1); 52 | 53 | const s = $.from([1, 2, 3]).map(async (n) => n * 10); 54 | 55 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(10), $.ok(20), $.ok(30)]); 56 | }); 57 | }); 58 | 59 | describe("collect", () => { 60 | test("simple stream without errors", async ({ expect }) => { 61 | expect.assertions(1); 62 | 63 | const s = $.from([1, 2, 3]).collect(); 64 | 65 | expect(await s.toArray({ atoms: true })).toEqual([$.ok([1, 2, 3])]); 66 | }); 67 | 68 | test("empty stream", async ({ expect }) => { 69 | expect.assertions(1); 70 | 71 | const s = $.from([]).collect(); 72 | 73 | expect(await s.toArray({ atoms: true })).toEqual([$.ok([])]); 74 | }); 75 | 76 | test("single error", async ({ expect }) => { 77 | expect.assertions(1); 78 | 79 | const s = $.from([$.error(1), $.ok(2), $.ok(3)]).collect(); 80 | 81 | expect(await s.toArray({ atoms: true })).toEqual([$.error(1), $.ok([2, 3])]); 82 | }); 83 | 84 | test("single unknown", async ({ expect }) => { 85 | expect.assertions(1); 86 | 87 | const s = $.from([$.exception(1, []), $.ok(2), $.ok(3)]).collect(); 88 | 89 | expect(await s.toArray({ atoms: true })).toEqual([$.exception(1, []), $.ok([2, 3])]); 90 | }); 91 | }); 92 | 93 | describe("mapError", () => { 94 | test("single error", async ({ expect }) => { 95 | expect.assertions(1); 96 | 97 | const s = $.from([$.error(1), $.ok(2), $.ok(3)]).mapError((_e) => $.ok("error")); 98 | 99 | expect(await s.toArray({ atoms: true })).toEqual([$.ok("error"), $.ok(2), $.ok(3)]); 100 | }); 101 | 102 | test("multiple errors", async ({ expect }) => { 103 | expect.assertions(1); 104 | 105 | const s = $.from([$.error(1), $.ok(2), $.error(3)]).mapError((e) => $.ok("error" + e)); 106 | 107 | expect(await s.toArray({ atoms: true })).toEqual([ 108 | $.ok("error1"), 109 | $.ok(2), 110 | $.ok("error3"), 111 | ]); 112 | }); 113 | }); 114 | 115 | describe("mapUnknown", () => { 116 | test("single unknown", async ({ expect }) => { 117 | expect.assertions(1); 118 | 119 | const s = $.from([$.exception(1, []), $.ok(2), $.ok(3)]).mapException((e) => 120 | $.error(e), 121 | ); 122 | 123 | expect(await s.toArray({ atoms: true })).toEqual([$.error(1), $.ok(2), $.ok(3)]); 124 | }); 125 | 126 | test("multiple unknown", async ({ expect }) => { 127 | expect.assertions(1); 128 | 129 | const s = $.from([$.exception(1, []), $.ok(2), $.exception(3, [])]).mapException((e) => 130 | $.error(e), 131 | ); 132 | 133 | expect(await s.toArray({ atoms: true })).toEqual([$.error(1), $.ok(2), $.error(3)]); 134 | }); 135 | }); 136 | 137 | describe("filter", () => { 138 | test("synchronous values", async ({ expect }) => { 139 | expect.assertions(1); 140 | 141 | const s = $.from([1, 2, 3, 4]).filter((n) => n % 2 === 0); 142 | 143 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(2), $.ok(4)]); 144 | }); 145 | 146 | test("synchronous atoms", async ({ expect }) => { 147 | expect.assertions(1); 148 | 149 | const s = $.from([1, $.error("an error"), 2, 3, 4]) 150 | // Perform the actual filter operation 151 | .filter((n) => n % 2 === 0); 152 | 153 | expect(await s.toArray({ atoms: true })).toEqual([ 154 | $.error("an error"), 155 | $.ok(2), 156 | $.ok(4), 157 | ]); 158 | }); 159 | }); 160 | 161 | describe("drop", () => { 162 | test("multiple values", async ({ expect }) => { 163 | expect.assertions(1); 164 | 165 | const s = $.from([1, 2, 3, 4, 5]).drop(2); 166 | 167 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(3), $.ok(4), $.ok(5)]); 168 | }); 169 | 170 | test("multiple values with errors", async ({ expect }) => { 171 | expect.assertions(1); 172 | 173 | const s = $.from([1, $.error("some error"), 2, 3, 4, 5]).drop(2); 174 | 175 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(3), $.ok(4), $.ok(5)]); 176 | }); 177 | 178 | test("multiple atoms", async ({ expect }) => { 179 | expect.assertions(1); 180 | 181 | const s = $.from([1, 2, 3, 4, 5]).drop(2, { atoms: true }); 182 | 183 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(3), $.ok(4), $.ok(5)]); 184 | }); 185 | 186 | test("multiple atoms with errors", async ({ expect }) => { 187 | expect.assertions(1); 188 | 189 | const s = $.from([1, $.error("some error"), 2, 3, 4, 5]).drop(2, { atoms: true }); 190 | 191 | expect(await s.toArray({ atoms: true })).toEqual([$.ok(2), $.ok(3), $.ok(4), $.ok(5)]); 192 | }); 193 | }); 194 | 195 | describe("bufferedMap", () => { 196 | beforeEach(() => { 197 | vi.useFakeTimers(); 198 | }); 199 | afterEach(() => { 200 | vi.restoreAllMocks(); 201 | }); 202 | 203 | function timeout(ms: number) { 204 | return new Promise((resolve) => { 205 | setTimeout(() => { 206 | resolve(); 207 | }, ms); 208 | }); 209 | } 210 | 211 | test("multiple values", async ({ expect }) => { 212 | expect.assertions(2); 213 | 214 | // Will infinitely produce values 215 | const counter = vi.fn(); 216 | 217 | let i = 0; 218 | const s = $.fromNext(async () => { 219 | if (i === 10) { 220 | return $.StreamEnd; 221 | } 222 | 223 | return i++; 224 | }) 225 | .bufferedMap(async (n) => { 226 | // Do some slow work 227 | await timeout(10); 228 | 229 | return n; 230 | }) 231 | .tap(counter) 232 | .toArray({ atoms: true }); 233 | 234 | await vi.advanceTimersByTimeAsync(50); 235 | 236 | expect(await s).toEqual([ 237 | $.ok(0), 238 | $.ok(1), 239 | $.ok(2), 240 | $.ok(3), 241 | $.ok(4), 242 | $.ok(5), 243 | $.ok(6), 244 | $.ok(7), 245 | $.ok(8), 246 | $.ok(9), 247 | ]); 248 | 249 | expect(counter).toBeCalledTimes(10); 250 | }); 251 | 252 | test("slow producer", async ({ expect }) => { 253 | expect.assertions(2); 254 | 255 | // Producer that will never produce a value 256 | const producer = vi.fn().mockReturnValue(new Promise(() => {})); 257 | const counter = vi.fn(); 258 | 259 | $.fromNext(producer).bufferedMap(counter).exhaust(); 260 | 261 | // Give some time for everything to spin 262 | await vi.advanceTimersByTimeAsync(50); 263 | 264 | expect(producer).toBeCalledTimes(1); 265 | expect(counter).toBeCalledTimes(0); 266 | }); 267 | 268 | test("slow producer, slow operation", async ({ expect }) => { 269 | expect.assertions(15); 270 | 271 | const producer = vi.fn(async () => { 272 | await timeout(10); 273 | return i++; 274 | }); 275 | const mapper = vi.fn(async (n) => { 276 | await timeout(20); 277 | 278 | return n; 279 | }); 280 | const counter = vi.fn(); 281 | 282 | let i = 0; 283 | 284 | $.fromNext(producer).bufferedMap(mapper).tap(counter).exhaust(); 285 | 286 | // 9ms, producer should only be called once 287 | await vi.advanceTimersByTimeAsync(9); 288 | expect(producer).toHaveBeenCalledTimes(1); 289 | expect(mapper).toHaveBeenCalledTimes(0); 290 | expect(counter).toHaveBeenCalledTimes(0); 291 | 292 | // 10ms, producer output value, mapper begins 293 | await vi.advanceTimersByTimeAsync(1); 294 | expect(producer).toHaveBeenCalledTimes(2); 295 | expect(mapper).toHaveBeenCalledTimes(1); 296 | expect(counter).toHaveBeenCalledTimes(0); 297 | 298 | // 20ms, producer output another value, another mapper begins 299 | await vi.advanceTimersByTimeAsync(10); 300 | expect(producer).toHaveBeenCalledTimes(3); 301 | expect(mapper).toHaveBeenCalledTimes(2); 302 | expect(counter).toHaveBeenCalledTimes(0); 303 | 304 | // 30ms, producer output another value, another mapper begins, first mapper finish 305 | await vi.advanceTimersByTimeAsync(10); 306 | expect(producer).toHaveBeenCalledTimes(4); 307 | expect(mapper).toHaveBeenCalledTimes(3); 308 | expect(counter).toHaveBeenCalledTimes(1); 309 | 310 | // 40ms, producer output another value, another mapper begins, second mapper finish 311 | await vi.advanceTimersByTimeAsync(10); 312 | expect(producer).toHaveBeenCalledTimes(5); 313 | expect(mapper).toHaveBeenCalledTimes(4); 314 | expect(counter).toHaveBeenCalledTimes(2); 315 | }); 316 | }); 317 | 318 | describe.sequential("batch", () => { 319 | beforeEach(() => { 320 | vi.useFakeTimers(); 321 | }); 322 | 323 | afterEach(() => { 324 | vi.restoreAllMocks(); 325 | }); 326 | 327 | test("2 items", async ({ expect }) => { 328 | expect.assertions(1); 329 | 330 | let i = 0; 331 | const s = $.fromNext(() => { 332 | return new Promise((res) => res(i++)); 333 | }) 334 | .batch({ n: 2 }) 335 | .take(3); 336 | 337 | expect(await s.toArray({ atoms: true })).toEqual([ 338 | $.ok([0, 1]), 339 | $.ok([2, 3]), 340 | $.ok([4, 5]), 341 | ]); 342 | }); 343 | 344 | test("yield remaining true", async ({ expect }) => { 345 | expect.assertions(1); 346 | 347 | const s = $.from([0, 1, 2, 3, 4]).batch({ n: 3, yieldRemaining: true }); 348 | 349 | expect(await s.toArray({ atoms: true })).toEqual([$.ok([0, 1, 2]), $.ok([3, 4])]); 350 | }); 351 | 352 | test("yield remaining false", async ({ expect }) => { 353 | expect.assertions(1); 354 | 355 | const s = $.from([0, 1, 2, 3, 4]).batch({ n: 3, yieldRemaining: false }); 356 | 357 | expect(await s.toArray({ atoms: true })).toEqual([$.ok([0, 1, 2])]); 358 | }); 359 | 360 | test("with timeout", async ({ expect }) => { 361 | expect.assertions(3); 362 | 363 | const mapper = vi.fn(); 364 | 365 | let i = 0; 366 | $.fromNext(async () => { 367 | return i++; 368 | }) 369 | .batch({ timeout: 100 }) 370 | .map(mapper) 371 | .exhaust(); 372 | 373 | await vi.advanceTimersByTimeAsync(100); 374 | expect(mapper).toHaveBeenCalledTimes(1); 375 | 376 | await vi.advanceTimersByTimeAsync(50); 377 | expect(mapper).toHaveBeenCalledTimes(1); 378 | 379 | await vi.advanceTimersByTimeAsync(50); 380 | expect(mapper).toHaveBeenCalledTimes(2); 381 | }); 382 | 383 | test("with timeout yield empty", async ({ expect }) => { 384 | expect.assertions(5); 385 | 386 | const mapper = vi.fn(); 387 | 388 | $.fromNext(() => { 389 | // Promise that will never resolve 390 | return new Promise(() => {}); 391 | }) 392 | .batch({ timeout: 100, yieldEmpty: true }) 393 | .map(mapper) 394 | .exhaust(); 395 | 396 | await vi.advanceTimersByTimeAsync(100); 397 | expect(mapper).toHaveBeenCalledTimes(1); 398 | expect(mapper).toHaveBeenNthCalledWith(1, []); 399 | 400 | await vi.advanceTimersByTimeAsync(50); 401 | expect(mapper).toHaveBeenCalledTimes(1); 402 | 403 | await vi.advanceTimersByTimeAsync(50); 404 | expect(mapper).toHaveBeenCalledTimes(2); 405 | expect(mapper).toHaveBeenNthCalledWith(2, []); 406 | }); 407 | 408 | test("n with timeout", async ({ expect }) => { 409 | const mapper = vi.fn(); 410 | 411 | $.from([1, 2, 3, 4, 5]).batch({ n: 3, timeout: 100 }).map(mapper).exhaust(); 412 | 413 | await vi.advanceTimersByTimeAsync(50); 414 | expect(mapper).toHaveBeenCalledTimes(1); 415 | expect(mapper).toHaveBeenNthCalledWith(1, [1, 2, 3]); 416 | 417 | await vi.advanceTimersByTimeAsync(50); 418 | expect(mapper).toHaveBeenCalledTimes(2); 419 | expect(mapper).toHaveBeenNthCalledWith(2, [4, 5]); 420 | }); 421 | 422 | test("with bucket", async ({ expect }) => { 423 | expect.assertions(1); 424 | 425 | const s = await $.from([1, 2, 3, 4, 5, 6]) 426 | .batch({ n: 2, byBucket: (n) => (n % 2 === 0 ? "even" : "odd") }) 427 | .toArray(); 428 | 429 | expect(s).toEqual([ 430 | [1, 3], 431 | [2, 4], 432 | ]); 433 | }); 434 | 435 | test("timeout bucket", async ({ expect }) => { 436 | const mapper = vi.fn(); 437 | 438 | $.from([1, 2, 3, 4, 5, 6]) 439 | .batch({ timeout: 100, byBucket: (n) => (n % 2 === 0 ? "even" : "odd") }) 440 | .map(mapper) 441 | .exhaust(); 442 | 443 | await vi.advanceTimersByTimeAsync(100); 444 | expect(mapper).toHaveBeenCalledTimes(2); 445 | expect(mapper).toHaveBeenNthCalledWith(1, [1, 3, 5]); 446 | expect(mapper).toHaveBeenNthCalledWith(2, [2, 4, 6]); 447 | }); 448 | 449 | test("timeout bucket no items", async ({ expect }) => { 450 | expect.assertions(2); 451 | 452 | const mapper = vi.fn(); 453 | 454 | $.fromNext(() => new Promise(() => {})) 455 | .batch({ timeout: 100, byBucket: (n) => (n % 2 === 0 ? "even" : "odd") }) 456 | .map(mapper) 457 | .exhaust(); 458 | 459 | await vi.advanceTimersByTimeAsync(100); 460 | expect(mapper).toHaveBeenCalledTimes(0); 461 | 462 | await vi.advanceTimersByTimeAsync(100); 463 | expect(mapper).toHaveBeenCalledTimes(0); 464 | }); 465 | 466 | test("yield remaining doesn't incorrectly yield empty", async ({ expect }) => { 467 | expect.assertions(3); 468 | 469 | const mapper = vi.fn(); 470 | 471 | const testItems = [1, 1, 1, 1]; 472 | $.fromNext(async () => { 473 | if (testItems.length > 0) { 474 | return testItems.shift(); 475 | } 476 | 477 | return new Promise(() => {}); 478 | }) 479 | .batch({ timeout: 100, n: 10, yieldRemaining: true, yieldEmpty: false }) 480 | .map(mapper) 481 | .exhaust(); 482 | 483 | await vi.advanceTimersByTimeAsync(100); 484 | expect(mapper).toHaveBeenCalledTimes(1); 485 | expect(mapper).toHaveBeenNthCalledWith(1, [1, 1, 1, 1]); 486 | 487 | await vi.advanceTimersByTimeAsync(100); 488 | expect(mapper).toHaveBeenCalledTimes(1); 489 | }); 490 | 491 | describe("batch weirdness", () => { 492 | test("5 items, n = 10", async ({ expect }) => { 493 | expect.assertions(1); 494 | 495 | const s = await $.from([1, 2, 3, 4, 5]).batch({ n: 10 }).toArray(); 496 | 497 | expect(s).toEqual([]); 498 | }); 499 | 500 | test("5 items, n = 10, yieldRemaining", async ({ expect }) => { 501 | expect.assertions(1); 502 | 503 | const s = await $.from([1, 2, 3, 4, 5]) 504 | .batch({ n: 10, yieldRemaining: true }) 505 | .toArray(); 506 | 507 | expect(s).toEqual([[1, 2, 3, 4, 5]]); 508 | }); 509 | 510 | function createHangingStream() { 511 | let i = 0; 512 | return $.fromNext(() => { 513 | if (i < 5) { 514 | return Promise.resolve(i++); 515 | } 516 | 517 | // Hang 518 | return new Promise(() => {}); 519 | }); 520 | } 521 | 522 | test("5 items, n = 10, timeout, yieldRemaining, infinite hang", async ({ expect }) => { 523 | expect.assertions(1); 524 | 525 | const a = createHangingStream() 526 | .batch({ n: 10, timeout: 5, yieldRemaining: true }) 527 | .take(1) 528 | .toArray(); 529 | 530 | await vi.advanceTimersByTimeAsync(5); 531 | expect(await a).toEqual([[0, 1, 2, 3, 4]]); 532 | }); 533 | 534 | test("5 items, n = 10, timeout, yieldEmpty, infinite hang", async ({ expect }) => { 535 | expect.assertions(1); 536 | 537 | const a = createHangingStream() 538 | .batch({ n: 10, timeout: 5, yieldEmpty: true }) 539 | .take(1) 540 | .toArray(); 541 | 542 | await vi.advanceTimersByTimeAsync(5); 543 | expect(await a).toEqual([[]]); 544 | }); 545 | 546 | test("5 items, n = 10, timeout, yieldEmpty, yieldRemaining, infinite hang", async ({ 547 | expect, 548 | }) => { 549 | expect.assertions(1); 550 | 551 | const a = createHangingStream() 552 | .batch({ n: 10, timeout: 5, yieldRemaining: true, yieldEmpty: true }) 553 | .take(2) 554 | .toArray(); 555 | 556 | await vi.advanceTimersByTimeAsync(10); 557 | expect(await a).toEqual([[0, 1, 2, 3, 4], []]); 558 | }); 559 | }); 560 | }); 561 | 562 | describe("onFirst", () => { 563 | async function testAtom( 564 | expect: ExpectStatic, 565 | atom: unknown, 566 | atomOption: boolean, 567 | spyCalled: boolean, 568 | ) { 569 | expect.assertions(2); 570 | 571 | const spy = vi.fn(); 572 | 573 | const s = await $.from([atom]) 574 | .onFirst(spy, { atom: atomOption }) 575 | .toArray({ atoms: true }); 576 | 577 | expect(s, "stream should not be altered").toEqual([atom]); 578 | 579 | if (spyCalled) { 580 | expect(spy, "callback must be called").toHaveBeenCalledOnce(); 581 | } else { 582 | expect(spy, "callback must not be called").not.toHaveBeenCalled(); 583 | } 584 | } 585 | 586 | describe("single item in stream", () => { 587 | test("ok atom", async ({ expect }) => { 588 | await testAtom(expect, $.ok(1), true, true); 589 | }); 590 | 591 | test("error atom", async ({ expect }) => { 592 | await testAtom(expect, $.error(1), true, true); 593 | }); 594 | 595 | test("exception atom", async ({ expect }) => { 596 | await testAtom(expect, $.exception(1, []), true, true); 597 | }); 598 | }); 599 | 600 | test("multiple items in stream", async ({ expect }) => { 601 | expect.assertions(2); 602 | 603 | const spy = vi.fn(); 604 | 605 | const s = await $.from([$.error(1), $.ok(2), $.ok(3), $.exception(4, []), $.ok(5)]) 606 | .onFirst(spy) 607 | .toArray({ atoms: true }); 608 | 609 | expect(s, "stream should not be altered").toEqual([ 610 | $.error(1), 611 | $.ok(2), 612 | $.ok(3), 613 | $.exception(4, []), 614 | $.ok(5), 615 | ]); 616 | expect(spy, "callback must be called once").toHaveBeenCalledOnce(); 617 | }); 618 | 619 | test("no items in stream", async ({ expect }) => { 620 | expect.assertions(2); 621 | 622 | const spy = vi.fn(); 623 | 624 | const s = await $.from([]).onFirst(spy).toArray(); 625 | 626 | expect(s, "stream should not be altered").toEqual([]); 627 | expect(spy, "callback must not be called").not.toHaveBeenCalled(); 628 | }); 629 | 630 | describe("with atom = false", () => { 631 | describe("single item in stream", () => { 632 | test("ok atom", async ({ expect }) => { 633 | await testAtom(expect, $.ok(1), false, true); 634 | }); 635 | 636 | test("error atom", async ({ expect }) => { 637 | await testAtom(expect, $.error(1), false, false); 638 | }); 639 | 640 | test("exception atom", async ({ expect }) => { 641 | await testAtom(expect, $.exception(1, []), false, false); 642 | }); 643 | }); 644 | 645 | test("error, ok", async ({ expect }) => { 646 | expect.assertions(4); 647 | 648 | const spy = vi.fn(); 649 | 650 | const s = $.from([$.error(1), $.ok(2)]) 651 | .onFirst(spy, { atom: false }) 652 | [Symbol.asyncIterator](); 653 | 654 | expect((await s.next()).value, "error should be emitted first").toEqual($.error(1)); 655 | expect(spy, "callback shouldn't be triggered on an error").not.toHaveBeenCalled(); 656 | 657 | expect((await s.next()).value, "ok value should be emitted next").toEqual($.ok(2)); 658 | expect(spy, "spy should only be called after the ok atom").toHaveBeenCalledOnce(); 659 | }); 660 | }); 661 | }); 662 | 663 | describe("onLast", () => { 664 | async function testAtom( 665 | expect: ExpectStatic, 666 | atom: unknown, 667 | atomOption: boolean, 668 | spyCalled: boolean, 669 | ) { 670 | expect.assertions(2); 671 | 672 | const spy = vi.fn(); 673 | 674 | const s = await $.from([atom]) 675 | .onLast(spy, { atom: atomOption }) 676 | .toArray({ atoms: true }); 677 | 678 | expect(s, "stream should not be altered").toEqual([atom]); 679 | 680 | if (spyCalled) { 681 | expect(spy, "callback must be called").toHaveBeenCalledOnce(); 682 | } else { 683 | expect(spy, "callback must not be called").not.toHaveBeenCalled(); 684 | } 685 | } 686 | 687 | describe("single item in stream", () => { 688 | test("ok atom", async ({ expect }) => { 689 | await testAtom(expect, $.ok(1), true, true); 690 | }); 691 | 692 | test("error atom", async ({ expect }) => { 693 | await testAtom(expect, $.error(1), true, true); 694 | }); 695 | 696 | test("exception atom", async ({ expect }) => { 697 | await testAtom(expect, $.exception(1, []), true, true); 698 | }); 699 | }); 700 | 701 | test("multiple items in stream", async ({ expect }) => { 702 | expect.assertions(2); 703 | 704 | const spy = vi.fn(); 705 | 706 | const s = await $.from([$.error(1), $.ok(2), $.ok(3), $.exception(4, []), $.ok(5)]) 707 | .onLast(spy) 708 | .toArray({ atoms: true }); 709 | 710 | expect(s, "stream should not be altered").toEqual([ 711 | $.error(1), 712 | $.ok(2), 713 | $.ok(3), 714 | $.exception(4, []), 715 | $.ok(5), 716 | ]); 717 | expect(spy, "callback must be called once").toHaveBeenCalledOnce(); 718 | }); 719 | 720 | test("no items in stream", async ({ expect }) => { 721 | expect.assertions(2); 722 | 723 | const spy = vi.fn(); 724 | 725 | const s = await $.from([]).onLast(spy).toArray(); 726 | 727 | expect(s, "stream should not be altered").toEqual([]); 728 | expect(spy, "callback must not be called").not.toHaveBeenCalled(); 729 | }); 730 | 731 | describe("with atom = false", () => { 732 | describe("single item in stream", () => { 733 | test("ok atom", async ({ expect }) => { 734 | await testAtom(expect, $.ok(1), false, true); 735 | }); 736 | 737 | test("error atom", async ({ expect }) => { 738 | await testAtom(expect, $.error(1), false, false); 739 | }); 740 | 741 | test("exception atom", async ({ expect }) => { 742 | await testAtom(expect, $.exception(1, []), false, false); 743 | }); 744 | }); 745 | 746 | test("error, ok", async ({ expect }) => { 747 | expect.assertions(4); 748 | 749 | const spy = vi.fn(); 750 | 751 | const s = $.from([$.ok(1), $.error(2)]) 752 | .onLast(spy, { atom: false }) 753 | [Symbol.asyncIterator](); 754 | 755 | expect((await s.next()).value, "ok value should be emitted first").toEqual($.ok(1)); 756 | expect(spy, "callback shouldn't be triggered on an error").not.toHaveBeenCalled(); 757 | 758 | expect((await s.next()).value, "error value should be emitted next").toEqual( 759 | $.error(2), 760 | ); 761 | expect( 762 | spy, 763 | "spy should only be called after the stream ends atom", 764 | ).toHaveBeenCalledOnce(); 765 | }); 766 | }); 767 | }); 768 | }); 769 | --------------------------------------------------------------------------------