├── .github ├── dependabot.yml └── workflows │ ├── js-test-and-release.yml │ ├── semantic-pull-request.yml │ └── stale.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── package.json ├── src ├── fifo.ts └── index.ts ├── test └── test.spec.ts ├── tsconfig.json └── typedoc.json /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "10:00" 8 | open-pull-requests-limit: 10 9 | commit-message: 10 | prefix: "deps" 11 | prefix-development: "deps(dev)" 12 | -------------------------------------------------------------------------------- /.github/workflows/js-test-and-release.yml: -------------------------------------------------------------------------------- 1 | name: test & maybe release 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | workflow_dispatch: 9 | 10 | permissions: 11 | contents: write 12 | packages: write 13 | 14 | concurrency: 15 | group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} 16 | cancel-in-progress: true 17 | 18 | jobs: 19 | js-test-and-release: 20 | uses: pl-strflt/uci/.github/workflows/js-test-and-release.yml@v0.0 21 | secrets: 22 | DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} 23 | DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} 24 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 25 | UCI_GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} 26 | -------------------------------------------------------------------------------- /.github/workflows/semantic-pull-request.yml: -------------------------------------------------------------------------------- 1 | name: Semantic PR 2 | 3 | on: 4 | pull_request_target: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | 10 | jobs: 11 | main: 12 | uses: pl-strflt/.github/.github/workflows/reusable-semantic-pull-request.yml@v0.3 13 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Close and mark stale issue 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | 7 | permissions: 8 | issues: write 9 | pull-requests: write 10 | 11 | jobs: 12 | stale: 13 | uses: pl-strflt/.github/.github/workflows/reusable-stale-issue.yml@v0.3 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | dist 4 | .docs 5 | .coverage 6 | node_modules 7 | package-lock.json 8 | yarn.lock 9 | .vscode 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## [3.2.3](https://github.com/alanshaw/it-pushable/compare/v3.2.2...v3.2.3) (2023-11-16) 2 | 3 | 4 | ### Bug Fixes 5 | 6 | * support onEmpty with onEnd ([#76](https://github.com/alanshaw/it-pushable/issues/76)) ([26b5d18](https://github.com/alanshaw/it-pushable/commit/26b5d18d788b16833d66c4063a1c9f07d5a57e53)) 7 | 8 | ## [3.2.2](https://github.com/alanshaw/it-pushable/compare/v3.2.1...v3.2.2) (2023-11-11) 9 | 10 | 11 | ### Trivial Changes 12 | 13 | * add or force update .github/workflows/js-test-and-release.yml ([#66](https://github.com/alanshaw/it-pushable/issues/66)) ([9dc3452](https://github.com/alanshaw/it-pushable/commit/9dc3452fb1d6dc7a978dec2b429204996e28eb3a)) 14 | * delete templates [skip ci] ([#65](https://github.com/alanshaw/it-pushable/issues/65)) ([623e049](https://github.com/alanshaw/it-pushable/commit/623e0496897df1db83c5677c3540b5b708251372)) 15 | 16 | 17 | ### Dependencies 18 | 19 | * **dev:** bump aegir from 39.0.13 to 41.0.5 ([#74](https://github.com/alanshaw/it-pushable/issues/74)) ([ca99910](https://github.com/alanshaw/it-pushable/commit/ca9991015a1852c2aedc16af149237ff5c083742)) 20 | 21 | ## [3.2.1](https://github.com/alanshaw/it-pushable/compare/v3.2.0...v3.2.1) (2023-07-04) 22 | 23 | 24 | ### Bug Fixes 25 | 26 | * await returned promise in waitNext ([#59](https://github.com/alanshaw/it-pushable/issues/59)) ([d19e8ca](https://github.com/alanshaw/it-pushable/commit/d19e8caf02a98989bd4c42f60ccc8d616bdbe12b)) 27 | 28 | ## [3.2.0](https://github.com/alanshaw/it-pushable/compare/v3.1.4...v3.2.0) (2023-07-03) 29 | 30 | 31 | ### Features 32 | 33 | * add onEmpty function that resolves when the queue is empty ([#58](https://github.com/alanshaw/it-pushable/issues/58)) ([2bed38a](https://github.com/alanshaw/it-pushable/commit/2bed38ad1e477efc30c1a800c4de51813c8319d0)) 34 | 35 | ## [3.1.4](https://github.com/alanshaw/it-pushable/compare/v3.1.3...v3.1.4) (2023-06-30) 36 | 37 | 38 | ### Dependencies 39 | 40 | * **dev:** bump aegir from 37.12.1 to 39.0.13 ([#52](https://github.com/alanshaw/it-pushable/issues/52)) ([9399579](https://github.com/alanshaw/it-pushable/commit/939957932ed19a6558748e06ce50822b8d062b9b)) 41 | 42 | ## [3.1.3](https://github.com/alanshaw/it-pushable/compare/v3.1.2...v3.1.3) (2023-04-18) 43 | 44 | 45 | ### Bug Fixes 46 | 47 | * extend async generator ([#56](https://github.com/alanshaw/it-pushable/issues/56)) ([3cf8f5c](https://github.com/alanshaw/it-pushable/commit/3cf8f5c5dd6d7aac8e93f7bfe003b99caf9267ca)) 48 | 49 | ## [3.1.2](https://github.com/alanshaw/it-pushable/compare/v3.1.1...v3.1.2) (2022-12-16) 50 | 51 | 52 | ### Documentation 53 | 54 | * publish api docs ([#39](https://github.com/alanshaw/it-pushable/issues/39)) ([95adf08](https://github.com/alanshaw/it-pushable/commit/95adf08e789b8ca4617163026e091835aba39706)) 55 | 56 | ## [3.1.1](https://github.com/alanshaw/it-pushable/compare/v3.1.0...v3.1.1) (2022-12-16) 57 | 58 | 59 | ### Trivial Changes 60 | 61 | * **deps-dev:** bump it-all from 1.0.6 to 2.0.0 ([#34](https://github.com/alanshaw/it-pushable/issues/34)) ([1b50e5e](https://github.com/alanshaw/it-pushable/commit/1b50e5ed9211711530103db140cd401ecb2339fd)) 62 | 63 | ## [3.1.0](https://github.com/alanshaw/it-pushable/compare/v3.0.0...v3.1.0) (2022-08-02) 64 | 65 | 66 | ### Features 67 | 68 | * support Uint8ArrayLists in the same way as Uint8Arrays ([#30](https://github.com/alanshaw/it-pushable/issues/30)) ([7bae368](https://github.com/alanshaw/it-pushable/commit/7bae3688b1363954539b56de79c3a81ff53df59f)) 69 | 70 | ## [3.0.0](https://github.com/alanshaw/it-pushable/compare/v2.0.2...v3.0.0) (2022-06-10) 71 | 72 | 73 | ### ⚠ BREAKING CHANGES 74 | 75 | * `Uint8Array`s are expected by default, pass `objectMode: true` to push any other data types. If using TypeScript, use generics to define the data type. 76 | 77 | ### Features 78 | 79 | * add readableLength property ([#27](https://github.com/alanshaw/it-pushable/issues/27)) ([f45aee3](https://github.com/alanshaw/it-pushable/commit/f45aee36e72e754b8a27dda48d3051c470aaa8e5)) 80 | 81 | 82 | ### Trivial Changes 83 | 84 | * fix flaky test ([#28](https://github.com/alanshaw/it-pushable/issues/28)) ([8c3dcc1](https://github.com/alanshaw/it-pushable/commit/8c3dcc1f8e64f2877317a835bb03545f7fa2dd53)) 85 | 86 | ### [2.0.2](https://github.com/alanshaw/it-pushable/compare/v2.0.1...v2.0.2) (2022-06-08) 87 | 88 | 89 | ### Bug Fixes 90 | 91 | * update aegir ([#24](https://github.com/alanshaw/it-pushable/issues/24)) ([9c1a478](https://github.com/alanshaw/it-pushable/commit/9c1a4783a536d90bcede5d29cc2d66d2a0d5321a)) 92 | 93 | 94 | ### Trivial Changes 95 | 96 | * update publish command ([#25](https://github.com/alanshaw/it-pushable/issues/25)) ([b82173f](https://github.com/alanshaw/it-pushable/commit/b82173f3c7d07581e1ece3ed3026ccbbf6c57056)) 97 | 98 | ### [2.0.1](https://github.com/alanshaw/it-pushable/compare/v2.0.0...v2.0.1) (2022-01-13) 99 | 100 | 101 | ### Trivial Changes 102 | 103 | * fix readme example ([#13](https://github.com/alanshaw/it-pushable/issues/13)) ([d4d7282](https://github.com/alanshaw/it-pushable/commit/d4d728275ba97977fd2004be749a57bbb74aebca)) 104 | 105 | ## [2.0.0](https://github.com/alanshaw/it-pushable/compare/v1.4.2...v2.0.0) (2022-01-13) 106 | 107 | 108 | ### ⚠ BREAKING CHANGES 109 | 110 | * switch to named exports, ESM only 111 | 112 | ### Features 113 | 114 | * convert to typescript ([#12](https://github.com/alanshaw/it-pushable/issues/12)) ([49e0805](https://github.com/alanshaw/it-pushable/commit/49e080564a410a5f3475dfaa389ad8f0f1d8582c)) 115 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This project is dual licensed under MIT and Apache-2.0. 2 | 3 | MIT: https://www.opensource.org/licenses/mit 4 | Apache-2.0: https://www.apache.org/licenses/license-2.0 5 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 2 | 3 | http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 6 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # it-pushable 2 | 3 | [![codecov](https://img.shields.io/codecov/c/github/alanshaw/it-pushable.svg?style=flat-square)](https://codecov.io/gh/alanshaw/it-pushable) 4 | [![CI](https://img.shields.io/github/actions/workflow/status/alanshaw/it-pushable/js-test-and-release.yml?branch=master\&style=flat-square)](https://github.com/alanshaw/it-pushable/actions/workflows/js-test-and-release.yml?query=branch%3Amaster) 5 | 6 | > An iterable that you can push values into 7 | 8 | ## Table of contents 9 | 10 | - [Install](#install) 11 | - [Browser ` 30 | ``` 31 | 32 | ## Usage 33 | 34 | ```js 35 | import { pushable } from 'it-pushable' 36 | 37 | const source = pushable() 38 | 39 | setTimeout(() => source.push('hello'), 100) 40 | setTimeout(() => source.push('world'), 200) 41 | setTimeout(() => source.end(), 300) 42 | 43 | const start = Date.now() 44 | 45 | for await (const value of source) { 46 | console.log(`got "${value}" after ${Date.now() - start}ms`) 47 | } 48 | console.log(`done after ${Date.now() - start}ms`) 49 | 50 | /* 51 | Output: 52 | got "hello" after 105ms 53 | got "world" after 207ms 54 | done after 309ms 55 | */ 56 | ``` 57 | 58 | ```js 59 | import { pushableV } from 'it-pushable' 60 | import all from 'it-all' 61 | 62 | const source = pushableV() 63 | 64 | source.push(1) 65 | source.push(2) 66 | source.push(3) 67 | source.end() 68 | 69 | console.info(await all(source)) 70 | /* 71 | Output: 72 | [ [1, 2, 3] ] 73 | */ 74 | ``` 75 | 76 | ## Related 77 | 78 | - [`it-pipe`](https://www.npmjs.com/package/it-pipe) Utility to "pipe" async iterables together 79 | 80 | ## API Docs 81 | 82 | - 83 | 84 | ## License 85 | 86 | Licensed under either of 87 | 88 | - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) 89 | - MIT ([LICENSE-MIT](LICENSE-MIT) / ) 90 | 91 | ## Contribution 92 | 93 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 94 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "it-pushable", 3 | "version": "3.2.3", 4 | "description": "An iterable that you can push values into", 5 | "author": "Alan Shaw", 6 | "license": "Apache-2.0 OR MIT", 7 | "homepage": "https://github.com/alanshaw/it-pushable#readme", 8 | "repository": { 9 | "type": "git", 10 | "url": "git+https://github.com/alanshaw/it-pushable.git" 11 | }, 12 | "bugs": { 13 | "url": "https://github.com/alanshaw/it-pushable/issues" 14 | }, 15 | "keywords": [ 16 | "iterable", 17 | "iterator", 18 | "push", 19 | "pushable" 20 | ], 21 | "type": "module", 22 | "types": "./dist/src/index.d.ts", 23 | "files": [ 24 | "src", 25 | "dist", 26 | "!dist/test", 27 | "!**/*.tsbuildinfo" 28 | ], 29 | "exports": { 30 | ".": { 31 | "types": "./dist/src/index.d.ts", 32 | "import": "./dist/src/index.js" 33 | } 34 | }, 35 | "eslintConfig": { 36 | "extends": "ipfs", 37 | "parserOptions": { 38 | "project": true, 39 | "sourceType": "module" 40 | } 41 | }, 42 | "release": { 43 | "branches": [ 44 | "master" 45 | ], 46 | "plugins": [ 47 | [ 48 | "@semantic-release/commit-analyzer", 49 | { 50 | "preset": "conventionalcommits", 51 | "releaseRules": [ 52 | { 53 | "breaking": true, 54 | "release": "major" 55 | }, 56 | { 57 | "revert": true, 58 | "release": "patch" 59 | }, 60 | { 61 | "type": "feat", 62 | "release": "minor" 63 | }, 64 | { 65 | "type": "fix", 66 | "release": "patch" 67 | }, 68 | { 69 | "type": "docs", 70 | "release": "patch" 71 | }, 72 | { 73 | "type": "test", 74 | "release": "patch" 75 | }, 76 | { 77 | "type": "deps", 78 | "release": "patch" 79 | }, 80 | { 81 | "scope": "no-release", 82 | "release": false 83 | } 84 | ] 85 | } 86 | ], 87 | [ 88 | "@semantic-release/release-notes-generator", 89 | { 90 | "preset": "conventionalcommits", 91 | "presetConfig": { 92 | "types": [ 93 | { 94 | "type": "feat", 95 | "section": "Features" 96 | }, 97 | { 98 | "type": "fix", 99 | "section": "Bug Fixes" 100 | }, 101 | { 102 | "type": "chore", 103 | "section": "Trivial Changes" 104 | }, 105 | { 106 | "type": "docs", 107 | "section": "Documentation" 108 | }, 109 | { 110 | "type": "deps", 111 | "section": "Dependencies" 112 | }, 113 | { 114 | "type": "test", 115 | "section": "Tests" 116 | } 117 | ] 118 | } 119 | } 120 | ], 121 | "@semantic-release/changelog", 122 | "@semantic-release/npm", 123 | "@semantic-release/github", 124 | "@semantic-release/git" 125 | ] 126 | }, 127 | "scripts": { 128 | "clean": "aegir clean", 129 | "lint": "aegir lint", 130 | "dep-check": "aegir dep-check", 131 | "build": "aegir build", 132 | "test": "aegir test -f ./dist/test", 133 | "test:chrome": "aegir test -t browser --cov", 134 | "test:chrome-webworker": "aegir test -t webworker", 135 | "test:firefox": "aegir test -t browser -- --browser firefox", 136 | "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", 137 | "test:node": "aegir test -t node --cov", 138 | "test:electron-main": "aegir test -t electron-main", 139 | "release": "aegir release", 140 | "docs": "aegir docs" 141 | }, 142 | "dependencies": { 143 | "p-defer": "^4.0.0" 144 | }, 145 | "devDependencies": { 146 | "@types/fast-fifo": "^1.0.0", 147 | "aegir": "^41.0.5", 148 | "it-all": "^3.0.1", 149 | "it-pipe": "^3.0.1", 150 | "uint8arraylist": "^2.0.0" 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/fifo.ts: -------------------------------------------------------------------------------- 1 | // ported from https://www.npmjs.com/package/fast-fifo 2 | 3 | export interface Next { 4 | done?: boolean 5 | error?: Error 6 | value?: T 7 | } 8 | 9 | class FixedFIFO { 10 | public buffer: Array | undefined> 11 | private readonly mask: number 12 | private top: number 13 | private btm: number 14 | public next: FixedFIFO | null 15 | 16 | constructor (hwm: number) { 17 | if (!(hwm > 0) || ((hwm - 1) & hwm) !== 0) { 18 | throw new Error('Max size for a FixedFIFO should be a power of two') 19 | } 20 | 21 | this.buffer = new Array(hwm) 22 | this.mask = hwm - 1 23 | this.top = 0 24 | this.btm = 0 25 | this.next = null 26 | } 27 | 28 | push (data: Next): boolean { 29 | if (this.buffer[this.top] !== undefined) { 30 | return false 31 | } 32 | 33 | this.buffer[this.top] = data 34 | this.top = (this.top + 1) & this.mask 35 | 36 | return true 37 | } 38 | 39 | shift (): Next | undefined { 40 | const last = this.buffer[this.btm] 41 | 42 | if (last === undefined) { 43 | return undefined 44 | } 45 | 46 | this.buffer[this.btm] = undefined 47 | this.btm = (this.btm + 1) & this.mask 48 | return last 49 | } 50 | 51 | isEmpty (): boolean { 52 | return this.buffer[this.btm] === undefined 53 | } 54 | } 55 | 56 | export interface FIFOOptions { 57 | /** 58 | * When the queue reaches this size, it will be split into head/tail parts 59 | */ 60 | splitLimit?: number 61 | } 62 | 63 | export class FIFO { 64 | public size: number 65 | private readonly hwm: number 66 | private head: FixedFIFO 67 | private tail: FixedFIFO 68 | 69 | constructor (options: FIFOOptions = {}) { 70 | this.hwm = options.splitLimit ?? 16 71 | this.head = new FixedFIFO(this.hwm) 72 | this.tail = this.head 73 | this.size = 0 74 | } 75 | 76 | calculateSize (obj: any): number { 77 | if (obj?.byteLength != null) { 78 | return obj.byteLength 79 | } 80 | 81 | return 1 82 | } 83 | 84 | push (val: Next): void { 85 | if (val?.value != null) { 86 | this.size += this.calculateSize(val.value) 87 | } 88 | 89 | if (!this.head.push(val)) { 90 | const prev = this.head 91 | this.head = prev.next = new FixedFIFO(2 * this.head.buffer.length) 92 | this.head.push(val) 93 | } 94 | } 95 | 96 | shift (): Next | undefined { 97 | let val = this.tail.shift() 98 | 99 | if (val === undefined && (this.tail.next != null)) { 100 | const next = this.tail.next 101 | this.tail.next = null 102 | this.tail = next 103 | val = this.tail.shift() 104 | } 105 | 106 | if (val?.value != null) { 107 | this.size -= this.calculateSize(val.value) 108 | } 109 | 110 | return val 111 | } 112 | 113 | isEmpty (): boolean { 114 | return this.head.isEmpty() 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @packageDocumentation 3 | * 4 | * An iterable that you can push values into. 5 | * 6 | * @example 7 | * 8 | * ```js 9 | * import { pushable } from 'it-pushable' 10 | * 11 | * const source = pushable() 12 | * 13 | * setTimeout(() => source.push('hello'), 100) 14 | * setTimeout(() => source.push('world'), 200) 15 | * setTimeout(() => source.end(), 300) 16 | * 17 | * const start = Date.now() 18 | * 19 | * for await (const value of source) { 20 | * console.log(`got "${value}" after ${Date.now() - start}ms`) 21 | * } 22 | * console.log(`done after ${Date.now() - start}ms`) 23 | * 24 | * // Output: 25 | * // got "hello" after 105ms 26 | * // got "world" after 207ms 27 | * // done after 309ms 28 | * ``` 29 | * 30 | * @example 31 | * 32 | * ```js 33 | * import { pushableV } from 'it-pushable' 34 | * import all from 'it-all' 35 | * 36 | * const source = pushableV() 37 | * 38 | * source.push(1) 39 | * source.push(2) 40 | * source.push(3) 41 | * source.end() 42 | * 43 | * console.info(await all(source)) 44 | * 45 | * // Output: 46 | * // [ [1, 2, 3] ] 47 | * ``` 48 | */ 49 | 50 | import deferred from 'p-defer' 51 | import { FIFO, type Next } from './fifo.js' 52 | 53 | export class AbortError extends Error { 54 | type: string 55 | code: string 56 | 57 | constructor (message?: string, code?: string) { 58 | super(message ?? 'The operation was aborted') 59 | this.type = 'aborted' 60 | this.code = code ?? 'ABORT_ERR' 61 | } 62 | } 63 | 64 | export interface AbortOptions { 65 | signal?: AbortSignal 66 | } 67 | 68 | interface BasePushable { 69 | /** 70 | * End the iterable after all values in the buffer (if any) have been yielded. If an 71 | * error is passed the buffer is cleared immediately and the next iteration will 72 | * throw the passed error 73 | */ 74 | end(err?: Error): this 75 | 76 | /** 77 | * Push a value into the iterable. Values are yielded from the iterable in the order 78 | * they are pushed. Values not yet consumed from the iterable are buffered. 79 | */ 80 | push(value: T): this 81 | 82 | /** 83 | * Returns a promise that resolves when the underlying queue becomes empty (e.g. 84 | * this.readableLength === 0). 85 | * 86 | * If an AbortSignal is passed as an option and that signal aborts, it only 87 | * causes the returned promise to reject - it does not end the pushable. 88 | */ 89 | onEmpty(options?: AbortOptions): Promise 90 | 91 | /** 92 | * This property contains the number of bytes (or objects) in the queue ready to be read. 93 | * 94 | * If `objectMode` is true, this is the number of objects in the queue, if false it's the 95 | * total number of bytes in the queue. 96 | */ 97 | readableLength: number 98 | } 99 | 100 | /** 101 | * An iterable that you can push values into. 102 | */ 103 | export interface Pushable extends AsyncGenerator, BasePushable {} 104 | 105 | /** 106 | * Similar to `pushable`, except it yields multiple buffered chunks at a time. All values yielded from the iterable will be arrays. 107 | */ 108 | export interface PushableV extends AsyncGenerator, BasePushable {} 109 | 110 | export interface Options { 111 | /** 112 | * A boolean value that means non-`Uint8Array`s will be passed to `.push`, default: `false` 113 | */ 114 | objectMode?: boolean 115 | 116 | /** 117 | * A function called after *all* values have been yielded from the iterator (including 118 | * buffered values). In the case when the iterator is ended with an error it will be 119 | * passed the error as a parameter. 120 | */ 121 | onEnd?(err?: Error): void 122 | } 123 | 124 | export interface DoneResult { done: true } 125 | export interface ValueResult { done: false, value: T } 126 | export type NextResult = ValueResult | DoneResult 127 | 128 | interface getNext { (buffer: FIFO): NextResult } 129 | 130 | export interface ObjectPushableOptions extends Options { 131 | objectMode: true 132 | } 133 | 134 | export interface BytePushableOptions extends Options { 135 | objectMode?: false 136 | } 137 | 138 | /** 139 | * Create a new async iterable. The values yielded from calls to `.next()` 140 | * or when used in a `for await of`loop are "pushed" into the iterable. 141 | * Returns an async iterable object with additional methods. 142 | */ 143 | export function pushable (options?: BytePushableOptions): Pushable 144 | export function pushable (options: ObjectPushableOptions): Pushable 145 | export function pushable (options: Options = {}): Pushable { 146 | const getNext = (buffer: FIFO): NextResult => { 147 | const next: Next | undefined = buffer.shift() 148 | 149 | if (next == null) { 150 | return { done: true } 151 | } 152 | 153 | if (next.error != null) { 154 | throw next.error 155 | } 156 | 157 | return { 158 | done: next.done === true, 159 | // @ts-expect-error if done is false, value will be present 160 | value: next.value 161 | } 162 | } 163 | 164 | return _pushable>(getNext, options) 165 | } 166 | 167 | export function pushableV (options?: BytePushableOptions): PushableV 168 | export function pushableV (options: ObjectPushableOptions): PushableV 169 | export function pushableV (options: Options = {}): PushableV { 170 | const getNext = (buffer: FIFO): NextResult => { 171 | let next: Next | undefined 172 | const values: T[] = [] 173 | 174 | while (!buffer.isEmpty()) { 175 | next = buffer.shift() 176 | 177 | if (next == null) { 178 | break 179 | } 180 | 181 | if (next.error != null) { 182 | throw next.error 183 | } 184 | 185 | if (next.done === false) { 186 | // @ts-expect-error if done is false value should be pushed 187 | values.push(next.value) 188 | } 189 | } 190 | 191 | if (next == null) { 192 | return { done: true } 193 | } 194 | 195 | return { 196 | done: next.done === true, 197 | value: values 198 | } 199 | } 200 | 201 | return _pushable>(getNext, options) 202 | } 203 | 204 | function _pushable (getNext: getNext, options?: Options): ReturnType { 205 | options = options ?? {} 206 | let onEnd = options.onEnd 207 | let buffer = new FIFO() 208 | let pushable: any 209 | let onNext: ((next: Next) => ReturnType) | null 210 | let ended: boolean 211 | let drain = deferred() 212 | 213 | const waitNext = async (): Promise> => { 214 | try { 215 | if (!buffer.isEmpty()) { 216 | return getNext(buffer) 217 | } 218 | 219 | if (ended) { 220 | return { done: true } 221 | } 222 | 223 | return await new Promise>((resolve, reject) => { 224 | onNext = (next: Next) => { 225 | onNext = null 226 | buffer.push(next) 227 | 228 | try { 229 | resolve(getNext(buffer)) 230 | } catch (err) { 231 | reject(err) 232 | } 233 | 234 | return pushable 235 | } 236 | }) 237 | } finally { 238 | if (buffer.isEmpty()) { 239 | // settle promise in the microtask queue to give consumers a chance to 240 | // await after calling .push 241 | queueMicrotask(() => { 242 | drain.resolve() 243 | drain = deferred() 244 | }) 245 | } 246 | } 247 | } 248 | 249 | const bufferNext = (next: Next): ReturnType => { 250 | if (onNext != null) { 251 | return onNext(next) 252 | } 253 | 254 | buffer.push(next) 255 | return pushable 256 | } 257 | 258 | const bufferError = (err: Error): ReturnType => { 259 | buffer = new FIFO() 260 | 261 | if (onNext != null) { 262 | return onNext({ error: err }) 263 | } 264 | 265 | buffer.push({ error: err }) 266 | return pushable 267 | } 268 | 269 | const push = (value: PushType): ReturnType => { 270 | if (ended) { 271 | return pushable 272 | } 273 | 274 | // @ts-expect-error `byteLength` is not declared on PushType 275 | if (options?.objectMode !== true && value?.byteLength == null) { 276 | throw new Error('objectMode was not true but tried to push non-Uint8Array value') 277 | } 278 | 279 | return bufferNext({ done: false, value }) 280 | } 281 | const end = (err?: Error): ReturnType => { 282 | if (ended) return pushable 283 | ended = true 284 | 285 | return (err != null) ? bufferError(err) : bufferNext({ done: true }) 286 | } 287 | const _return = (): DoneResult => { 288 | buffer = new FIFO() 289 | end() 290 | 291 | return { done: true } 292 | } 293 | const _throw = (err: Error): DoneResult => { 294 | end(err) 295 | 296 | return { done: true } 297 | } 298 | 299 | pushable = { 300 | [Symbol.asyncIterator] () { return this }, 301 | next: waitNext, 302 | return: _return, 303 | throw: _throw, 304 | push, 305 | end, 306 | get readableLength (): number { 307 | return buffer.size 308 | }, 309 | onEmpty: async (options?: AbortOptions) => { 310 | const signal = options?.signal 311 | signal?.throwIfAborted() 312 | 313 | if (buffer.isEmpty()) { 314 | return 315 | } 316 | 317 | let cancel: Promise | undefined 318 | let listener: (() => void) | undefined 319 | 320 | if (signal != null) { 321 | cancel = new Promise((resolve, reject) => { 322 | listener = () => { 323 | reject(new AbortError()) 324 | } 325 | 326 | signal.addEventListener('abort', listener) 327 | }) 328 | } 329 | 330 | try { 331 | await Promise.race([ 332 | drain.promise, 333 | cancel 334 | ]) 335 | } finally { 336 | if (listener != null && signal != null) { 337 | signal?.removeEventListener('abort', listener) 338 | } 339 | } 340 | } 341 | } 342 | 343 | if (onEnd == null) { 344 | return pushable 345 | } 346 | 347 | const _pushable = pushable 348 | 349 | pushable = { 350 | [Symbol.asyncIterator] () { return this }, 351 | next () { 352 | return _pushable.next() 353 | }, 354 | throw (err: Error) { 355 | _pushable.throw(err) 356 | 357 | if (onEnd != null) { 358 | onEnd(err) 359 | onEnd = undefined 360 | } 361 | 362 | return { done: true } 363 | }, 364 | return () { 365 | _pushable.return() 366 | 367 | if (onEnd != null) { 368 | onEnd() 369 | onEnd = undefined 370 | } 371 | 372 | return { done: true } 373 | }, 374 | push, 375 | end (err: Error) { 376 | _pushable.end(err) 377 | 378 | if (onEnd != null) { 379 | onEnd(err) 380 | onEnd = undefined 381 | } 382 | 383 | return pushable 384 | }, 385 | get readableLength () { 386 | return _pushable.readableLength 387 | }, 388 | onEmpty: (opts?: AbortOptions) => { 389 | return _pushable.onEmpty(opts) 390 | } 391 | } 392 | 393 | return pushable 394 | } 395 | -------------------------------------------------------------------------------- /test/test.spec.ts: -------------------------------------------------------------------------------- 1 | import { expect } from 'aegir/chai' 2 | import all from 'it-all' 3 | import { pipe } from 'it-pipe' 4 | import pDefer from 'p-defer' 5 | import { Uint8ArrayList } from 'uint8arraylist' 6 | import { pushable, pushableV } from '../src/index.js' 7 | 8 | describe('it-pushable', () => { 9 | it('should push input slowly', async () => { 10 | const source = pushable({ 11 | objectMode: true 12 | }) 13 | const input = [1, 2, 3] 14 | for (let i = 0; i < input.length; i++) { 15 | setTimeout(() => source.push(input[i]), i * 10) 16 | } 17 | setTimeout(() => source.end(), input.length * 10) 18 | const output = await pipe(source, async (source) => all(source)) 19 | expect(output).to.deep.equal(input) 20 | }) 21 | 22 | it('should buffer input', async () => { 23 | const source = pushable({ 24 | objectMode: true 25 | }) 26 | const input = [1, 2, 3] 27 | input.forEach(v => source.push(v)) 28 | setTimeout(() => source.end()) 29 | const output = await pipe(source, async (source) => all(source)) 30 | expect(output).to.deep.equal(input) 31 | }) 32 | 33 | it('should buffer falsy input', async () => { 34 | const source = pushable({ 35 | objectMode: true 36 | }) 37 | const input = [1, 2, 3, undefined, null, 0, 4] 38 | input.forEach(v => source.push(v)) 39 | setTimeout(() => source.end()) 40 | const output = await pipe(source, async (source) => all(source)) 41 | expect(output).to.deep.equal(input) 42 | }) 43 | 44 | it('should buffer some inputs', async () => { 45 | const source = pushable({ 46 | objectMode: true 47 | }) 48 | const input = [1, [2.1, 2.2, 2.3], 3, 4, 5, [6.1, 6.2, 6.3, 6.4], 7] 49 | for (let i = 0; i < input.length; i++) { 50 | setTimeout(() => { 51 | if (Array.isArray(input[i])) { 52 | (input[i] as number[]).forEach(v => source.push(v)) 53 | } else { 54 | source.push(input[i]) 55 | } 56 | }, i * 10) 57 | } 58 | setTimeout(() => source.end(), input.length * 10) 59 | const output = await pipe(source, async (source) => all(source)) 60 | 61 | expect(output).to.deep.equal(input.flat()) 62 | }) 63 | 64 | it('should allow end before start', async () => { 65 | const source = pushable({ 66 | objectMode: true 67 | }) 68 | const input = [1, 2, 3] 69 | input.forEach(v => source.push(v)) 70 | source.end() 71 | const output = await pipe(source, async (source) => all(source)) 72 | expect(output).to.deep.equal(input) 73 | }) 74 | 75 | it('should end with error immediately', async () => { 76 | const source = pushable({ 77 | objectMode: true 78 | }) 79 | const input = [1, 2, 3] 80 | input.forEach(v => source.push(v)) 81 | source.end(new Error('boom')) 82 | 83 | await expect(pipe(source, async (source) => all(source))) 84 | .to.eventually.be.rejected.with.property('message', 'boom') 85 | }) 86 | 87 | it('should end with error in the middle', async () => { 88 | const source = pushable({ 89 | objectMode: true 90 | }) 91 | const input = [1, new Error('boom'), 3] 92 | for (let i = 0; i < input.length; i++) { 93 | setTimeout(() => { 94 | if (input[i] instanceof Error) { 95 | source.end(input[i] as Error) 96 | } else { 97 | source.push(input[i]) 98 | } 99 | }, i * 10) 100 | } 101 | setTimeout(() => source.end(), input.length * 10) 102 | 103 | await expect(pipe(source, async (source) => all(source))) 104 | .to.eventually.be.rejected.with.property('message', 'boom') 105 | }) 106 | 107 | it('should allow end without push', async () => { 108 | const source = pushable() 109 | const input: any[] = [] 110 | source.end() 111 | const output = await pipe(source, async (source) => all(source)) 112 | expect(output).to.deep.equal(input) 113 | }) 114 | 115 | it('should allow next after end', async () => { 116 | const source = pushable({ 117 | objectMode: true 118 | }) 119 | const input = [1] 120 | source.push(input[0]) 121 | let next = await source.next() 122 | expect(next.done).to.be.false() 123 | expect(next.value).to.equal(input[0]) 124 | source.end() 125 | next = await source.next() 126 | expect(next.done).to.be.true() 127 | next = await source.next() 128 | expect(next.done).to.be.true() 129 | }) 130 | 131 | it('should call onEnd', (done) => { 132 | const source = pushable({ 133 | objectMode: true, 134 | onEnd: () => { done() } 135 | }) 136 | const input = [1, 2, 3] 137 | for (let i = 0; i < input.length; i++) { 138 | setTimeout(() => source.push(input[i]), i * 10) 139 | } 140 | setTimeout(() => source.end(), input.length * 10) 141 | void pipe(source, async (source) => all(source)) 142 | }) 143 | 144 | it('should call onEnd after onEmpty', async () => { 145 | const ended = pDefer() 146 | const source = pushable({ 147 | objectMode: true, 148 | onEnd: () => { 149 | ended.resolve() 150 | } 151 | }) 152 | source.push(1) 153 | source.push(2) 154 | source.push(3) 155 | source.end() 156 | 157 | await source.onEmpty() 158 | await ended.promise 159 | }) 160 | 161 | it('should call onEnd if passed in options object', (done) => { 162 | const source = pushable({ 163 | objectMode: true, 164 | onEnd: () => { done() } 165 | }) 166 | const input = [1, 2, 3] 167 | for (let i = 0; i < input.length; i++) { 168 | setTimeout(() => source.push(input[i]), i * 10) 169 | } 170 | setTimeout(() => source.end(), input.length * 10) 171 | void pipe(source, async (source) => all(source)) 172 | }) 173 | 174 | it('should call onEnd even if not piped', (done) => { 175 | const source = pushable({ 176 | onEnd: () => { done() } 177 | }) 178 | source.end() 179 | }) 180 | 181 | it('should call onEnd with error', (done) => { 182 | const source = pushable({ 183 | onEnd: err => { 184 | expect(err).to.have.property('message', 'boom') 185 | done() 186 | } 187 | }) 188 | setTimeout(() => source.end(new Error('boom')), 10) 189 | void pipe(source, async (source) => all(source)).catch(() => {}) 190 | }) 191 | 192 | it('should call onEnd on return before end', (done) => { 193 | const input = [1, 2, 3, 4, 5] 194 | const max = 2 195 | const output: number[] = [] 196 | 197 | const source = pushable({ 198 | objectMode: true, 199 | onEnd: () => { 200 | expect(output).to.deep.equal(input.slice(0, max)) 201 | done() 202 | } 203 | }) 204 | 205 | input.forEach((v, i) => setTimeout(() => source.push(v), i * 10)) 206 | setTimeout(() => source.end(), input.length * 10) 207 | 208 | void (async () => { 209 | let i = 0 210 | for await (const value of source) { 211 | output.push(value) 212 | i++ 213 | if (i === max) break 214 | } 215 | })() 216 | }) 217 | 218 | it('should call onEnd by calling return', (done) => { 219 | const input = [1, 2, 3, 4, 5] 220 | const max = 2 221 | const output: number[] = [] 222 | 223 | const source = pushable({ 224 | objectMode: true, 225 | onEnd: () => { 226 | expect(output).to.deep.equal(input.slice(0, max)) 227 | done() 228 | } 229 | }) 230 | 231 | let index = 0 232 | input.forEach((v, i) => { 233 | setTimeout(() => { 234 | source.push(input[index]) 235 | index++ 236 | }, i * 10) 237 | }) 238 | setTimeout(() => source.end(), input.length * 10) 239 | 240 | void (async () => { 241 | let i = 0 242 | while (i !== max) { 243 | i++ 244 | const { value } = await source.next() 245 | 246 | if (value != null) { 247 | output.push(value) 248 | } 249 | } 250 | await source.return() 251 | })() 252 | }) 253 | 254 | it('should call onEnd once', (done) => { 255 | const input = [1, 2, 3, 4, 5] 256 | 257 | let count = 0 258 | const source = pushable({ 259 | objectMode: true, 260 | onEnd: () => { 261 | count++ 262 | expect(count).to.equal(1) 263 | setTimeout(() => { done() }, 50) 264 | } 265 | }) 266 | 267 | input.forEach((v, i) => setTimeout(() => source.push(v), i * 10)) 268 | 269 | void (async () => { 270 | await source.next() 271 | await source.return() 272 | await source.next() 273 | })() 274 | }) 275 | 276 | it('should call onEnd by calling throw', (done) => { 277 | const input = [1, 2, 3, 4, 5] 278 | const max = 2 279 | const output: number[] = [] 280 | 281 | const source = pushable({ 282 | objectMode: true, 283 | onEnd: err => { 284 | expect(err).to.have.property('message', 'boom') 285 | expect(output).to.deep.equal(input.slice(0, max)) 286 | done() 287 | } 288 | }) 289 | 290 | input.forEach((v, i) => setTimeout(() => source.push(v), i * 10)) 291 | setTimeout(() => source.end(), input.length * 10) 292 | 293 | void (async () => { 294 | let i = 0 295 | while (i !== max) { 296 | i++ 297 | const { value } = await source.next() 298 | 299 | if (value != null) { 300 | output.push(value) 301 | } 302 | } 303 | await source.throw(new Error('boom')) 304 | })() 305 | }) 306 | 307 | it('should support writev', async () => { 308 | const source = pushableV({ 309 | objectMode: true 310 | }) 311 | const input = [1, 2, 3] 312 | input.forEach(v => source.push(v)) 313 | setTimeout(() => source.end()) 314 | const output = await pipe(source, async (source) => all(source)) 315 | expect(output[0]).to.deep.equal(input) 316 | }) 317 | 318 | it('should always yield arrays when using writev', async () => { 319 | const source = pushableV({ 320 | objectMode: true 321 | }) 322 | const input = [1, 2, 3] 323 | setTimeout(() => { 324 | input.forEach(v => source.push(v)) 325 | setTimeout(() => source.end()) 326 | }) 327 | const output = await pipe(source, async (source) => all(source)) 328 | output.forEach(v => expect(Array.isArray(v)).to.be.true()) 329 | }) 330 | 331 | it('should support writev and end with error', async () => { 332 | const source = pushableV({ 333 | objectMode: true 334 | }) 335 | const input = [1, 2, 3] 336 | input.forEach(v => source.push(v)) 337 | source.end(new Error('boom')) 338 | 339 | await expect(pipe(source, async (source) => all(source))) 340 | .to.eventually.be.rejected.with.property('message', 'boom') 341 | }) 342 | 343 | it('should support readableLength for objects', async () => { 344 | const source = pushable({ 345 | objectMode: true 346 | }) 347 | 348 | expect(source).to.have.property('readableLength', 0) 349 | 350 | source.push(1) 351 | expect(source).to.have.property('readableLength', 1) 352 | 353 | source.push(1) 354 | expect(source).to.have.property('readableLength', 2) 355 | 356 | await source.next() 357 | expect(source).to.have.property('readableLength', 1) 358 | 359 | await source.next() 360 | expect(source).to.have.property('readableLength', 0) 361 | }) 362 | 363 | it('should support readableLength for bytes', async () => { 364 | const source = pushable() 365 | 366 | expect(source).to.have.property('readableLength', 0) 367 | 368 | source.push(Uint8Array.from([1, 2])) 369 | expect(source).to.have.property('readableLength', 2) 370 | 371 | source.push(Uint8Array.from([3, 4, 5])) 372 | expect(source).to.have.property('readableLength', 5) 373 | 374 | await source.next() 375 | expect(source).to.have.property('readableLength', 3) 376 | 377 | await source.next() 378 | expect(source).to.have.property('readableLength', 0) 379 | }) 380 | 381 | it('should support readableLength for Uint8ArrayLists', async () => { 382 | const source = pushable() 383 | 384 | expect(source).to.have.property('readableLength', 0) 385 | 386 | source.push(new Uint8ArrayList(Uint8Array.from([1, 2]))) 387 | expect(source).to.have.property('readableLength', 2) 388 | 389 | source.push(new Uint8ArrayList(Uint8Array.from([3, 4, 5]))) 390 | expect(source).to.have.property('readableLength', 5) 391 | 392 | await source.next() 393 | expect(source).to.have.property('readableLength', 3) 394 | 395 | await source.next() 396 | expect(source).to.have.property('readableLength', 0) 397 | }) 398 | 399 | it('should support readableLength for mixed Uint8ArrayLists and Uint8Arrays', async () => { 400 | const source = pushable() 401 | 402 | expect(source).to.have.property('readableLength', 0) 403 | 404 | source.push(new Uint8ArrayList(Uint8Array.from([1, 2]))) 405 | expect(source).to.have.property('readableLength', 2) 406 | 407 | source.push(Uint8Array.from([3, 4, 5])) 408 | expect(source).to.have.property('readableLength', 5) 409 | 410 | await source.next() 411 | expect(source).to.have.property('readableLength', 3) 412 | 413 | await source.next() 414 | expect(source).to.have.property('readableLength', 0) 415 | }) 416 | 417 | it('should throw if passed an object when objectMode is not true', async () => { 418 | const source = pushable() 419 | 420 | // @ts-expect-error incorrect argument type 421 | expect(() => source.push('hello')).to.throw().with.property('message').that.includes('tried to push non-Uint8Array value') 422 | }) 423 | 424 | it('should return from onEmpty when the queue is empty', async () => { 425 | const source = pushableV({ 426 | objectMode: true 427 | }) 428 | 429 | await expect(source.onEmpty()).to.eventually.be.undefined() 430 | }) 431 | 432 | it('should return from onEmpty when the pushable becomes empty', async () => { 433 | const source = pushable({ 434 | objectMode: true 435 | }) 436 | 437 | source.push(1) 438 | 439 | let resolved = false 440 | const onEmptyPromise = source.onEmpty().then(() => { 441 | resolved = true 442 | }) 443 | 444 | expect(resolved).to.be.false() 445 | 446 | source.push(2) 447 | expect(resolved).to.be.false() 448 | 449 | await source.next() 450 | expect(resolved).to.be.false() 451 | 452 | await source.next() 453 | await onEmptyPromise 454 | expect(resolved).to.be.true() 455 | }) 456 | 457 | it('should return from onEmpty when the pushableV becomes empty', async () => { 458 | const source = pushableV({ 459 | objectMode: true 460 | }) 461 | 462 | source.push(1) 463 | 464 | let resolved = false 465 | 466 | const onEmptyPromise = source.onEmpty().then(() => { 467 | resolved = true 468 | }) 469 | 470 | expect(resolved).to.be.false() 471 | 472 | source.push(2) 473 | expect(resolved).to.be.false() 474 | 475 | await source.next() 476 | await onEmptyPromise 477 | expect(resolved).to.be.true() 478 | }) 479 | 480 | it('should reject from onEmpty when the passed abort signal is aborted', async () => { 481 | const source = pushable({ 482 | objectMode: true 483 | }) 484 | 485 | source.push(1) 486 | 487 | const controller = new AbortController() 488 | const p = source.onEmpty({ signal: controller.signal }) 489 | 490 | source.push(2) 491 | 492 | controller.abort() 493 | 494 | await expect(p).to.eventually.be.rejected 495 | .with.property('code', 'ABORT_ERR') 496 | }) 497 | }) 498 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src", 8 | "test" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "entryPoints": [ 3 | "./src/index.ts" 4 | ] 5 | } 6 | --------------------------------------------------------------------------------