├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ └── bevry.yml ├── index.cjs ├── test.cjs ├── bin.cjs ├── example.cjs ├── tsconfig.json ├── source ├── bin.ts ├── worker.ts ├── test.ts └── index.ts ├── .editorconfig ├── .prettierignore ├── .gitignore ├── .npmignore ├── SECURITY.md ├── CONTRIBUTING.md ├── HISTORY.md ├── LICENSE.md ├── package.json └── README.md /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [balupton] 2 | patreon: bevry 3 | open_collective: bevry 4 | ko_fi: balupton 5 | liberapay: bevry 6 | tidelift: npm/readdir-cluster 7 | custom: ['https://bevry.me/fund'] 8 | -------------------------------------------------------------------------------- /index.cjs: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | // auto-generated by boundation, do not update manually 3 | /** @type {typeof import("./edition-types/index.d.ts") } */ 4 | module.exports = require('editions').requirePackage(__dirname, require, 'index.js') -------------------------------------------------------------------------------- /test.cjs: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | // auto-generated by boundation, do not update manually 3 | /** @type {typeof import("./edition-types/test.d.ts") } */ 4 | module.exports = require('editions').requirePackage(__dirname, require, 'test.js') -------------------------------------------------------------------------------- /bin.cjs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict' 3 | // auto-generated by boundation, do not update manually 4 | /** @type {typeof import("./edition-types/bin.d.ts") } */ 5 | module.exports = require('editions').requirePackage(__dirname, require, 'bin.js') -------------------------------------------------------------------------------- /example.cjs: -------------------------------------------------------------------------------- 1 | var path = __dirname 2 | if (process.argv.indexOf('old') === -1) { 3 | console.log('using cluster method') 4 | require('./')(path, console.log, console.log) 5 | } else { 6 | console.log('using old method') 7 | require('scandirectory')(path, console.log) 8 | } 9 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowJs": true, 4 | "downlevelIteration": true, 5 | "esModuleInterop": true, 6 | "isolatedModules": true, 7 | "maxNodeModuleJsDepth": 5, 8 | "module": "ESNext", 9 | "moduleResolution": "Node", 10 | "strict": true, 11 | "target": "ES2022" 12 | }, 13 | "include": ["source"] 14 | } 15 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: / 5 | schedule: 6 | interval: weekly 7 | day: sunday 8 | time: '00:00' 9 | timezone: Australia/Perth 10 | - package-ecosystem: npm 11 | directory: / 12 | schedule: 13 | interval: weekly 14 | day: sunday 15 | time: '00:00' 16 | timezone: Australia/Perth 17 | open-pull-requests-limit: 0 18 | -------------------------------------------------------------------------------- /source/bin.ts: -------------------------------------------------------------------------------- 1 | // builtin 2 | import { argv, stdout, exit } from 'process' 3 | import { resolve } from 'path' 4 | 5 | // local 6 | import readdirCluster from './index.js' 7 | 8 | // for each path, readdir 9 | for (const path of argv.slice(2)) { 10 | readdirCluster({ directory: path }) 11 | .then((paths) => { 12 | if (paths.length) { 13 | stdout.write(paths.join('\n') + '\n') 14 | } 15 | }) 16 | .catch((error) => { 17 | console.error(error) 18 | exit(1) 19 | }) 20 | } 21 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # 2023 June 22 2 | # https://github.com/bevry/base 3 | 4 | root = true 5 | 6 | [*] 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = false 11 | indent_style = tab 12 | 13 | [{*.mk,*.py}] 14 | indent_style = tab 15 | indent_size = 4 16 | 17 | [*.md] 18 | indent_style = space 19 | indent_size = 4 20 | 21 | [{*.json,*.lsrules,*.yaml,*.yml,*.bowerrc,*.babelrc,*.code-workspace}] 22 | indent_style = space 23 | indent_size = 2 24 | 25 | [{*.json,*.lsrules}] 26 | insert_final_newline = true 27 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | # 2023 November 13 2 | # https://github.com/bevry/base 3 | 4 | # VCS Files 5 | .git 6 | .svn 7 | .hg 8 | 9 | # System Files 10 | **/.DS_Store 11 | 12 | # Temp Files 13 | **/.docpad.db 14 | **/*.log 15 | **/*.cpuprofile 16 | **/*.heapsnapshot 17 | 18 | # Yarn Files 19 | .yarn/* 20 | !.yarn/releases 21 | !.yarn/plugins 22 | !.yarn/sdks 23 | !.yarn/versions 24 | .pnp.* 25 | .pnp/ 26 | 27 | # Build Caches 28 | build/ 29 | components/ 30 | bower_components/ 31 | node_modules/ 32 | 33 | # Build Outputs 34 | **/*.cjs 35 | **/*.mjs 36 | **/out.* 37 | **/*.out.* 38 | **/out/ 39 | **/output/ 40 | *compiled* 41 | edition*/ 42 | coffeejs/ 43 | coffee/ 44 | es5/ 45 | es2015/ 46 | esnext/ 47 | docs/ 48 | 49 | # Development Files 50 | test/ 51 | **/*fixtures* 52 | 53 | # Ecosystem Caches 54 | .trunk/*/ 55 | 56 | # ===================================== 57 | # CUSTOM 58 | 59 | # None 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # 2020 June 3 2 | # https://github.com/bevry/base 3 | 4 | # System Files 5 | **/.DS_Store 6 | 7 | # Temp Files 8 | **/.docpad.db 9 | **/*.log 10 | **/*.cpuprofile 11 | **/*.heapsnapshot 12 | 13 | # Editor Files 14 | .c9/ 15 | .vscode/ 16 | 17 | # Yarn Files 18 | .yarn/* 19 | !.yarn/releases 20 | !.yarn/plugins 21 | !.yarn/sdks 22 | !.yarn/versions 23 | .pnp.* 24 | .pnp/ 25 | 26 | # Private Files 27 | .env 28 | .idea 29 | .cake_task_cache 30 | 31 | # Build Caches 32 | build/ 33 | bower_components/ 34 | node_modules/ 35 | .next/ 36 | 37 | # ------------------------------------- 38 | # CDN Inclusions, Git Exclusions 39 | 40 | # Build Outputs 41 | **/out.* 42 | **/*.out.* 43 | **/out/ 44 | **/output/ 45 | *compiled* 46 | edition*/ 47 | coffeejs/ 48 | coffee/ 49 | es5/ 50 | es2015/ 51 | esnext/ 52 | docs/ 53 | 54 | # ===================================== 55 | # CUSTOM 56 | 57 | # None 58 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | # 2020 May 5 2 | # https://github.com/bevry/base 3 | 4 | # System Files 5 | **/.DS_Store 6 | 7 | # Temp Files 8 | **/.docpad.db 9 | **/*.log 10 | **/*.cpuprofile 11 | **/*.heapsnapshot 12 | 13 | # Editor Files 14 | .c9/ 15 | .vscode/ 16 | 17 | # Private Files 18 | .env 19 | .idea 20 | .cake_task_cache 21 | 22 | # Build Caches 23 | build/ 24 | components/ 25 | bower_components/ 26 | node_modules/ 27 | .pnp/ 28 | .pnp.js 29 | 30 | # Ecosystem Files 31 | .dependabout 32 | .github 33 | 34 | # ------------------------------------- 35 | # CDN Inclusions, Package Exclusions 36 | 37 | # Documentation Files 38 | docs/ 39 | guides/ 40 | BACKERS.md 41 | CONTRIBUTING.md 42 | HISTORY.md 43 | 44 | # Development Files 45 | web/ 46 | **/example* 47 | **/test* 48 | .babelrc* 49 | .editorconfig 50 | .eslintrc* 51 | .jshintrc 52 | .jscrc 53 | coffeelint* 54 | .travis* 55 | nakefile* 56 | Cakefile 57 | Makefile 58 | 59 | # Other Package Definitions 60 | template.js 61 | component.json 62 | bower.json 63 | 64 | # ===================================== 65 | # CUSTOM MODIFICATIONS 66 | 67 | # None 68 | -------------------------------------------------------------------------------- /source/worker.ts: -------------------------------------------------------------------------------- 1 | // builtin 2 | import { readdir, stat } from 'fs' 3 | import process from 'process' 4 | 5 | // local 6 | import { Message, Action } from './index.js' 7 | 8 | // listen 9 | process.on('message', function (message: Message) { 10 | // console.log('worker', message.id) 11 | const { id, action, path } = message 12 | if (action === Action.readdir) { 13 | readdir(path, function (error, files) { 14 | if (error) 15 | return process.send!({ 16 | id, 17 | action, 18 | path, 19 | error, 20 | }) 21 | process.send!({ 22 | id, 23 | action, 24 | path, 25 | data: files, 26 | }) 27 | }) 28 | } else if (action === Action.stat) { 29 | stat(path, function (error, stat) { 30 | if (error) 31 | return process.send!({ 32 | id, 33 | action, 34 | path, 35 | error, 36 | }) 37 | process.send!({ 38 | id, 39 | action, 40 | path, 41 | data: Object.assign({ directory: stat.isDirectory() }, stat), 42 | }) 43 | }) 44 | } else { 45 | process.send!({ 46 | id, 47 | action, 48 | path, 49 | error: new Error('unknown action'), 50 | }) 51 | } 52 | }) 53 | -------------------------------------------------------------------------------- /source/test.ts: -------------------------------------------------------------------------------- 1 | // external 2 | import kava from 'kava' 3 | import { deepEqual } from 'assert-helpers' 4 | import filedirname from 'filedirname' 5 | 6 | // builtin 7 | import { resolve, join } from 'path' 8 | 9 | // local 10 | import readdirCluster, { Paths, Stat } from './index.js' 11 | 12 | // prepare 13 | const [file, dir] = filedirname() 14 | const rootPath = resolve(dir, '..') 15 | const sourcePath = resolve(dir, '..', 'source') 16 | 17 | // Test 18 | kava.suite('readdir-cluster', function (suite, test) { 19 | test('works on source directory with test filter', function (done) { 20 | const filteredPathsActual: Paths = [] 21 | const filteredPathsExpected: Paths = [ 22 | sourcePath, 23 | join(sourcePath, 'bin.ts'), 24 | join(sourcePath, 'index.ts'), 25 | join(sourcePath, 'worker.ts'), 26 | ].sort() 27 | function iterator( 28 | path: string, 29 | filename: string, 30 | stat: Stat 31 | ): boolean | void { 32 | let iterate = false 33 | if (stat.directory) { 34 | if (path === sourcePath && filename === 'source') iterate = true 35 | } else if (path.startsWith(sourcePath) && filename !== 'test.ts') 36 | iterate = true 37 | if (iterate) filteredPathsActual.push(path) 38 | return iterate 39 | } 40 | readdirCluster({ directory: rootPath, iterator }) 41 | .then(function (filteredPaths) { 42 | deepEqual( 43 | filteredPaths.sort(), 44 | filteredPathsExpected, 45 | 'filtered paths result was as expected' 46 | ) 47 | deepEqual( 48 | filteredPaths.sort(), 49 | filteredPathsActual.sort(), 50 | 'filtered paths result was as actual' 51 | ) 52 | done() 53 | }) 54 | .catch(function (error) { 55 | done(error) 56 | }) 57 | }) 58 | }) 59 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Security Practices 4 | 5 | This project meets standardized secure software development practices, including 2FA for all members, password managers with monitoring, secure secret retrieval instead of storage. [Learn about our practices.](https://tidelift.com/funding/github/npm/readdir-cluster) 6 | 7 | ## Supported Versions 8 | 9 | This project uses [Bevry's automated tooling](https://github.com/bevry/boundation) to deliver the latest updates, fixes, and improvements inside the latest release while still maintaining widespread ecosystem compatibility. 10 | 11 | [Refer to supported ecosystem versions: `Editions` section in `README.md`](https://github.com/bevry/readdir-cluster/blob/master/README.md#Editions) 12 | 13 | [Refer to automated support of ecosystem versions: `boundation` entries in `HISTORY.md`](https://github.com/bevry/readdir-cluster/blob/master/HISTORY.md) 14 | 15 | Besides testing and verification, out CI also [auto-merges](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions) [Dependabot security updates](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/about-dependabot-security-updates) and [auto-publishes](https://github.com/bevry-actions/npm) successful builds of the [`master` branch](https://github.com/bevry/wait/actions?query=branch%3Amaster) to the [`next` version tag](https://www.npmjs.com/package/readdir-cluster?activeTab=versions), offering immediate resolutions before scheduled maintenance releases. 16 | 17 | ## Reporting a Vulnerability 18 | 19 | [Report the vulnerability to the project owners.](https://github.com/bevry/readdir-cluster/security/advisories) 20 | 21 | [Report the vulnerability to Tidelift.](https://tidelift.com/security) 22 | -------------------------------------------------------------------------------- /.github/workflows/bevry.yml: -------------------------------------------------------------------------------- 1 | name: bevry 2 | 'on': 3 | - push 4 | - pull_request 5 | jobs: 6 | test: 7 | strategy: 8 | matrix: 9 | os: 10 | - ubuntu-latest 11 | - macos-latest 12 | - windows-latest 13 | node: 14 | - '16' 15 | - '18' 16 | - '20' 17 | - '21' 18 | runs-on: ${{ matrix.os }} 19 | continue-on-error: ${{ contains('macos-latest windows-latest', matrix.os) }} 20 | steps: 21 | - uses: actions/checkout@v6 22 | - name: Install desired Node.js version 23 | uses: actions/setup-node@v6 24 | with: 25 | node-version: '20' 26 | - name: Verify Node.js Versions 27 | run: >- 28 | printf '%s' 'node: ' && node --version && printf '%s' 'npm: ' && npm 29 | --version && node -e 'console.log(process.versions)' 30 | - run: npm run our:setup 31 | - run: npm run our:compile 32 | - run: npm run our:verify 33 | - name: Install targeted Node.js 34 | if: ${{ matrix.node != 20 }} 35 | uses: actions/setup-node@v6 36 | with: 37 | node-version: ${{ matrix.node }} 38 | - name: Verify Node.js Versions 39 | run: >- 40 | printf '%s' 'node: ' && node --version && printf '%s' 'npm: ' && npm 41 | --version && node -e 'console.log(process.versions)' 42 | - run: npm test 43 | publish: 44 | if: ${{ github.event_name == 'push' }} 45 | needs: test 46 | runs-on: ubuntu-latest 47 | steps: 48 | - uses: actions/checkout@v6 49 | - name: Install desired Node.js version 50 | uses: actions/setup-node@v6 51 | with: 52 | node-version: '20' 53 | - name: Verify Node.js Versions 54 | run: >- 55 | printf '%s' 'node: ' && node --version && printf '%s' 'npm: ' && npm 56 | --version && node -e 'console.log(process.versions)' 57 | - run: npm run our:setup 58 | - run: npm run our:compile 59 | - run: npm run our:meta 60 | - name: publish to npm 61 | uses: bevry-actions/npm@v1.1.7 62 | with: 63 | npmAuthToken: ${{ secrets.NPM_AUTH_TOKEN }} 64 | npmBranchTag: ':next' 65 | - name: publish to surge 66 | uses: bevry-actions/surge@v1.1.0 67 | with: 68 | surgeLogin: ${{ secrets.SURGE_LOGIN }} 69 | surgeToken: ${{ secrets.SURGE_TOKEN }} 70 | automerge: 71 | permissions: 72 | contents: write 73 | pull-requests: write 74 | runs-on: ubuntu-latest 75 | if: github.actor == 'dependabot[bot]' 76 | steps: 77 | - name: Enable auto-merge for Dependabot PRs 78 | run: gh pr merge --auto --squash "$PR_URL" 79 | env: 80 | PR_URL: ${{github.event.pull_request.html_url}} 81 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 82 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | # Before You Post! 7 | 8 | ## Support 9 | 10 | We offer support through our [Official Support Channels](https://bevry.me/support). Do not use GitHub Issues for support, your issue will be closed. 11 | 12 | ## Contribute 13 | 14 | Our [Contributing Guide](https://bevry.me/contribute) contains useful tips and suggestions for how to contribute to this project, it's worth the read. 15 | 16 | ## Development 17 | 18 | ### Setup 19 | 20 | 1. [Install Node.js](https://bevry.me/install/node) 21 | 22 | 1. Fork the project and clone your fork - [guide](https://help.github.com/articles/fork-a-repo/) 23 | 24 | 1. Setup the project for development 25 | 26 | ```bash 27 | npm run our:setup 28 | ``` 29 | 30 | ### Developing 31 | 32 | 1. Compile changes 33 | 34 | ```bash 35 | npm run our:compile 36 | ``` 37 | 38 | 1. Run tests 39 | 40 | ```bash 41 | npm test 42 | ``` 43 | 44 | ### Publishing 45 | 46 | Follow these steps in order to implement your changes/improvements into your desired project: 47 | 48 | #### Preparation 49 | 50 | 1. Make sure your changes are on their own branch that is branched off from master. 51 | 52 | 1. You can do this by: `git checkout master; git checkout -b your-new-branch` 53 | 1. And push the changes up by: `git push origin your-new-branch` 54 | 55 | 1. Ensure all tests pass: 56 | 57 | ```bash 58 | npm test 59 | ``` 60 | 61 | > If possible, add tests for your change, if you don't know how, mention this in your pull request 62 | 63 | 1. Ensure the project is ready for publishing: 64 | 65 | ``` 66 | npm run our:release:prepare 67 | ``` 68 | 69 | #### Pull Request 70 | 71 | To send your changes for the project owner to merge in: 72 | 73 | 1. Submit your pull request 74 | 1. When submitting, if the original project has a `dev` or `integrate` branch, use that as the target branch for your pull request instead of the default `master` 75 | 1. By submitting a pull request you agree for your changes to have the same license as the original plugin 76 | 77 | #### Publish 78 | 79 | To publish your changes as the project owner: 80 | 81 | 1. Switch to the master branch: 82 | 83 | ```bash 84 | git checkout master 85 | ``` 86 | 87 | 1. Merge in the changes of the feature branch (if applicable) 88 | 89 | 1. Increment the version number in the `package.json` file according to the [semantic versioning](http://semver.org) standard, that is: 90 | 91 | 1. `x.0.0` MAJOR version when you make incompatible API changes (note: DocPad plugins must use v2 as the major version, as v2 corresponds to the current DocPad v6.x releases) 92 | 1. `x.y.0` MINOR version when you add functionality in a backwards-compatible manner 93 | 1. `x.y.z` PATCH version when you make backwards-compatible bug fixes 94 | 95 | 1. Add an entry to the changelog following the format of the previous entries, an example of this is: 96 | 97 | ```markdown 98 | ## v6.29.0 2013 April 1 99 | 100 | - Progress on [issue #474](https://github.com/docpad/docpad/issues/474) 101 | - DocPad will now set permissions based on the process's ability 102 | - Thanks to [Avi Deitcher](https://github.com/deitch), [Stephan Lough](https://github.com/stephanlough) for [issue #165](https://github.com/docpad/docpad/issues/165) 103 | - Updated dependencies 104 | ``` 105 | 106 | 1. Commit the changes with the commit title set to something like `v6.29.0. Bugfix. Improvement.` and commit description set to the changelog entry 107 | 108 | 1. Ensure the project is ready for publishing: 109 | 110 | ``` 111 | npm run our:release:prepare 112 | ``` 113 | 114 | 1. Prepare the release and publish it to npm and git: 115 | 116 | ```bash 117 | npm run our:release 118 | ``` 119 | -------------------------------------------------------------------------------- /source/index.ts: -------------------------------------------------------------------------------- 1 | // external 2 | import filedirname from 'filedirname' 3 | 4 | // builtin 5 | import { join } from 'path' 6 | import * as os from 'os' // { availableParallelism } not available on node.js <18 7 | import cluster, { Worker } from 'cluster' 8 | import { Stats } from 'fs' 9 | 10 | // prepare 11 | const [file, dir] = filedirname() 12 | const workerPath = join(dir, 'worker.js') 13 | 14 | export enum Action { 15 | readdir = 'readdir', 16 | stat = 'stat', 17 | } 18 | 19 | export type Paths = Array 20 | export type Stat = Omit< 21 | Stats, 22 | | 'isFile' 23 | | 'isDirectory' 24 | | 'isBlockDevice' 25 | | 'isCharacterDevice' 26 | | 'isSymbolicLink' 27 | | 'isFIFO' 28 | | 'isSocket' 29 | > & { directory: boolean } 30 | 31 | export interface MessageRequest { 32 | id: number 33 | action: Action 34 | path: string 35 | error?: undefined 36 | data?: undefined 37 | } 38 | export interface MessageFailure { 39 | id: number 40 | action: Action 41 | path: string 42 | error: Error 43 | data: undefined 44 | } 45 | export interface MessageReaddir { 46 | id: number 47 | action: Action.readdir 48 | path: string 49 | error: undefined 50 | data: Paths 51 | } 52 | export interface MessageStat { 53 | id: number 54 | action: Action.stat 55 | path: string 56 | error: undefined 57 | data: Stat 58 | } 59 | export type Message = 60 | | MessageRequest 61 | | MessageFailure 62 | | MessageReaddir 63 | | MessageStat 64 | 65 | export type Iterator = ( 66 | path: string, 67 | filename: string, 68 | stat: Stat 69 | ) => boolean | void 70 | 71 | export interface Options { 72 | /** The directory to read */ 73 | directory: string 74 | 75 | /** A custom iterator, return false to ignore the path. */ 76 | iterator?: Iterator 77 | 78 | /** How long in milliseconds until we try the path again? Defaults to 10 */ 79 | retryDelay?: number 80 | } 81 | 82 | export default async function readdirCluster({ 83 | directory, 84 | iterator, 85 | retryDelay, 86 | }: Options): Promise { 87 | // handle the messages 88 | const resolvers = new Map() 89 | let resolverCursor = 0 90 | function sendMessageAndWait( 91 | action: Action.readdir, 92 | path: string 93 | ): Promise 94 | function sendMessageAndWait( 95 | action: Action.stat, 96 | path: string 97 | ): Promise 98 | function sendMessageAndWait( 99 | action: Action, 100 | path: string 101 | ): Promise { 102 | return new Promise(function (resolve, reject) { 103 | ++resolverCursor 104 | resolvers.set(resolverCursor, resolve) 105 | const message: MessageRequest = { id: resolverCursor, action, path } 106 | function sendAndTimeout() { 107 | const worker = nextWorker() 108 | if (!worker) { 109 | reject(new Error('no workers available')) 110 | return 111 | } 112 | worker.send(message) 113 | setTimeout(function () { 114 | const resolver = resolvers.has(message.id) 115 | // console.log('yo', message.id, resolver) 116 | if (resolver) { 117 | // console.log('trying again', message.id, worker) 118 | sendAndTimeout() 119 | } 120 | }, retryDelay ?? 10) 121 | } 122 | sendAndTimeout() 123 | }) 124 | } 125 | function receiveMessageAndResume(message: Message) { 126 | const resolver = resolvers.get(message.id) 127 | // console.log('resolve', message.id, resolver, resolvers.keys()) 128 | if (!resolver) { 129 | // console.log('already completed', message.id) 130 | } else { 131 | resolvers.delete(message.id) 132 | resolver(message) 133 | } 134 | } 135 | // handle the workers 136 | const workersCount = os.availableParallelism 137 | ? os.availableParallelism() 138 | : os.cpus().length || 4 139 | const workers: Array = [] 140 | function openWorker() { 141 | return new Promise(function (resolve, reject) { 142 | const worker = cluster.fork() 143 | workers.push(worker) 144 | worker.on('online', resolve) 145 | worker.on('message', receiveMessageAndResume) 146 | // worker.on('error', () => {}) // ignore, as it is likely just a complaint about a retry sending a message once we have disconnected 147 | // worker.on('disconnect', console.error.bind(console, 'worker disconnect')) 148 | // worker.on('exit', console.error.bind(console, 'worker exit')) 149 | }) 150 | } 151 | async function openWorkers() { 152 | ;(cluster.setupPrimary || cluster.setupMaster)({ exec: workerPath }) 153 | for (let i = 0; i < workersCount; i++) await openWorker() 154 | } 155 | function closeWorkers() { 156 | // console.log('closing workers') 157 | for (const worker of workers) { 158 | worker.destroy() 159 | } 160 | } 161 | function nextWorker(): Worker | null { 162 | const worker = workers.shift() 163 | if (!worker) { 164 | // ignore, we are shutting down 165 | } else if (worker.isConnected()) { 166 | // re-add the worker to make it available 167 | workers.push(worker) 168 | } else { 169 | // the worker died, don't re-add it 170 | } 171 | return worker || null 172 | } 173 | // readdir 174 | async function readdir(directory: string): Promise { 175 | // prepare 176 | const results: Paths = [] 177 | const message: MessageReaddir | MessageFailure = await sendMessageAndWait( 178 | Action.readdir, 179 | directory 180 | ) 181 | if (message.error) return Promise.reject(message.error) 182 | await Promise.all( 183 | message.data.map(async (file) => { 184 | const path = join(directory, file) 185 | const stat: MessageStat | MessageFailure = await sendMessageAndWait( 186 | Action.stat, 187 | path 188 | ) 189 | if (stat.error) return Promise.reject(stat.error) 190 | const iterate = iterator && iterator(path, file, stat.data) 191 | if (iterate === false) return 192 | results.push(path) 193 | if (stat.data.directory) { 194 | const subpaths = await readdir(path) 195 | results.push(...subpaths) 196 | } 197 | }) 198 | ) 199 | return results 200 | } 201 | // process 202 | try { 203 | await openWorkers() 204 | const results = await readdir(directory) 205 | closeWorkers() 206 | return results 207 | } catch (error) { 208 | closeWorkers() 209 | return Promise.reject(error) 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /HISTORY.md: -------------------------------------------------------------------------------- 1 | # History 2 | 3 | ## v6.0.0 2023 December 30 4 | 5 | - Rewrote in TypeScript and changed API to use promises and accept a single options object 6 | - Also improve the reliability by using modern APIs when available (with fallbacks when not available), as well as a customisable retry delay 7 | - Added an extremely basic CLI 8 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 9 | - Thank you to the sponsors: [Andrew Nesbitt](https://nesbitt.io), [Balsa](https://balsa.com), [Codecov](https://codecov.io), [Poonacha Medappa](https://poonachamedappa.com), [Rob Morris](https://github.com/Rob-Morris), [Sentry](https://sentry.io), [Syntax](https://syntax.fm) 10 | 11 | ## v5.3.0 2023 December 6 12 | 13 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 14 | 15 | ## v5.2.0 2023 November 25 16 | 17 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 18 | 19 | ## v5.1.0 2023 November 21 20 | 21 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 22 | 23 | ## v5.0.0 2023 November 15 24 | 25 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 26 | - Minimum required Node.js version changed from `node: >=8` to `node: >=4` adapting to ecosystem changes 27 | 28 | ## v4.0.0 2023 November 14 29 | 30 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 31 | - Minimum required Node.js version changed from `node: >=10` to `node: >=8` adapting to ecosystem changes 32 | 33 | ## v3.18.0 2023 November 2 34 | 35 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 36 | - Updated license from [`MIT`](http://spdx.org/licenses/MIT.html) to [`Artistic-2.0`](http://spdx.org/licenses/Artistic-2.0.html) 37 | 38 | ## v3.17.0 2021 July 30 39 | 40 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 41 | 42 | ## v3.16.0 2021 July 29 43 | 44 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 45 | 46 | ## v3.15.0 2021 July 28 47 | 48 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 49 | 50 | ## v3.14.0 2020 October 29 51 | 52 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 53 | 54 | ## v3.13.0 2020 September 5 55 | 56 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 57 | 58 | ## v3.12.0 2020 August 18 59 | 60 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 61 | 62 | ## v3.11.0 2020 August 4 63 | 64 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 65 | 66 | ## v3.10.0 2020 July 23 67 | 68 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 69 | 70 | ## v3.9.0 2020 June 25 71 | 72 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 73 | 74 | ## v3.8.0 2020 June 22 75 | 76 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 77 | 78 | ## v3.7.0 2020 June 21 79 | 80 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 81 | 82 | ## v3.6.0 2020 June 20 83 | 84 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 85 | 86 | ## v3.5.0 2020 June 11 87 | 88 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 89 | 90 | ## v3.4.0 2020 June 10 91 | 92 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 93 | 94 | ## v3.3.0 2020 May 22 95 | 96 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 97 | 98 | ## v3.2.0 2020 May 21 99 | 100 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 101 | 102 | ## v3.1.0 2020 May 21 103 | 104 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 105 | 106 | ## v3.0.0 2020 May 11 107 | 108 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 109 | - Minimum required node version changed from `node: >=8` to `node: >=10` to keep up with mandatory ecosystem changes 110 | 111 | ## v2.3.0 2019 December 10 112 | 113 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 114 | 115 | ## v2.2.0 2019 December 1 116 | 117 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 118 | 119 | ## v2.1.0 2019 December 1 120 | 121 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 122 | 123 | ## v2.0.0 2019 November 18 124 | 125 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 126 | - Minimum required node version changed from `node: >=0.12` to `node: >=8` to keep up with mandatory ecosystem changes 127 | 128 | ## v1.3.0 2019 November 13 129 | 130 | - Updated dependencies, [base files](https://github.com/bevry/base), and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 131 | 132 | ## v1.2.0 2019 January 1 133 | 134 | - Updated [base files](https://github.com/bevry/base) and [editions](https://editions.bevry.me) using [boundation](https://github.com/bevry/boundation) 135 | 136 | ## v1.1.0 2016 May 28 137 | 138 | - Updated internal conventions 139 | - Moved from [ESNextGuardian](https://github.com/bevry/esnextguardian) to [Editions](https://github.com/bevry/editions) 140 | 141 | ## v1.0.1 2015 December 10 142 | 143 | - Updated internal conventions 144 | 145 | ## v1.0.0 2015 November 30 146 | 147 | - Initial working release 148 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # License 4 | 5 | Unless stated otherwise all works are: 6 | 7 | - Copyright © [Benjamin Lupton](https://balupton.com) 8 | 9 | and licensed under: 10 | 11 | - [Artistic License 2.0](http://spdx.org/licenses/Artistic-2.0.html) 12 | 13 | ## The Artistic License 2.0 14 | 15 | 
 16 | Copyright (c) 2000-2006, The Perl Foundation.
 17 | 
 18 | Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
 19 | 
 20 | Preamble
 21 | 
 22 | This license establishes the terms under which a given free software Package may be copied, modified, distributed, and/or redistributed. The intent is that the Copyright Holder maintains some artistic control over the development of that Package while still keeping the Package available as open source and free software.
 23 | 
 24 | You are always permitted to make arrangements wholly outside of this license directly with the Copyright Holder of a given Package.  If the terms of this license do not permit the full use that you propose to make of the Package, you should contact the Copyright Holder and seek a different licensing arrangement.
 25 | 
 26 | Definitions
 27 | 
 28 |      "Copyright Holder" means the individual(s) or organization(s) named in the copyright notice for the entire Package.
 29 | 
 30 |      "Contributor" means any party that has contributed code or other material to the Package, in accordance with the Copyright Holder's procedures.
 31 | 
 32 |      "You" and "your" means any person who would like to copy, distribute, or modify the Package.
 33 | 
 34 |      "Package" means the collection of files distributed by the Copyright Holder, and derivatives of that collection and/or of those files. A given Package may consist of either the Standard Version, or a Modified Version.
 35 | 
 36 |      "Distribute" means providing a copy of the Package or making it accessible to anyone else, or in the case of a company or organization, to others outside of your company or organization.
 37 | 
 38 |      "Distributor Fee" means any fee that you charge for Distributing this Package or providing support for this Package to another party.  It does not mean licensing fees.
 39 | 
 40 |      "Standard Version" refers to the Package if it has not been modified, or has been modified only in ways explicitly requested by the Copyright Holder.
 41 | 
 42 |      "Modified Version" means the Package, if it has been changed, and such changes were not explicitly requested by the Copyright Holder.
 43 | 
 44 |      "Original License" means this Artistic License as Distributed with the Standard Version of the Package, in its current version or as it may be modified by The Perl Foundation in the future.
 45 | 
 46 |      "Source" form means the source code, documentation source, and configuration files for the Package.
 47 | 
 48 |      "Compiled" form means the compiled bytecode, object code, binary, or any other form resulting from mechanical transformation or translation of the Source form.
 49 | 
 50 | Permission for Use and Modification Without Distribution
 51 | 
 52 | (1) You are permitted to use the Standard Version and create and use Modified Versions for any purpose without restriction, provided that you do not Distribute the Modified Version.
 53 | 
 54 | Permissions for Redistribution of the Standard Version
 55 | 
 56 | (2) You may Distribute verbatim copies of the Source form of the Standard Version of this Package in any medium without restriction, either gratis or for a Distributor Fee, provided that you duplicate all of the original copyright notices and associated disclaimers.  At your discretion, such verbatim copies may or may not include a Compiled form of the Package.
 57 | 
 58 | (3) You may apply any bug fixes, portability changes, and other modifications made available from the Copyright Holder.  The resulting Package will still be considered the Standard Version, and as such will be subject to the Original License.
 59 | 
 60 | Distribution of Modified Versions of the Package as Source
 61 | 
 62 | (4) You may Distribute your Modified Version as Source (either gratis or for a Distributor Fee, and with or without a Compiled form of the Modified Version) provided that you clearly document how it differs from the Standard Version, including, but not limited to, documenting any non-standard features, executables, or modules, and provided that you do at least ONE of the following:
 63 | 
 64 |      (a) make the Modified Version available to the Copyright Holder of the Standard Version, under the Original License, so that the Copyright Holder may include your modifications in the Standard Version.
 65 |      (b) ensure that installation of your Modified Version does not prevent the user installing or running the Standard Version. In addition, the Modified Version must bear a name that is different from the name of the Standard Version.
 66 |      (c) allow anyone who receives a copy of the Modified Version to make the Source form of the Modified Version available to others under
 67 | 
 68 |           (i) the Original License or
 69 |           (ii) a license that permits the licensee to freely copy, modify and redistribute the Modified Version using the same licensing terms that apply to the copy that the licensee received, and requires that the Source form of the Modified Version, and of any works derived from it, be made freely available in that license fees are prohibited but Distributor Fees are allowed.
 70 | 
 71 | Distribution of Compiled Forms of the Standard Version or Modified Versions without the Source
 72 | 
 73 | (5)  You may Distribute Compiled forms of the Standard Version without the Source, provided that you include complete instructions on how to get the Source of the Standard Version.  Such instructions must be valid at the time of your distribution.  If these instructions, at any time while you are carrying out such distribution, become invalid, you must provide new instructions on demand or cease further distribution. If you provide valid instructions or cease distribution within thirty days after you become aware that the instructions are invalid, then you do not forfeit any of your rights under this license.
 74 | 
 75 | (6)  You may Distribute a Modified Version in Compiled form without the Source, provided that you comply with Section 4 with respect to the Source of the Modified Version.
 76 | 
 77 | Aggregating or Linking the Package
 78 | 
 79 | (7)  You may aggregate the Package (either the Standard Version or Modified Version) with other packages and Distribute the resulting aggregation provided that you do not charge a licensing fee for the Package.  Distributor Fees are permitted, and licensing fees for other components in the aggregation are permitted. The terms of this license apply to the use and Distribution of the Standard or Modified Versions as included in the aggregation.
 80 | 
 81 | (8) You are permitted to link Modified and Standard Versions with other works, to embed the Package in a larger work of your own, or to build stand-alone binary or bytecode versions of applications that include the Package, and Distribute the result without restriction, provided the result does not expose a direct interface to the Package.
 82 | 
 83 | Items That are Not Considered Part of a Modified Version
 84 | 
 85 | (9) Works (including, but not limited to, modules and scripts) that merely extend or make use of the Package, do not, by themselves, cause the Package to be a Modified Version.  In addition, such works are not considered parts of the Package itself, and are not subject to the terms of this license.
 86 | 
 87 | General Provisions
 88 | 
 89 | (10)  Any use, modification, and distribution of the Standard or Modified Versions is governed by this Artistic License. By using, modifying or distributing the Package, you accept this license. Do not use, modify, or distribute the Package, if you do not accept this license.
 90 | 
 91 | (11)  If your Modified Version has been derived from a Modified Version made by someone other than you, you are nevertheless required to ensure that your Modified Version complies with the requirements of this license.
 92 | 
 93 | (12)  This license does not grant you the right to use any trademark, service mark, tradename, or logo of the Copyright Holder.
 94 | 
 95 | (13)  This license includes the non-exclusive, worldwide, free-of-charge patent license to make, have made, use, offer to sell, sell, import and otherwise transfer the Package with respect to any patent claims licensable by the Copyright Holder that are necessarily infringed by the Package. If you institute patent litigation (including a cross-claim or counterclaim) against any party alleging that the Package constitutes direct or contributory patent infringement, then this Artistic License to you shall terminate on the date that such litigation is filed.
 96 | 
 97 | (14)  Disclaimer of Warranty:
 98 | THE PACKAGE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES. THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT ARE DISCLAIMED TO THE EXTENT PERMITTED BY YOUR LOCAL LAW. UNLESS REQUIRED BY LAW, NO COPYRIGHT HOLDER OR CONTRIBUTOR WILL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING IN ANY WAY OUT OF THE USE OF THE PACKAGE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 99 |
100 | 101 | 102 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "readdir-cluster", 3 | "name": "readdir-cluster", 4 | "version": "6.0.0", 5 | "license": "Artistic-2.0", 6 | "description": "Create a cluster of workers to iterate through the filesystem", 7 | "homepage": "https://github.com/bevry/readdir-cluster", 8 | "funding": "https://bevry.me/fund", 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/bevry/readdir-cluster.git" 12 | }, 13 | "bugs": { 14 | "url": "https://github.com/bevry/readdir-cluster/issues" 15 | }, 16 | "keywords": [ 17 | "cluster", 18 | "directory", 19 | "es2015", 20 | "es2017", 21 | "es2022", 22 | "es5", 23 | "export-default", 24 | "folder", 25 | "fs", 26 | "nested", 27 | "node", 28 | "readdir", 29 | "recursive", 30 | "scan", 31 | "tree", 32 | "typed", 33 | "types", 34 | "typescript", 35 | "watch", 36 | "watching", 37 | "watchman" 38 | ], 39 | "badges": { 40 | "list": [ 41 | "githubworkflow", 42 | "npmversion", 43 | "npmdownloads", 44 | "---", 45 | "githubsponsors", 46 | "thanksdev", 47 | "patreon", 48 | "liberapay", 49 | "buymeacoffee", 50 | "opencollective", 51 | "crypto", 52 | "paypal", 53 | "---", 54 | "discord", 55 | "twitch" 56 | ], 57 | "config": { 58 | "githubWorkflow": "bevry", 59 | "githubSponsorsUsername": "balupton", 60 | "thanksdevGithubUsername": "bevry", 61 | "buymeacoffeeUsername": "balupton", 62 | "cryptoURL": "https://bevry.me/crypto", 63 | "flattrUsername": "balupton", 64 | "liberapayUsername": "bevry", 65 | "opencollectiveUsername": "bevry", 66 | "patreonUsername": "bevry", 67 | "paypalURL": "https://bevry.me/paypal", 68 | "wishlistURL": "https://bevry.me/wishlist", 69 | "discordServerID": "1147436445783560193", 70 | "discordServerInvite": "nQuXddV7VP", 71 | "twitchUsername": "balupton", 72 | "githubUsername": "bevry", 73 | "githubRepository": "readdir-cluster", 74 | "githubSlug": "bevry/readdir-cluster", 75 | "npmPackageName": "readdir-cluster" 76 | } 77 | }, 78 | "author": "Benjamin Lupton (https://balupton.com) (https://github.com/balupton)", 79 | "authors": [ 80 | "Benjamin Lupton (https://balupton.com) (https://github.com/balupton): Accelerating collaborative wisdom." 81 | ], 82 | "maintainers": [ 83 | "Benjamin Lupton (https://balupton.com) (https://github.com/balupton): Accelerating collaborative wisdom." 84 | ], 85 | "contributors": [ 86 | "Benjamin Lupton (https://balupton.com) (https://github.com/balupton)" 87 | ], 88 | "sponsors": [ 89 | "Andrew Nesbitt (https://nesbitt.io) (https://github.com/andrew): Software engineer and researcher", 90 | "Balsa (https://balsa.com) (https://github.com/balsa): We're Balsa, and we're building tools for builders.", 91 | "Codecov (https://codecov.io) (https://github.com/codecov): Empower developers with tools to improve code quality and testing.", 92 | "Poonacha Medappa (https://poonachamedappa.com) (https://github.com/km-Poonacha)", 93 | "Rob Morris (https://github.com/Rob-Morris)", 94 | "Sentry (https://sentry.io) (https://github.com/getsentry): Real-time crash reporting for your web apps, mobile apps, and games.", 95 | "Syntax (https://syntax.fm) (https://github.com/syntaxfm): Syntax Podcast" 96 | ], 97 | "donors": [ 98 | "Andrew Nesbitt (https://nesbitt.io) (https://github.com/andrew)", 99 | "Armen Mkrtchian (https://mogoni.dev) (https://github.com/Armenm)", 100 | "Balsa (https://balsa.com) (https://github.com/balsa)", 101 | "Chad (https://opencollective.com/chad8)", 102 | "Codecov (https://codecov.io) (https://github.com/codecov)", 103 | "dr.dimitru (https://veliovgroup.com) (https://github.com/dr-dimitru)", 104 | "Elliott Ditman (https://elliottditman.com) (https://github.com/elliottditman)", 105 | "entroniq (https://gitlab.com/entroniq) (https://thanks.dev/d/gl/entroniq)", 106 | "GitHub (https://github.com/about) (https://github.com/github)", 107 | "Hunter Beast (https://cryptoquick.com) (https://github.com/cryptoquick)", 108 | "Jean-Luc Geering (https://github.com/jlgeering) (https://opencollective.com/jlgeering) (https://twitter.com/jlgeering)", 109 | "Michael Duane Mooring (https://mdm.cc) (https://github.com/mikeumus) (https://opencollective.com/mikeumus) (https://twitter.com/mikeumus)", 110 | "Michael Harry Scepaniak (https://michaelscepaniak.com) (https://github.com/hispanic)", 111 | "Mohammed Shah (https://github.com/smashah) (https://thanks.dev/d/gh/smashah) (https://twitter.com/smashah)", 112 | "Mr. Henry (https://mrhenry.be) (https://github.com/mrhenry)", 113 | "Nermal (https://arjunaditya.vercel.app) (https://github.com/nermalcat69)", 114 | "Pleo (https://pleo.io) (https://github.com/pleo-io)", 115 | "Poonacha Medappa (https://poonachamedappa.com) (https://github.com/km-Poonacha)", 116 | "Rob Morris (https://github.com/Rob-Morris)", 117 | "Robert de Forest (https://github.com/rdeforest)", 118 | "Sentry (https://sentry.io) (https://github.com/getsentry)", 119 | "ServieJS (https://github.com/serviejs) (https://thanks.dev/d/gh/serviejs)", 120 | "Skunk Team (https://skunk.team) (https://github.com/skunkteam)", 121 | "Syntax (https://syntax.fm) (https://github.com/syntaxfm)", 122 | "WriterJohnBuck (https://github.com/WriterJohnBuck)" 123 | ], 124 | "engines": { 125 | "node": ">=4" 126 | }, 127 | "editions": [ 128 | { 129 | "description": "TypeScript source code with Import for modules", 130 | "directory": "source", 131 | "entry": "index.ts", 132 | "tags": [ 133 | "source", 134 | "typescript", 135 | "import" 136 | ], 137 | "engines": false 138 | }, 139 | { 140 | "description": "TypeScript compiled against ES2022 for Node.js 14 || 16 || 18 || 20 || 21 with Require for modules", 141 | "directory": "edition-es2022", 142 | "entry": "index.js", 143 | "tags": [ 144 | "compiled", 145 | "javascript", 146 | "es2022", 147 | "require" 148 | ], 149 | "engines": { 150 | "node": "14 || 16 || 18 || 20 || 21", 151 | "browsers": false 152 | } 153 | }, 154 | { 155 | "description": "TypeScript compiled against ES2017 for Node.js 8 || 10 || 12 || 14 || 16 || 18 || 20 || 21 with Require for modules", 156 | "directory": "edition-es2017", 157 | "entry": "index.js", 158 | "tags": [ 159 | "compiled", 160 | "javascript", 161 | "es2017", 162 | "require" 163 | ], 164 | "engines": { 165 | "node": "8 || 10 || 12 || 14 || 16 || 18 || 20 || 21", 166 | "browsers": false 167 | } 168 | }, 169 | { 170 | "description": "TypeScript compiled against ES2015 for Node.js 6 || 8 || 10 || 12 || 14 || 16 || 18 || 20 || 21 with Require for modules", 171 | "directory": "edition-es2015", 172 | "entry": "index.js", 173 | "tags": [ 174 | "compiled", 175 | "javascript", 176 | "es2015", 177 | "require" 178 | ], 179 | "engines": { 180 | "node": "6 || 8 || 10 || 12 || 14 || 16 || 18 || 20 || 21", 181 | "browsers": false 182 | } 183 | }, 184 | { 185 | "description": "TypeScript compiled against ES5 for Node.js 4 || 6 || 8 || 10 || 12 || 14 || 16 || 18 || 20 || 21 with Require for modules", 186 | "directory": "edition-es5", 187 | "entry": "index.js", 188 | "tags": [ 189 | "compiled", 190 | "javascript", 191 | "es5", 192 | "require" 193 | ], 194 | "engines": { 195 | "node": "4 || 6 || 8 || 10 || 12 || 14 || 16 || 18 || 20 || 21", 196 | "browsers": false 197 | } 198 | }, 199 | { 200 | "description": "TypeScript compiled against ES2017 for Node.js 12 || 14 || 16 || 18 || 20 || 21 with Import for modules", 201 | "directory": "edition-es2017-esm", 202 | "entry": "index.js", 203 | "tags": [ 204 | "compiled", 205 | "javascript", 206 | "es2017", 207 | "import" 208 | ], 209 | "engines": { 210 | "node": "12 || 14 || 16 || 18 || 20 || 21", 211 | "browsers": false 212 | } 213 | }, 214 | { 215 | "description": "TypeScript compiled Types with Import for modules", 216 | "directory": "edition-types", 217 | "entry": "index.d.ts", 218 | "tags": [ 219 | "compiled", 220 | "types", 221 | "import" 222 | ], 223 | "engines": false 224 | } 225 | ], 226 | "bin": "bin.cjs", 227 | "types": "edition-types/index.d.ts", 228 | "type": "module", 229 | "main": "index.cjs", 230 | "exports": { 231 | "node": { 232 | "types": "./edition-types/index.d.ts", 233 | "import": "./edition-es2017-esm/index.js", 234 | "default": "./index.cjs", 235 | "require": "./edition-es2022/index.js" 236 | } 237 | }, 238 | "dependencies": { 239 | "editions": "^6.21.0" 240 | }, 241 | "devDependencies": { 242 | "@types/node": "^20.10.6", 243 | "@typescript-eslint/eslint-plugin": "^6.16.0", 244 | "@typescript-eslint/parser": "^6.16.0", 245 | "assert-helpers": "^11.12.0", 246 | "eslint": "^8.56.0", 247 | "eslint-config-bevry": "^5.4.0", 248 | "eslint-config-prettier": "^9.1.0", 249 | "eslint-plugin-prettier": "^5.1.2", 250 | "filedirname": "^3.4.0", 251 | "kava": "^7.8.0", 252 | "prettier": "^3.1.1", 253 | "projectz": "^4.2.0", 254 | "typedoc": "^0.25.4", 255 | "typescript": "5.3.3", 256 | "valid-directory": "^4.9.0" 257 | }, 258 | "scripts": { 259 | "our:bin": "node ./bin.cjs", 260 | "our:clean": "rm -rf ./docs ./edition* ./es2015 ./es5 ./out ./.next", 261 | "our:compile": "npm run our:compile:edition-es2015 && npm run our:compile:edition-es2017 && npm run our:compile:edition-es2017-esm && npm run our:compile:edition-es2022 && npm run our:compile:edition-es5 && npm run our:compile:edition-types", 262 | "our:compile:edition-es2015": "tsc --module commonjs --target ES2015 --outDir ./edition-es2015 --project tsconfig.json && ( test ! -d edition-es2015/source || ( mv edition-es2015/source edition-temp && rm -rf edition-es2015 && mv edition-temp edition-es2015 ) ) && printf '%s' '{\"type\": \"commonjs\"}' > edition-es2015/package.json", 263 | "our:compile:edition-es2017": "tsc --module commonjs --target ES2017 --outDir ./edition-es2017 --project tsconfig.json && ( test ! -d edition-es2017/source || ( mv edition-es2017/source edition-temp && rm -rf edition-es2017 && mv edition-temp edition-es2017 ) ) && printf '%s' '{\"type\": \"commonjs\"}' > edition-es2017/package.json", 264 | "our:compile:edition-es2017-esm": "tsc --module ESNext --target ES2017 --outDir ./edition-es2017-esm --project tsconfig.json && ( test ! -d edition-es2017-esm/source || ( mv edition-es2017-esm/source edition-temp && rm -rf edition-es2017-esm && mv edition-temp edition-es2017-esm ) ) && printf '%s' '{\"type\": \"module\"}' > edition-es2017-esm/package.json", 265 | "our:compile:edition-es2022": "tsc --module commonjs --target ES2022 --outDir ./edition-es2022 --project tsconfig.json && ( test ! -d edition-es2022/source || ( mv edition-es2022/source edition-temp && rm -rf edition-es2022 && mv edition-temp edition-es2022 ) ) && printf '%s' '{\"type\": \"commonjs\"}' > edition-es2022/package.json", 266 | "our:compile:edition-es5": "tsc --module commonjs --target ES5 --outDir ./edition-es5 --project tsconfig.json && ( test ! -d edition-es5/source || ( mv edition-es5/source edition-temp && rm -rf edition-es5 && mv edition-temp edition-es5 ) ) && printf '%s' '{\"type\": \"commonjs\"}' > edition-es5/package.json", 267 | "our:compile:edition-types": "tsc --emitDeclarationOnly --declaration --declarationMap --declarationDir ./edition-types --project tsconfig.json && ( test ! -d edition-types/source || ( mv edition-types/source edition-temp && rm -rf edition-types && mv edition-temp edition-types ) )", 268 | "our:deploy": "printf '%s\n' 'no need for this project'", 269 | "our:meta": "npm run our:meta:docs && npm run our:meta:projectz", 270 | "our:meta:docs": "npm run our:meta:docs:typedoc", 271 | "our:meta:docs:typedoc": "rm -rf ./docs && typedoc --exclude '**/+(*test*|node_modules)' --excludeExternals --out ./docs ./source", 272 | "our:meta:projectz": "projectz --offline", 273 | "our:release": "npm run our:release:prepare && npm run our:release:check-changelog && npm run our:release:check-dirty && npm run our:release:tag && npm run our:release:push", 274 | "our:release:check-changelog": "cat ./HISTORY.md | grep \"v$npm_package_version\" || (printf '%s\n' \"add a changelog entry for v$npm_package_version\" && exit -1)", 275 | "our:release:check-dirty": "git diff --exit-code", 276 | "our:release:prepare": "npm run our:clean && npm run our:compile && npm run our:test && npm run our:meta", 277 | "our:release:push": "git push origin && git push origin --tags", 278 | "our:release:tag": "export MESSAGE=$(cat ./HISTORY.md | sed -n \"/## v$npm_package_version/,/##/p\" | sed 's/## //' | awk 'NR>1{print buf}{buf = $0}') && test \"$MESSAGE\" || (printf '%s\n' 'proper changelog entry not found' && exit -1) && git tag \"v$npm_package_version\" -am \"$MESSAGE\"", 279 | "our:setup": "npm run our:setup:install", 280 | "our:setup:install": "npm install", 281 | "our:test": "npm run our:verify && npm test", 282 | "our:verify": "npm run our:verify:eslint && npm run our:verify:prettier", 283 | "our:verify:eslint": "eslint --fix --ignore-pattern '**/*.d.ts' --ignore-pattern '**/vendor/' --ignore-pattern '**/node_modules/' --ext .mjs,.js,.jsx,.ts,.tsx ./source", 284 | "our:verify:prettier": "prettier --write .", 285 | "test": "node ./test.cjs" 286 | }, 287 | "eslintConfig": { 288 | "extends": [ 289 | "bevry" 290 | ] 291 | }, 292 | "prettier": { 293 | "semi": false, 294 | "singleQuote": true, 295 | "trailingComma": "es5", 296 | "endOfLine": "lf" 297 | } 298 | } 299 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # readdir-cluster 4 | 5 | 6 | 7 | 8 | 9 | Status of the GitHub Workflow: bevry 10 | NPM version 11 | NPM downloads 12 |
13 | GitHub Sponsors donate button 14 | ThanksDev donate button 15 | Patreon donate button 16 | Liberapay donate button 17 | Buy Me A Coffee donate button 18 | Open Collective donate button 19 | crypto donate button 20 | PayPal donate button 21 |
22 | Discord server badge 23 | Twitch community badge 24 | 25 | 26 | 27 | 28 | 29 | Create a cluster of workers to iterate through the filesystem 30 | 31 | 32 | 33 | ## Usage 34 | 35 | [Complete API Documentation.](http://master.readdir-cluster.bevry.surge.sh/docs/) 36 | 37 | ```typescript 38 | import readdirCluster, { Stat } from 'readdir-cluster' 39 | 40 | // note that Stat is not the same as fs.Stats as it has functions removed, as it needed to be serialisable 41 | function iterator(path: string, filename: string, stat: Stat) { 42 | // skip directories and files that start with . 43 | if (filename[0] === '.') return false 44 | // do not recurse into directories 45 | if (stat.directory) return false 46 | } 47 | 48 | const paths = await readdirCluster({ directory: '.', iterator }) 49 | console.log(paths) 50 | ``` 51 | 52 | ### Performance 53 | 54 | Benchmarks: 55 | 56 | - Running `readdir-cluster .` returns 7388 files in 500ms 57 | - Running [`readdir`](https://nodejs.org/api/fs.html#fsreaddirpath-options-callback) with `recursive: true` returns 7388 files in 100ms 58 | 59 | ```javascript 60 | import { readdir } from 'fs' 61 | readdir('.', { recursive: true }, (err, files) => { 62 | if (err) console.error(err) 63 | else if (files.length) process.stdout.write(files.join('\n') + '\n') 64 | }) 65 | ``` 66 | 67 | - Running [fdir](https://github.com/thecodrr/fdir) returns 6480 files in 100ms 68 | 69 | ```javascript 70 | import { fdir } from 'fdir' 71 | const api = new fdir().withBasePath().crawl(process.argv[2]) 72 | api.withPromise().then((files) => { 73 | if (files.length) process.stdout.write(files.join('\n') + '\n') 74 | }) 75 | ``` 76 | 77 | Recommendations: 78 | 79 | - if you target Node.js 18.7 and above, you should use [`fs.readdir`](https://nodejs.org/api/fs.html#fsreaddirpath-options-callback) with `recursive: true` 80 | - if you target older Node.js versions, you should use [`@bevry/fs-list`](https://github.com/bevry/fs-list) 81 | - if you target older Node.js versions and you want a stat object, use `readdir-cluster` 82 | - if you target Nodejs 12 and above, and want a lot of customisation, use [fdir](https://github.com/thecodrr/fdir) 83 | 84 | As for why this package exists, `readdir-cluster` was created in 2005, `recursive` was added to Node.js in 2023, and `fdir` was created in 2020. That said, there are [several issues](https://github.com/bevry/readdir-cluster/issues) that could potentially improve `readdir-cluster` performance. 85 | 86 | 87 | 88 | ## Install 89 | 90 | ### [npm](https://npmjs.com 'npm is a package manager for javascript') 91 | 92 | #### Install Globally 93 | 94 | - Install: `npm install --global readdir-cluster` 95 | - Executable: `readdir-cluster` 96 | 97 | #### Install Locally 98 | 99 | - Install: `npm install --save readdir-cluster` 100 | - Executable: `npx readdir-cluster` 101 | - Import: `import pkg from ('readdir-cluster')` 102 | - Require: `const pkg = require('readdir-cluster').default` 103 | 104 | ### [Editions](https://editions.bevry.me 'Editions are the best way to produce and consume packages you care about.') 105 | 106 | This package is published with the following editions: 107 | 108 | - `readdir-cluster` aliases `readdir-cluster/index.cjs` which uses the [Editions Autoloader](https://github.com/bevry/editions 'You can use the Editions Autoloader to autoload the appropriate edition for your consumers environment') to automatically select the correct edition for the consumer's environment 109 | - `readdir-cluster/source/index.ts` is [TypeScript](https://www.typescriptlang.org/ 'TypeScript is a typed superset of JavaScript that compiles to plain JavaScript.') source code with [Import](https://babeljs.io/docs/learn-es2015/#modules 'ECMAScript Modules') for modules 110 | - `readdir-cluster/edition-es2022/index.js` is [TypeScript](https://www.typescriptlang.org/ 'TypeScript is a typed superset of JavaScript that compiles to plain JavaScript.') compiled against [ES2022](https://en.wikipedia.org/wiki/ES2022 'ECMAScript 2022') for [Node.js](https://nodejs.org "Node.js is a JavaScript runtime built on Chrome's V8 JavaScript engine") 14 || 16 || 18 || 20 || 21 with [Require](https://nodejs.org/dist/latest-v5.x/docs/api/modules.html 'Node/CJS Modules') for modules 111 | - `readdir-cluster/edition-es2017/index.js` is [TypeScript](https://www.typescriptlang.org/ 'TypeScript is a typed superset of JavaScript that compiles to plain JavaScript.') compiled against [ES2017](https://en.wikipedia.org/wiki/ES2017 'ECMAScript 2017') for [Node.js](https://nodejs.org "Node.js is a JavaScript runtime built on Chrome's V8 JavaScript engine") 8 || 10 || 12 || 14 || 16 || 18 || 20 || 21 with [Require](https://nodejs.org/dist/latest-v5.x/docs/api/modules.html 'Node/CJS Modules') for modules 112 | - `readdir-cluster/edition-es2015/index.js` is [TypeScript](https://www.typescriptlang.org/ 'TypeScript is a typed superset of JavaScript that compiles to plain JavaScript.') compiled against [ES2015](https://babeljs.io/docs/en/learn#ecmascript-2015-features 'ECMAScript 2015') for [Node.js](https://nodejs.org "Node.js is a JavaScript runtime built on Chrome's V8 JavaScript engine") 6 || 8 || 10 || 12 || 14 || 16 || 18 || 20 || 21 with [Require](https://nodejs.org/dist/latest-v5.x/docs/api/modules.html 'Node/CJS Modules') for modules 113 | - `readdir-cluster/edition-es5/index.js` is [TypeScript](https://www.typescriptlang.org/ 'TypeScript is a typed superset of JavaScript that compiles to plain JavaScript.') compiled against ES5 for [Node.js](https://nodejs.org "Node.js is a JavaScript runtime built on Chrome's V8 JavaScript engine") 4 || 6 || 8 || 10 || 12 || 14 || 16 || 18 || 20 || 21 with [Require](https://nodejs.org/dist/latest-v5.x/docs/api/modules.html 'Node/CJS Modules') for modules 114 | - `readdir-cluster/edition-es2017-esm/index.js` is [TypeScript](https://www.typescriptlang.org/ 'TypeScript is a typed superset of JavaScript that compiles to plain JavaScript.') compiled against [ES2017](https://en.wikipedia.org/wiki/ES2017 'ECMAScript 2017') for [Node.js](https://nodejs.org "Node.js is a JavaScript runtime built on Chrome's V8 JavaScript engine") 12 || 14 || 16 || 18 || 20 || 21 with [Import](https://babeljs.io/docs/learn-es2015/#modules 'ECMAScript Modules') for modules 115 | - `readdir-cluster/edition-types/index.d.ts` is [TypeScript](https://www.typescriptlang.org/ 'TypeScript is a typed superset of JavaScript that compiles to plain JavaScript.') compiled Types with [Import](https://babeljs.io/docs/learn-es2015/#modules 'ECMAScript Modules') for modules 116 | 117 | 118 | 119 | 120 | 121 | ## History 122 | 123 | [Discover the release history by heading on over to the `HISTORY.md` file.](https://github.com/bevry/readdir-cluster/blob/HEAD/HISTORY.md#files) 124 | 125 | 126 | 127 | 128 | 129 | ## Backers 130 | 131 | ### Code 132 | 133 | [Discover how to contribute via the `CONTRIBUTING.md` file.](https://github.com/bevry/readdir-cluster/blob/HEAD/CONTRIBUTING.md#files) 134 | 135 | #### Authors 136 | 137 | - [Benjamin Lupton](https://balupton.com) — Accelerating collaborative wisdom. 138 | 139 | #### Maintainers 140 | 141 | - [Benjamin Lupton](https://balupton.com) — Accelerating collaborative wisdom. 142 | 143 | #### Contributors 144 | 145 | - [Benjamin Lupton](https://github.com/balupton) — [view contributions](https://github.com/bevry/readdir-cluster/commits?author=balupton 'View the GitHub contributions of Benjamin Lupton on repository bevry/readdir-cluster') 146 | 147 | ### Finances 148 | 149 | GitHub Sponsors donate button 150 | ThanksDev donate button 151 | Patreon donate button 152 | Liberapay donate button 153 | Buy Me A Coffee donate button 154 | Open Collective donate button 155 | crypto donate button 156 | PayPal donate button 157 | 158 | #### Sponsors 159 | 160 | - [Andrew Nesbitt](https://nesbitt.io) — Software engineer and researcher 161 | - [Balsa](https://balsa.com) — We're Balsa, and we're building tools for builders. 162 | - [Codecov](https://codecov.io) — Empower developers with tools to improve code quality and testing. 163 | - [Poonacha Medappa](https://poonachamedappa.com) 164 | - [Rob Morris](https://github.com/Rob-Morris) 165 | - [Sentry](https://sentry.io) — Real-time crash reporting for your web apps, mobile apps, and games. 166 | - [Syntax](https://syntax.fm) — Syntax Podcast 167 | 168 | #### Donors 169 | 170 | - [Andrew Nesbitt](https://nesbitt.io) 171 | - [Armen Mkrtchian](https://mogoni.dev) 172 | - [Balsa](https://balsa.com) 173 | - [Chad](https://opencollective.com/chad8) 174 | - [Codecov](https://codecov.io) 175 | - [dr.dimitru](https://veliovgroup.com) 176 | - [Elliott Ditman](https://elliottditman.com) 177 | - [entroniq](https://gitlab.com/entroniq) 178 | - [GitHub](https://github.com/about) 179 | - [Hunter Beast](https://cryptoquick.com) 180 | - [Jean-Luc Geering](https://github.com/jlgeering) 181 | - [Michael Duane Mooring](https://mdm.cc) 182 | - [Michael Harry Scepaniak](https://michaelscepaniak.com) 183 | - [Mohammed Shah](https://github.com/smashah) 184 | - [Mr. Henry](https://mrhenry.be) 185 | - [Nermal](https://arjunaditya.vercel.app) 186 | - [Pleo](https://pleo.io) 187 | - [Poonacha Medappa](https://poonachamedappa.com) 188 | - [Rob Morris](https://github.com/Rob-Morris) 189 | - [Robert de Forest](https://github.com/rdeforest) 190 | - [Sentry](https://sentry.io) 191 | - [ServieJS](https://github.com/serviejs) 192 | - [Skunk Team](https://skunk.team) 193 | - [Syntax](https://syntax.fm) 194 | - [WriterJohnBuck](https://github.com/WriterJohnBuck) 195 | 196 | 197 | 198 | 199 | 200 | ## License 201 | 202 | Unless stated otherwise all works are: 203 | 204 | - Copyright © [Benjamin Lupton](https://balupton.com) 205 | 206 | and licensed under: 207 | 208 | - [Artistic License 2.0](http://spdx.org/licenses/Artistic-2.0.html) 209 | 210 | 211 | --------------------------------------------------------------------------------