├── .github └── semantic.yml ├── .gitignore ├── .prettierrc ├── .travis.yml ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── bin └── wsrun.js ├── package.json ├── src ├── cmd-process.ts ├── console.spec.ts ├── console.ts ├── enums.ts ├── filter-changed-packages.spec.ts ├── filter-changed-packages.ts ├── fix-paths.spec.ts ├── fix-paths.ts ├── index.ts ├── rev-deps.spec.ts ├── rev-deps.ts ├── run-graph.ts ├── utils.ts └── workspace.ts ├── tests ├── __snapshots__ │ └── basic.ts.snap ├── basic.ts ├── runner.sh └── test.util.ts └── tsconfig.json /.github/semantic.yml: -------------------------------------------------------------------------------- 1 | # Always validate the PR title, and ignore the commits 2 | titleOnly: true 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | node_modules 3 | build 4 | tmp 5 | yarn-error.log 6 | /yarn.lock 7 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 100, 3 | "singleQuote": true, 4 | "semi": false, 5 | "tabWidth": 2 6 | } 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | language: node_js 3 | node_js: 'lts/*' 4 | script: yarn test -i --ci 5 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": true, 3 | "search.exclude": { 4 | "**/node_modules": true, 5 | "**/bower_components": true, 6 | "build/**": true 7 | }, 8 | "files.exclude": { 9 | "**/.git": true, 10 | "**/.svn": true, 11 | "**/.hg": true, 12 | "**/CVS": true, 13 | "**/.DS_Store": true, 14 | "**/node_modules": true, 15 | "**/build": true 16 | }, 17 | "typescript.tsdk": "node_modules/typescript/lib" 18 | } 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Andrej Trajchevski 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Workspace script runner 2 | 3 | Run npm scripts or custom commands in a yarn workspace 4 | 5 | ### Usage: 6 | 7 | ``` 8 | wsrun [options] -c [ ...] 9 | 10 | Mode (choose one): 11 | --parallel, -a Fully parallel mode (default) [boolean] 12 | --stages, -t Run in stages: start with packages that have no deps [boolean] 13 | --serial, -s Same as "stages" but with no parallelism at the stage level [boolean] 14 | 15 | Package Options: 16 | --recursive, -r Execute the same script on all of its dependencies, too [boolean] 17 | --package, -p Run only for packages matching this glob. Can be used multiple times. [array] 18 | --changedSince Runs commands in packages that have changed since the provided source control 19 | branch. [string] 20 | 21 | Misc Options: 22 | --if Run main command only if this condition runs successfully 23 | --ifDependency Run main command only if packages dependencies passed the condition 24 | (not available in parallel mode) [boolean] 25 | --fast-exit, -e If at least one script exits with code > 0, abort [boolean] 26 | --collect-logs, -l Collect per-package output and print it at the end of each script [boolean] 27 | --no-prefix Don't prefix output [boolean] 28 | --rewrite-paths Rewrite relative paths in the standard output, by prepending the 29 | /. [boolean] 30 | --bin The program to pass the command to [string] 31 | --done-criteria Consider a process "done" when an output line matches the specified RegExp 32 | --exclude, -x Skip running the command for that package [string] 33 | --exclude-missing, -m Skip packages which lack the specified command in the scripts section 34 | of their package.json [boolean] 35 | --report Show an execution report once the command has finished in each 36 | package [boolean] 37 | 38 | Other Options: 39 | --help Show help [boolean] 40 | --version Show version number [boolean] 41 | -c Denotes the end of the package list and the beginning of the command. 42 | Can be used instead of "--" [boolean] 43 | --revRecursive Include all dependents of the filtered packages. Runs after resolving 44 | the other package options. [boolean] 45 | --prefix Prefix output with package name [boolean] 46 | --concurrency, -y Maximum number of commands to be executed at once [number] 47 | 48 | ``` 49 | 50 | ### Examples: 51 | 52 | `yarn wsrun watch` will run `yarn watch` on every individual package, in parallel. 53 | 54 | `yarn wsrun --stages build` will build all packages, in stages, starting from those that don't depend on other packages. 55 | 56 | #### Specific packages: 57 | 58 | `yarn wsrun -p planc -r watch` will watch planc and all of its dependencies. 59 | 60 | `yarn wsrun -p planc -c watch` will watch planc only. Note that `-c` is passed here explicitly to 61 | denote the beginning of the command. This is needed because `-p` can accept multiple packages. (`-c` 62 | can also be substituted with `--` but that generates warnings in yarn) 63 | 64 | `yarn wsrun -p 'app-*-frontend' -r watch` will watch all packages matching the glob 65 | `'app-*-frontend'` and their dependencies. Globstar and extglobs are supported. Make sure to pass 66 | the option quoted to prevent bash from trying to expand it! 67 | 68 | `yarn wsrun -p h4zip planc -c test` - run tests for both `h4zip` and `planc 69 | 70 | `yarn wsrun -p planc --exclude planc -r watch` will watch all of planc's dependencies but not planc 71 | 72 | `yarn wsrun -p h4zip -r --stages build` will build all deps of h4zip, in order, then build h4zip 73 | 74 | `yarn wsrun -p planc --stages --done-criteria='Compilation complete' -r watch` will watch planc deps, 75 | in order, continuing when command outputs a line containing "Compilation complete" 76 | 77 | `yarn wsrun --exclude-missing test` will run the test script only on packages that have it 78 | 79 | `yarn wsrun --changedSince --exclude-missing test` will run the test script only on packages that have c 80 | hanged since master branch and have `test` command 81 | 82 | #### Additional arguments to scripts 83 | 84 | If you want to pass additional arguments to the command you can do that by adding them after the 85 | command: 86 | 87 | `yarn wsrun -r --stages build -p tsconfig.alternative.json` - build all packages in stages with 88 | and pass an alternative tsconfig to the build script 89 | 90 | #### Commands not in the scripts field 91 | 92 | When `--skip-missing` is not used, you can pass a command that doesn't exist in the scripts field: 93 | 94 | `yarn wsrun -r --stages tsc -p tsconfig.alternative.json` - run tsc for all packages with an alternative tsconfig 95 | 96 | #### Conditional execution 97 | 98 | Conditional execution is supported with `--if` and `--ifDependency` 99 | 100 | Examples 101 | 102 | `yarn wsrun --stages --if build-needed build` - for each package it will first try `yarn wsrun build-needed` and only if the exit code is zero (success) it will run `yarn wsrun build` 103 | 104 | `yarn wsrun --stages --if build-needed --ifDependency build` - it will run `build` for each package in stages, if either the package's own condition command was success, or any of the dependencies had a successful condition. 105 | -------------------------------------------------------------------------------- /bin/wsrun.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | require('../build/index.js') -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "wsrun", 3 | "version": "5.2.4", 4 | "description": "executes commands on packages in parallel, but is aware of the dependencies between them", 5 | "main": "./build/index.js", 6 | "repository": "hfour/wsrun", 7 | "author": "hfour", 8 | "license": "MIT", 9 | "jest": { 10 | "verbose": false, 11 | "preset": "ts-jest", 12 | "testPathIgnorePatterns": [ 13 | "/tmp", 14 | "/node_modules/", 15 | "\\.util\\.ts$" 16 | ], 17 | "testMatch": [ 18 | "/tests/**/*.ts", 19 | "/src/**/*.spec.ts" 20 | ], 21 | "modulePathIgnorePatterns": [ 22 | "/node_modules/", 23 | "/tmp" 24 | ] 25 | }, 26 | "bin": { 27 | "wsrun": "./bin/wsrun.js" 28 | }, 29 | "files": [ 30 | "bin/*", 31 | "build/**/!(*.spec.js|*.spec.js.map)" 32 | ], 33 | "devDependencies": { 34 | "@types/bluebird": "^3.5.18", 35 | "@types/glob": "^5.0.33", 36 | "@types/jest": "^21.1.6", 37 | "@types/lodash": "^4.14.85", 38 | "@types/minimatch": "^3.0.3", 39 | "@types/mkdirp": "^0.5.2", 40 | "@types/mz": "^0.0.32", 41 | "@types/node": "^8.0.53", 42 | "@types/rimraf": "^2.0.2", 43 | "@types/split": "^0.3.28", 44 | "@types/yargs": "^13.0.0", 45 | "jest": "^23.0.0", 46 | "mkdirp": "^0.5.1", 47 | "mz": "^2.7.0", 48 | "npm-run-all": "^4.1.3", 49 | "rimraf": "^2.6.2", 50 | "semantic-release": "^15.13.18", 51 | "ts-jest": "23.10.5", 52 | "ts-mockito": "^2.5.0", 53 | "typescript": "^3.1.1", 54 | "prettier": "^1.19.1" 55 | }, 56 | "scripts": { 57 | "build": "tsc", 58 | "watch": "tsc -w", 59 | "test": "yarn build && yarn test:prettier && jest", 60 | "test:prettier": "prettier -c '**/*.ts' '**/*.json'", 61 | "test:watch": "jest --watch", 62 | "dev": "run-p test:watch watch", 63 | "prepublish": "tsc", 64 | "release": "semantic-release" 65 | }, 66 | "dependencies": { 67 | "bluebird": "^3.5.1", 68 | "chalk": "^2.3.0", 69 | "glob": "^7.1.2", 70 | "lodash": "^4.17.4", 71 | "minimatch": "^3.0.4", 72 | "split": "^1.0.1", 73 | "throat": "^4.1.0", 74 | "yargs": "^13.0.0", 75 | "jest-changed-files": "^24.9.0" 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/cmd-process.ts: -------------------------------------------------------------------------------- 1 | import { ChildProcess, spawn } from 'child_process' 2 | import * as Bromise from 'bluebird' 3 | 4 | import originalSplit = require('split') 5 | 6 | import { Result, ResultSpecialValues } from './enums' 7 | import { defer } from './utils' 8 | 9 | import { IConsole } from './console' 10 | 11 | export interface CmdOptions { 12 | rejectOnNonZeroExit: boolean 13 | silent?: boolean 14 | collectLogs: boolean 15 | prefixer?: (basePath: string, pkg: string, line: string) => string 16 | pathRewriter?: (currentPath: string, line: string) => string 17 | doneCriteria?: string 18 | path: string 19 | } 20 | 21 | const SPLIT_OPTIONS = { trailing: false } 22 | const SPLIT_MAPPER = (x: string) => x 23 | 24 | const split = () => originalSplit(/\r?\n/, SPLIT_MAPPER, SPLIT_OPTIONS as any) 25 | 26 | export class CmdProcess { 27 | private cp!: ChildProcess 28 | private _closed = defer() 29 | private _finished = defer() 30 | private _exitCode = defer() 31 | private _cancelled = defer() 32 | 33 | private doneCriteria?: RegExp 34 | 35 | /** 36 | * Finished will return true even if the process hasn't exited, if doneCriteria was found in 37 | * the output. Useful for watch processes that have initialization. 38 | * 39 | * It will also get rejected if there is a non-favorable exit code. 40 | */ 41 | get finished() { 42 | return this._finished.promise 43 | } 44 | 45 | /** 46 | * Exitcode is always resolved with the exit code, never rejected. 47 | */ 48 | get exitCode() { 49 | return this._exitCode.promise 50 | } 51 | 52 | get result() { 53 | return Bromise.race([this._exitCode.promise, this._cancelled.promise]) 54 | } 55 | 56 | /** 57 | * ExitError is like exitCode, except it gets rejected when the exit code is nonzero 58 | */ 59 | get exitError() { 60 | return this.exitCode.then(c => { 61 | if (c != 0) throw new Error('`' + this.cmdString + '` failed with exit code ' + c) 62 | }) 63 | } 64 | 65 | get cmdString() { 66 | return this.cmd.join(' ') 67 | } 68 | 69 | constructor( 70 | private console: IConsole, 71 | private cmd: string[], 72 | private pkgName: string, 73 | private opts: CmdOptions 74 | ) { 75 | this.pkgName = pkgName 76 | this.opts = opts 77 | 78 | if (this.opts.doneCriteria) this.doneCriteria = new RegExp(this.opts.doneCriteria) 79 | } 80 | 81 | start() { 82 | this._start(this.cmd) 83 | this.cp.once('close', code => { 84 | this._closed.resolve(code) 85 | this._exitCode.resolve(code) 86 | }) 87 | 88 | this.cp.once('exit', code => this._exitCode.resolve(code)) 89 | 90 | this.exitCode.then(code => { 91 | if (code > 0) { 92 | const msg = '`' + this.cmdString + '` failed with exit code ' + code 93 | if (!this.opts.silent) this.console.error(this.autoAugmentLine(msg)) 94 | if (this.opts.rejectOnNonZeroExit) return this._finished.reject(new Error(msg)) 95 | } 96 | this._finished.resolve() 97 | }) 98 | 99 | // ignore if unhandled 100 | this._finished.promise.catch(() => {}) 101 | } 102 | 103 | stop() { 104 | if (this.cp) { 105 | this.cp.removeAllListeners('close') 106 | this.cp.removeAllListeners('exit') 107 | this.cp.kill('SIGINT') 108 | } 109 | this._cancelled.resolve(ResultSpecialValues.Cancelled) 110 | } 111 | 112 | private autoPrefix(line: string) { 113 | return this.opts.prefixer ? this.opts.prefixer(this.opts.path, this.pkgName, line) : line 114 | } 115 | 116 | private autoPathRewrite(line: string) { 117 | return this.opts.pathRewriter ? this.opts.pathRewriter(this.opts.path, line) : line 118 | } 119 | 120 | private autoAugmentLine(line: string) { 121 | line = this.autoPathRewrite(line) 122 | line = this.autoPrefix(line) 123 | return line 124 | } 125 | 126 | private _start(cmd: string[]) { 127 | let sh: string 128 | let args: string[] 129 | 130 | // cross platform compatibility 131 | if (process.platform === 'win32') { 132 | sh = 'cmd' 133 | args = ['/c'].concat(cmd) 134 | } else { 135 | ;[sh, ...args] = cmd 136 | //sh = 'bash' 137 | //shFlag = '-c' 138 | } 139 | 140 | this.cmd = cmd 141 | this.cp = spawn(sh, args, { 142 | cwd: 143 | this.opts.path || 144 | ((process.versions.node < '8.0.0' ? process.cwd : process.cwd()) as string), 145 | env: Object.assign( 146 | process.env, 147 | process.stdout.isTTY ? { FORCE_COLOR: process.env.FORCE_COLOR || '1' } : {} 148 | ), 149 | stdio: 150 | this.opts.collectLogs || this.opts.prefixer != null || this.opts.doneCriteria 151 | ? 'pipe' 152 | : 'inherit' 153 | }) 154 | 155 | if (this.cp.stdout) 156 | this.cp.stdout.pipe(split()).on('data', (line: string) => { 157 | this.console.log(this.autoAugmentLine(line)) 158 | if (this.doneCriteria && this.doneCriteria.test(line)) this._finished.resolve() 159 | }) 160 | if (this.cp.stderr) 161 | this.cp.stderr.pipe(split()).on('data', (line: string) => { 162 | this.console.error(this.autoAugmentLine(line)) 163 | if (this.doneCriteria && this.doneCriteria.test(line)) this._finished.resolve() 164 | }) 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/console.spec.ts: -------------------------------------------------------------------------------- 1 | import { SerializedConsole } from './console' 2 | import { mock, instance, verify } from 'ts-mockito' 3 | 4 | function nextTick() { 5 | return new Promise(resolve => setImmediate(resolve)) 6 | } 7 | 8 | describe('serialized console', () => { 9 | let console: Console 10 | 11 | let c: SerializedConsole 12 | 13 | beforeEach(() => { 14 | console = mock() 15 | c = new SerializedConsole(instance(console)) 16 | }) 17 | 18 | it('should only output from the active console', () => { 19 | let c1 = c.create() 20 | let c2 = c.create() 21 | 22 | c1.log('hello 1') 23 | verify(console.log('hello 1')).once() 24 | c2.log('hello 2') 25 | verify(console.log('hello 2')).never() 26 | c1.log('hello 3') 27 | verify(console.log('hello 3')).once() 28 | c2.log('hello 4') 29 | verify(console.log('hello 4')).never() 30 | }) 31 | 32 | it('should output the second console when the first is done', async () => { 33 | let c1 = c.create() 34 | let c2 = c.create() 35 | 36 | c1.log('hello 1') 37 | verify(console.log('hello 1')).once() 38 | c2.log('hello 2') 39 | verify(console.log('hello 2')).never() 40 | 41 | c.done(c1) 42 | await nextTick() 43 | 44 | verify(console.log('hello 2')).once() 45 | 46 | c2.log('hello 3') 47 | verify(console.log('hello 3')).once() 48 | }) 49 | 50 | it('should output from all consoles when they are done in the wrong order', async () => { 51 | let c1 = c.create() 52 | let c2 = c.create() 53 | let c3 = c.create() 54 | 55 | c1.log('hello 1') 56 | verify(console.log('hello 1')).once() 57 | c2.log('hello 2') 58 | verify(console.log('hello 2')).never() 59 | c3.log('hello 3') 60 | verify(console.log('hello 3')).never() 61 | 62 | c.done(c2) 63 | await nextTick() 64 | 65 | verify(console.log('hello 2')).never() 66 | verify(console.log('hello 3')).never() 67 | 68 | c.done(c1) 69 | await nextTick() 70 | 71 | verify(console.log('hello 2')).once() 72 | verify(console.log('hello 3')).once() 73 | }) 74 | 75 | it('should log from console created after the first one is done', async () => { 76 | let c1 = c.create() 77 | 78 | c1.log('hello 1') 79 | verify(console.log('hello 1')).once() 80 | c.done(c1) 81 | await nextTick() 82 | 83 | let c2 = c.create() 84 | 85 | c2.log('hello 2') 86 | verify(console.log('hello 2')).once() 87 | }) 88 | }) 89 | -------------------------------------------------------------------------------- /src/console.ts: -------------------------------------------------------------------------------- 1 | import { defer } from './utils' 2 | 3 | export interface IConsole { 4 | log(msg: string): void 5 | error(msg: string): void 6 | } 7 | 8 | export interface ConsoleFactory { 9 | create(): IConsole 10 | done(c: IConsole): void 11 | } 12 | 13 | class SerializedConsoleImpl implements IConsole { 14 | private _activeOutput = false 15 | private _outputBuffer: { type: 'stderr' | 'stdout'; line: string }[] = [] 16 | public finished = defer() 17 | 18 | constructor(private _console: Console) {} 19 | 20 | activeOutput() { 21 | this._activeOutput = true 22 | 23 | this._outputBuffer.forEach(line => { 24 | if (line.type === 'stdout') this._console.log(line.line) 25 | else this._console.error(line.line) 26 | }) 27 | this._outputBuffer = [] 28 | } 29 | 30 | log(msg: string) { 31 | if (this._activeOutput) this._console.log(msg) 32 | else this._outputBuffer.push({ type: 'stdout', line: msg }) 33 | } 34 | 35 | error(msg: string) { 36 | if (this._activeOutput) this._console.error(msg) 37 | else this._outputBuffer.push({ type: 'stderr', line: msg }) 38 | } 39 | } 40 | 41 | export class SerializedConsole implements ConsoleFactory { 42 | private _active: SerializedConsoleImpl | undefined 43 | private _list: SerializedConsoleImpl[] = [] 44 | 45 | constructor(private _console: Console) {} 46 | 47 | private _start(c: SerializedConsoleImpl) { 48 | this._active = c 49 | this._active.activeOutput() 50 | 51 | this._active.finished.promise.then(() => { 52 | this._active = undefined 53 | 54 | let next = this._list.shift() 55 | if (next) { 56 | this._start(next) 57 | } 58 | }) 59 | } 60 | 61 | create() { 62 | let c = new SerializedConsoleImpl(this._console) 63 | if (!this._active) { 64 | this._start(c) 65 | } else { 66 | this._list.push(c) 67 | } 68 | return c 69 | } 70 | 71 | done(c: IConsole) { 72 | ;(c as SerializedConsoleImpl).finished.resolve() 73 | } 74 | } 75 | 76 | export class DefaultConsole implements ConsoleFactory { 77 | create() { 78 | return console 79 | } 80 | 81 | done(c: IConsole) {} 82 | } 83 | -------------------------------------------------------------------------------- /src/enums.ts: -------------------------------------------------------------------------------- 1 | export enum ResultSpecialValues { 2 | Pending = 'PENDING', 3 | Excluded = 'EXCLUDED', 4 | MissingScript = 'MISSING_SCRIPT', 5 | NotStarted = 'NOT_STARTED', 6 | Cancelled = 'CANCELLED' 7 | } 8 | 9 | export type Result = number | ResultSpecialValues 10 | 11 | export enum ProcResolution { 12 | Normal = 'Normal', 13 | Missing = 'Missing', 14 | Excluded = 'Excluded' 15 | } 16 | -------------------------------------------------------------------------------- /src/filter-changed-packages.spec.ts: -------------------------------------------------------------------------------- 1 | import { filterChangedPackages } from './filter-changed-packages' 2 | 3 | describe('filterChangedPackages', () => { 4 | it('should filter only the right package', async () => { 5 | const res = filterChangedPackages( 6 | ['packages/my-package-2/e.ts', 'packages/not-a-part-of-the-workspace/e.ts'], 7 | { 8 | a: 'my-package', 9 | b: 'my-package-2' 10 | }, 11 | 'packages' 12 | ) 13 | 14 | expect(res).toEqual(['b']) 15 | }) 16 | 17 | it('should filter out all packages if no package match is found', async () => { 18 | const res = filterChangedPackages(['packages2/c/example.ts'], { a: 'a', b: 'b' }, 'packages2') 19 | 20 | expect(res).toEqual([]) 21 | }) 22 | }) 23 | -------------------------------------------------------------------------------- /src/filter-changed-packages.ts: -------------------------------------------------------------------------------- 1 | import { Dict } from './workspace' 2 | import * as path from 'path' 3 | 4 | /** 5 | * filter the packages by checking if they have any changed files. This way is quicker 6 | * (mapping over packages) because package count is usually lower than changed files count 7 | * and we only need to check once per package. 8 | */ 9 | export const filterChangedPackages = ( 10 | files: string[], 11 | pkgPaths: Dict, 12 | workspacePath: string 13 | ) => { 14 | return Object.keys(pkgPaths).filter(pkg => { 15 | const pkgPath = pkgPaths[pkg] 16 | const p = `${path.join(workspacePath, pkgPath)}${path.sep}` 17 | 18 | return files.some(f => f.startsWith(p)) 19 | }) 20 | } 21 | -------------------------------------------------------------------------------- /src/fix-paths.spec.ts: -------------------------------------------------------------------------------- 1 | import 'jest' 2 | import { fixPaths } from './fix-paths' 3 | import chalk from 'chalk' 4 | 5 | describe('fix paths', () => { 6 | it('works', () => { 7 | let res = fixPaths('/a/b/c', '/a/b/c/packages/p', 'Testing (src/test.ts:12)') 8 | expect(res).toEqual('Testing (packages/p/src/test.ts:12)') 9 | }) 10 | 11 | it('works without brackets', () => { 12 | let res = fixPaths('/a/b/c', '/a/b/c/packages/p', 'Testing src/test.ts:12') 13 | expect(res).toEqual('Testing packages/p/src/test.ts:12') 14 | }) 15 | it('does not do absolute paths', () => { 16 | let res = fixPaths('/a/b/c', '/a/b/c/packages/p', 'Testing /src/test.ts:12') 17 | expect(res).toEqual('Testing /src/test.ts:12') 18 | }) 19 | 20 | it('ignores absolute paths that contain @', () => { 21 | let logLine = 'at (/Thing/src/node_modules/@h4bff/backend/src/rpc/serviceRegistry.ts:54:54)' 22 | let res = fixPaths('/Thing/src/', '/Thing/src/packages/app-lib-ca-backend', logLine) 23 | expect(res).toEqual(logLine) 24 | }) 25 | 26 | it('does not do absolute paths without brackets', () => { 27 | let res = fixPaths('/a/b/c', '/a/b/c/packages/p', 'Testing /src/test.ts:12') 28 | expect(res).toEqual('Testing /src/test.ts:12') 29 | }) 30 | 31 | it('applies relative paths', () => { 32 | let res = fixPaths('/a/b/c', '/a/b/c/packages/p', 'Testing ../src/test.ts:12') 33 | expect(res).toEqual('Testing packages/src/test.ts:12') 34 | }) 35 | 36 | it('works with color codes', () => { 37 | let res = fixPaths('/a/b/c', '/a/b/c/packages/p', 'Testing ' + chalk.blue('src/test.ts:12')) 38 | expect(res).toEqual('Testing ' + chalk.blue('packages/p/src/test.ts:12')) 39 | }) 40 | 41 | it('doesnt rewrite URLs', () => { 42 | let res = fixPaths('/a/b/c', '/a/b/c/packages/p', 'Testing http://src/test.ts') 43 | expect(res).toEqual('Testing http://src/test.ts') 44 | }) 45 | 46 | it('works with dashes', () => { 47 | let res = fixPaths( 48 | '/a/b/c', 49 | '/a/b/c/packages/p', 50 | 'Testing (/a/b/c/packages/test-package/src/file.ts:12' 51 | ) 52 | expect(res).toEqual('Testing (/a/b/c/packages/test-package/src/file.ts:12') 53 | }) 54 | }) 55 | -------------------------------------------------------------------------------- /src/fix-paths.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path' 2 | 3 | export function fixPaths(workspacePath: string, packagePath: string, logLine: string) { 4 | return logLine.replace( 5 | /([\u001b][^m]*m|[^-/_@0-9a-zA-Z])(([-_0-9a-zA-Z.]([^\s/'"*:]*[/]){1,})([^/'"*]+)\.[0-9a-zA-Z]{1,6})/, 6 | (_m, before, file) => { 7 | return before + path.relative(workspacePath, path.resolve(packagePath, file)) 8 | } 9 | ) 10 | } 11 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Tool for running command in yarn workspaces. 3 | */ 4 | 5 | import * as fs from 'fs' 6 | import * as yargs from 'yargs' 7 | import * as _ from 'lodash' 8 | import chalk from 'chalk' 9 | 10 | import { RunGraph } from './run-graph' 11 | import { listPkgs } from './workspace' 12 | 13 | let yargsParser = yargs 14 | .env('WSRUN') 15 | .wrap(yargs.terminalWidth() - 1) 16 | .updateStrings({ 17 | 'Options:': 'Other Options:' 18 | }) 19 | .usage('$0 [options] -c [ ...] ') 20 | // Note: these examples are chained here as they do not show up otherwise 21 | // when the required positional is not specified 22 | .example('$0 clean', 'Runs "yarn clean" in each of the packages in parallel') 23 | .example( 24 | '$0 -p app -r --stages build', 25 | 'Runs "yarn build" in app and all of its dependencies in stages, moving up the dependency tree' 26 | ) 27 | .example( 28 | '$0 --stages --done-criteria="Finished" watch', 29 | 'Runs "yarn watch" in each of the packages in stages, continuing when the process outputs "Finished"' 30 | ) 31 | .example('$0 --exclude-missing test', 'Runs "yarn test" in all packages that have such a script') 32 | .group(['parallel', 'stages', 'serial'], 'Mode (choose one):') 33 | .options({ 34 | parallel: { 35 | alias: 'a', 36 | boolean: true, 37 | describe: 'Fully parallel mode (default)' 38 | }, 39 | stages: { 40 | alias: 't', 41 | boolean: true, 42 | describe: 'Run in stages: start with packages that have no deps' 43 | }, 44 | serial: { 45 | alias: 's', 46 | boolean: true, 47 | describe: 'Same as "stages" but with no parallelism at the stage level' 48 | } 49 | }) 50 | .group(['recursive', 'package', 'changedSince'], 'Package Options:') 51 | .options({ 52 | package: { 53 | alias: 'p', 54 | describe: 'Run only for packages matching this glob. Can be used multiple times.', 55 | type: 'array' 56 | }, 57 | c: { 58 | boolean: true, 59 | describe: 60 | 'Denotes the end of the package list and the beginning of the command. Can be used instead of "--"' 61 | }, 62 | recursive: { 63 | alias: 'r', 64 | default: false, 65 | boolean: true, 66 | describe: 'Execute the same script on all of its dependencies, too' 67 | }, 68 | changedSince: { 69 | type: 'string', 70 | nargs: 1, 71 | describe: 72 | 'Runs commands in packages that have changed since the provided source control branch.' 73 | }, 74 | revRecursive: { 75 | default: false, 76 | boolean: true, 77 | describe: 78 | 'Include all dependents of the filtered packages. Runs after resolving the other package options.' 79 | } 80 | }) 81 | .group( 82 | [ 83 | 'if', 84 | 'ifDependency', 85 | 'fast-exit', 86 | 'collect-logs', 87 | 'no-prefix', 88 | 'rewrite-paths', 89 | 'bin', 90 | 'done-criteria', 91 | 'exclude', 92 | 'exclude-missing', 93 | 'report' 94 | ], 95 | 'Misc Options:' 96 | ) 97 | .options({ 98 | if: { 99 | describe: 'Run main command only if this condition runs successfully' 100 | }, 101 | ifDependency: { 102 | describe: 103 | 'Run main command only if packages dependencies passed the condition (not available in parallel mode)', 104 | boolean: true 105 | }, 106 | 'fast-exit': { 107 | alias: 'e', 108 | default: false, 109 | boolean: true, 110 | describe: 'If at least one script exits with code > 0, abort' 111 | }, 112 | 'collect-logs': { 113 | alias: 'l', 114 | default: false, 115 | boolean: true, 116 | describe: 'Collect per-package output and print it at the end of each script' 117 | }, 118 | prefix: { 119 | default: true, 120 | boolean: true, 121 | describe: 'Prefix output with package name' 122 | }, 123 | 'rewrite-paths': { 124 | default: false, 125 | boolean: true, 126 | describe: 127 | 'Rewrite relative paths in the standard output, by prepending the /.' 128 | }, 129 | bin: { 130 | default: 'yarn', 131 | describe: 'The program to pass the command to', 132 | type: 'string' 133 | }, 134 | 'done-criteria': { 135 | describe: 'Consider a process "done" when an output line matches the specified RegExp' 136 | }, 137 | exclude: { 138 | alias: 'x', 139 | type: 'string', 140 | describe: 'Skip running the command for that package' 141 | }, 142 | 'exclude-missing': { 143 | alias: 'm', 144 | default: false, 145 | boolean: true, 146 | describe: 147 | 'Skip packages which lack the specified command in the scripts section of their package.json' 148 | }, 149 | report: { 150 | default: false, 151 | boolean: true, 152 | describe: 'Show an execution report once the command has finished in each package' 153 | }, 154 | concurrency: { 155 | alias: 'y', 156 | type: 'number', 157 | describe: 'Maximum number of commands to be executed at once' 158 | } 159 | }) 160 | 161 | function parsePositionally(yargs: yargs.Argv, cmd: string[]) { 162 | let newCmd = cmd.map((c, i) => (c.startsWith('-') ? c : c + ':' + i.toString())) 163 | let positional = yargs.parse(newCmd) 164 | if (!positional._.length) return yargs.parse(cmd) 165 | 166 | let position = Number(positional._[0].substr(positional._[0].lastIndexOf(':') + 1)) 167 | 168 | let result = yargs.parse(cmd.slice(0, position)) 169 | result._ = result._.concat(cmd.slice(position)) 170 | return result 171 | } 172 | 173 | const argv = parsePositionally(yargsParser, process.argv.slice(2)) as any // yargs.argv 174 | 175 | let mode: string 176 | if (argv.stages) { 177 | mode = 'stages' 178 | } else if (argv.serial) { 179 | mode = 'serial' 180 | } else { 181 | mode = 'parallel' 182 | } 183 | 184 | const exclude: string[] = 185 | (argv.exclude && (Array.isArray(argv.exclude) ? argv.exclude : [argv.exclude])) || [] 186 | 187 | const concurrency: number | null = typeof argv.concurrency === 'number' ? argv.concurrency : null 188 | 189 | const cmd = argv._ 190 | 191 | if (!cmd.length) { 192 | yargs.showHelp() 193 | process.exit(1) 194 | } 195 | 196 | const packageJsonWorkspaces = JSON.parse(fs.readFileSync('./package.json', 'utf8')).workspaces 197 | const packageJsonWorkspacesNohoistFormat = packageJsonWorkspaces && packageJsonWorkspaces.packages 198 | 199 | const workspaceGlobs = packageJsonWorkspacesNohoistFormat || packageJsonWorkspaces || ['packages/*'] 200 | 201 | let pkgs 202 | try { 203 | pkgs = listPkgs('./', workspaceGlobs) 204 | } catch (err) { 205 | console.error(chalk.red(`\nERROR: ${err.message}`)) 206 | process.exit(1) 207 | } 208 | 209 | const pkgPaths = _.mapValues( 210 | _.keyBy(pkgs, p => p.json.name), 211 | v => v.path 212 | ) 213 | 214 | const pkgJsons = _.map(pkgs, pkg => pkg.json) 215 | 216 | let runner = new RunGraph( 217 | pkgJsons, 218 | { 219 | bin: argv.bin, 220 | fastExit: argv.fastExit, 221 | collectLogs: argv.collectLogs, 222 | addPrefix: argv.prefix, 223 | rewritePaths: argv.rewritePaths, 224 | mode: mode as any, 225 | recursive: argv.recursive, 226 | doneCriteria: argv.doneCriteria, 227 | changedSince: argv.changedSince, 228 | revRecursive: argv.revRecursive, 229 | exclude, 230 | excludeMissing: argv.excludeMissing, 231 | showReport: argv.report, 232 | if: argv.if || null, 233 | ifDependency: argv.ifDependency || false, 234 | workspacePath: process.cwd(), 235 | concurrency 236 | }, 237 | pkgPaths 238 | ) 239 | 240 | let cycle = runner.detectCycles() 241 | if (cycle.length > 0) { 242 | console.error('\nERROR: Dependency cycle detected:\n', ' ', cycle.join(' <- '), '\n') 243 | process.exit(1) 244 | } 245 | 246 | let runlist = argv.package || [] 247 | 248 | runner.run(cmd, runlist.length > 0 ? runlist : undefined).then(hadError => { 249 | if (hadError && argv.fastExit) { 250 | console.error(chalk.red(`Aborted execution due to previous error`)) 251 | } 252 | process.exit(hadError ? 1 : 0) 253 | }) 254 | 255 | // close all children on ctrl+c 256 | process.on('SIGINT', () => { 257 | runner.closeAll() 258 | process.exit(130) 259 | }) 260 | -------------------------------------------------------------------------------- /src/rev-deps.spec.ts: -------------------------------------------------------------------------------- 1 | import { expandRevDeps } from './rev-deps' 2 | 3 | describe('expandRevDeps', () => { 4 | it('should return empty array when no packages are supplied', async () => { 5 | const res = expandRevDeps([], [{ name: 'a' }, { name: 'b' }]) 6 | 7 | expect(res).toEqual([]) 8 | }) 9 | 10 | it('should return the same packages if no matches are found', async () => { 11 | const res = expandRevDeps(['c'], [{ name: 'a' }, { name: 'b' }]) 12 | 13 | expect(res).toEqual(['c']) 14 | }) 15 | 16 | it('should return the original list plus the proper reverse dependencies', async () => { 17 | const res = expandRevDeps( 18 | ['c'], 19 | [ 20 | { 21 | name: 'a', 22 | dependencies: { 23 | c: '*' 24 | } 25 | }, 26 | { 27 | name: 'b', 28 | devDependencies: { 29 | c: '*' 30 | } 31 | }, 32 | { 33 | name: 'd' 34 | } 35 | ] 36 | ) 37 | 38 | expect(res).toEqual(['c', 'a', 'b']) 39 | }) 40 | }) 41 | -------------------------------------------------------------------------------- /src/rev-deps.ts: -------------------------------------------------------------------------------- 1 | import { PkgJson } from './workspace' 2 | 3 | /** 4 | * For given list of packages, expand that list with all dependents on them 5 | * @param pkgs original list of packages (to be filtered) 6 | * @param pkgJsons list of packages in the workspace 7 | */ 8 | export const expandRevDeps = (pkgs: string[], pkgJsons: PkgJson[]) => { 9 | let index: number = 0 10 | while (index < pkgs.length) { 11 | const pkg = pkgs[index] 12 | 13 | // find the packages which have the iteratee as dependency or devDependency 14 | const found = pkgJsons 15 | .filter( 16 | p => 17 | (p.dependencies && p.dependencies[pkg]) || (p.devDependencies && p.devDependencies[pkg]) 18 | ) 19 | .map(p => p.name) 20 | .filter(p => pkgs.indexOf(p) === -1) 21 | 22 | pkgs = pkgs.concat(found) 23 | 24 | index++ 25 | } 26 | 27 | return pkgs 28 | } 29 | -------------------------------------------------------------------------------- /src/run-graph.ts: -------------------------------------------------------------------------------- 1 | import * as Bromise from 'bluebird' 2 | import chalk from 'chalk' 3 | 4 | import { PkgJson, Dict } from './workspace' 5 | import { ResultSpecialValues, Result, ProcResolution } from './enums' 6 | import { uniq, intersection } from 'lodash' 7 | import { CmdProcess } from './cmd-process' 8 | import minimatch = require('minimatch') 9 | import { fixPaths } from './fix-paths' 10 | import { ConsoleFactory, SerializedConsole, DefaultConsole } from './console' 11 | import { getChangedFilesForRoots } from 'jest-changed-files' 12 | import { filterChangedPackages } from './filter-changed-packages' 13 | import { expandRevDeps } from './rev-deps' 14 | 15 | type PromiseFn = () => Bromise 16 | type PromiseFnRunner = (f: PromiseFn) => Bromise 17 | 18 | let mkThroat = require('throat')(Bromise) as (limit: number) => PromiseFnRunner 19 | 20 | let passThrough: PromiseFnRunner = f => f() 21 | 22 | class Prefixer { 23 | constructor() {} 24 | private currentName = '' 25 | prefixer = (basePath: string, pkg: string, line: string) => { 26 | let l = '' 27 | if (this.currentName != pkg) l += chalk.bold((this.currentName = pkg)) + '\n' 28 | l += ' | ' + line // this.processFilePaths(basePath, line) 29 | return l 30 | } 31 | } 32 | 33 | export interface GraphOptions { 34 | bin: string 35 | fastExit: boolean 36 | collectLogs: boolean 37 | addPrefix: boolean 38 | rewritePaths: boolean 39 | mode: 'parallel' | 'serial' | 'stages' 40 | recursive: boolean 41 | doneCriteria: string | undefined 42 | changedSince: string | undefined 43 | revRecursive: boolean 44 | workspacePath: string 45 | exclude: string[] 46 | excludeMissing: boolean 47 | showReport: boolean 48 | if: string 49 | ifDependency: boolean 50 | concurrency: number | null 51 | } 52 | 53 | export class RunGraph { 54 | private procmap = new Map>() 55 | children: CmdProcess[] 56 | finishedAll!: Bromise 57 | private jsonMap = new Map() 58 | private runList = new Set() 59 | private resultMap = new Map() 60 | private throat: PromiseFnRunner = passThrough 61 | private consoles: ConsoleFactory 62 | pathRewriter = (pkgPath: string, line: string) => fixPaths(this.opts.workspacePath, pkgPath, line) 63 | 64 | constructor( 65 | public pkgJsons: PkgJson[], 66 | public opts: GraphOptions, 67 | public pkgPaths: Dict 68 | ) { 69 | this.checkResultsAndReport = this.checkResultsAndReport.bind(this) 70 | 71 | pkgJsons.forEach(j => this.jsonMap.set(j.name, j)) 72 | this.children = [] 73 | // serial always has a concurrency of 1 74 | if (this.opts.mode === 'serial') this.throat = mkThroat(1) 75 | // max 16 proc unless otherwise specified 76 | else if (this.opts.mode === 'stages') this.throat = mkThroat(opts.concurrency || 16) 77 | else if (opts.concurrency) this.throat = mkThroat(opts.concurrency) 78 | 79 | if (opts.collectLogs) this.consoles = new SerializedConsole(console) 80 | else this.consoles = new DefaultConsole() 81 | } 82 | 83 | private globalPrefixer = new Prefixer().prefixer 84 | /** 85 | * Creates or provides the global prefixer. This depends on the collect-logs flag which describes whether the processes should use a shared prefixer. 86 | */ 87 | private createOrProvidePrefixerForProcess = () => { 88 | if (this.opts.addPrefix) { 89 | if (this.opts.collectLogs) { 90 | return new Prefixer().prefixer 91 | } else { 92 | return this.globalPrefixer 93 | } 94 | } 95 | return undefined 96 | } 97 | 98 | closeAll() { 99 | console.log('Stopping', this.children.length, 'active children') 100 | this.children.forEach(ch => ch.stop()) 101 | } 102 | 103 | private lookupOrRun(cmd: string[], pkg: string): Bromise { 104 | let proc = this.procmap.get(pkg) 105 | if (proc == null) { 106 | proc = Bromise.resolve().then(() => this.runOne(cmd, pkg)) 107 | this.procmap.set(pkg, proc) 108 | return proc 109 | } 110 | return proc 111 | } 112 | 113 | private allDeps(pkg: PkgJson) { 114 | let findMyDeps = uniq( 115 | Object.keys(pkg.dependencies || {}).concat(Object.keys(pkg.devDependencies || {})) 116 | ).filter(d => this.jsonMap.has(d) && (this.opts.recursive || this.runList.has(d))) 117 | return findMyDeps 118 | } 119 | 120 | detectCycles() { 121 | let topLevelPkgs: { [name: string]: any } = {} 122 | for (let key of this.jsonMap.keys()) { 123 | topLevelPkgs[key] = '*' 124 | } 125 | let top = { name: '$', dependencies: topLevelPkgs } 126 | let self = this 127 | function deepCycle(json: PkgJson, pathLookup: string[]): string[] { 128 | let newPathLookup = pathLookup.concat([json.name]) 129 | let index = pathLookup.indexOf(json.name) 130 | if (index >= 0) { 131 | return newPathLookup.slice(index) 132 | } 133 | let currentDeps = Object.keys(json.dependencies || {}).concat( 134 | Object.keys(json.devDependencies || {}) 135 | ) 136 | for (let name of currentDeps) { 137 | let d = self.jsonMap.get(name) 138 | if (!d) continue 139 | let res = deepCycle(d, newPathLookup) 140 | if (res.length) return res 141 | } 142 | return [] 143 | } 144 | let res = deepCycle(top, []) 145 | return res 146 | } 147 | 148 | private makeCmd(cmd: string[]) { 149 | return [this.opts.bin].concat(cmd) 150 | } 151 | 152 | private runCondition(cmd: string, pkg: string) { 153 | let cmdLine = this.makeCmd(cmd.split(' ')) 154 | let c = this.consoles.create() 155 | const child = new CmdProcess(c, cmdLine, pkg, { 156 | rejectOnNonZeroExit: false, 157 | silent: true, 158 | collectLogs: this.opts.collectLogs, 159 | prefixer: this.createOrProvidePrefixerForProcess(), 160 | doneCriteria: this.opts.doneCriteria, 161 | path: this.pkgPaths[pkg] 162 | }) 163 | child.finished.then(() => this.consoles.done(c)) 164 | let rres = child.exitCode.then(code => code === 0) 165 | child.start() 166 | return rres 167 | } 168 | 169 | private runOne(cmdArray: string[], pkg: string): Bromise { 170 | let p = this.jsonMap.get(pkg) 171 | if (p == null) throw new Error('Unknown package: ' + pkg) 172 | let myDeps = Bromise.all(this.allDeps(p).map(d => this.lookupOrRun(cmdArray, d))) 173 | 174 | return myDeps.then(depsStatuses => { 175 | this.resultMap.set(pkg, ResultSpecialValues.Pending) 176 | 177 | if (this.opts.exclude.indexOf(pkg) >= 0) { 178 | console.log(chalk.bold(pkg), 'in exclude list, skipping') 179 | this.resultMap.set(pkg, ResultSpecialValues.Excluded) 180 | return Bromise.resolve(ProcResolution.Excluded) 181 | } 182 | if (this.opts.excludeMissing && (!p || !p.scripts || !p.scripts[cmdArray[0]])) { 183 | console.log(chalk.bold(pkg), 'has no', cmdArray[0], 'script, skipping missing') 184 | this.resultMap.set(pkg, ResultSpecialValues.MissingScript) 185 | return Bromise.resolve(ProcResolution.Missing) 186 | } 187 | 188 | let ifCondtition = Bromise.resolve(true) 189 | 190 | if ( 191 | this.opts.if && 192 | (!this.opts.ifDependency || !depsStatuses.find(ds => ds === ProcResolution.Normal)) 193 | ) { 194 | ifCondtition = this.runCondition(this.opts.if, pkg) 195 | } 196 | 197 | let child = ifCondtition.then(shouldExecute => { 198 | if (!shouldExecute) { 199 | this.resultMap.set(pkg, ResultSpecialValues.Excluded) 200 | return Bromise.resolve({ 201 | status: ProcResolution.Excluded, 202 | process: null as null | CmdProcess 203 | }) 204 | } 205 | 206 | let cmdLine = this.makeCmd(cmdArray) 207 | let c = this.consoles.create() 208 | const child = new CmdProcess(c, cmdLine, pkg, { 209 | rejectOnNonZeroExit: this.opts.fastExit, 210 | collectLogs: this.opts.collectLogs, 211 | prefixer: this.createOrProvidePrefixerForProcess(), 212 | pathRewriter: this.opts.rewritePaths ? this.pathRewriter : undefined, 213 | doneCriteria: this.opts.doneCriteria, 214 | path: this.pkgPaths[pkg] 215 | }) 216 | child.finished.then(() => this.consoles.done(c)) 217 | child.exitCode.then(code => this.resultMap.set(pkg, code)) 218 | this.children.push(child) 219 | return Promise.resolve({ status: ProcResolution.Normal, process: child }) 220 | }) 221 | 222 | return child.then(ch => { 223 | let processRun = this.throat(() => { 224 | if (ch.process) { 225 | ch.process.start() 226 | return ch.process.finished 227 | } 228 | return Bromise.resolve() 229 | }) 230 | if (this.opts.mode === 'parallel' || !ch.process) return ch.status 231 | else return processRun.thenReturn(ProcResolution.Normal) 232 | }) 233 | }) 234 | } 235 | 236 | private checkResultsAndReport(cmdLine: string[], pkgs: string[]) { 237 | let cmd = cmdLine.join(' ') 238 | const pkgsInError: string[] = [] 239 | const pkgsSuccessful: string[] = [] 240 | const pkgsPending: string[] = [] 241 | const pkgsSkipped: string[] = [] 242 | const pkgsMissingScript: string[] = [] 243 | 244 | this.resultMap.forEach((result, pkg) => { 245 | switch (result) { 246 | case ResultSpecialValues.Excluded: 247 | pkgsSkipped.push(pkg) 248 | break 249 | 250 | case ResultSpecialValues.MissingScript: 251 | pkgsMissingScript.push(pkg) 252 | break 253 | 254 | case ResultSpecialValues.Pending: 255 | pkgsPending.push(pkg) 256 | break 257 | 258 | case 0: 259 | pkgsSuccessful.push(pkg) 260 | break 261 | 262 | default: 263 | pkgsInError.push(pkg) 264 | break 265 | } 266 | }) 267 | 268 | if (this.opts.showReport) { 269 | const formatPkgs = (pgks: string[]): string => pgks.join(', ') 270 | const pkgsNotStarted = pkgs.filter(pkg => !this.resultMap.has(pkg)) 271 | 272 | console.log(chalk.bold('\nReport:')) 273 | 274 | if (pkgsInError.length) 275 | console.log( 276 | chalk.red( 277 | ` ${pkgsInError.length} packages finished \`${cmd}\` with error: ${formatPkgs( 278 | pkgsInError 279 | )}` 280 | ) 281 | ) 282 | if (pkgsSuccessful.length) 283 | console.log( 284 | chalk.green( 285 | ` ${pkgsSuccessful.length} packages finished \`${cmd}\` successfully: ${formatPkgs( 286 | pkgsSuccessful 287 | )}` 288 | ) 289 | ) 290 | if (pkgsPending.length) 291 | console.log( 292 | chalk.white( 293 | ` ${pkgsPending.length} packages have been cancelled running \`${cmd}\`: ${formatPkgs( 294 | pkgsPending 295 | )}` 296 | ) 297 | ) 298 | if (pkgsNotStarted.length) 299 | console.log( 300 | chalk.white( 301 | ` ${pkgsNotStarted.length} packages have not started running \`${cmd}\`: ${formatPkgs( 302 | pkgsNotStarted 303 | )}` 304 | ) 305 | ) 306 | if (pkgsMissingScript.length) 307 | console.log( 308 | chalk.gray( 309 | ` ${pkgsMissingScript.length} packages are missing script \`${cmd}\`: ${formatPkgs( 310 | pkgsMissingScript 311 | )}` 312 | ) 313 | ) 314 | if (pkgsSkipped.length) 315 | console.log( 316 | chalk.gray( 317 | ` ${pkgsSkipped.length} packages have been skipped: ${formatPkgs(pkgsSkipped)}` 318 | ) 319 | ) 320 | 321 | console.log() 322 | } 323 | 324 | return pkgsInError.length > 0 325 | } 326 | 327 | filterByGlobs(pkgs: string[], globs: string[]) { 328 | if (globs && globs.length > 0) { 329 | pkgs = pkgs.filter(name => globs.some(glob => minimatch(name, glob))) 330 | } 331 | 332 | return Bromise.resolve(pkgs) 333 | } 334 | 335 | filterByChangedFiles(pkgs: string[]) { 336 | // if changedSince is defined, filter the packages to contain only changed packages (according to git) 337 | if (this.opts.changedSince) { 338 | return getChangedFilesForRoots([this.opts.workspacePath], { 339 | changedSince: this.opts.changedSince 340 | }) 341 | .then(data => { 342 | if (!data.repos || (data.repos.git.size === 0 && data.repos.hg.size === 0)) { 343 | throw new Error( 344 | "The workspace is not a git/hg repo and it cannot work with 'changedSince'" 345 | ) 346 | } 347 | 348 | return filterChangedPackages( 349 | [...data.changedFiles], 350 | this.pkgPaths, 351 | this.opts.workspacePath 352 | ) 353 | }) 354 | .then(changedPackages => intersection(pkgs, changedPackages)) 355 | } 356 | 357 | return Promise.resolve(pkgs) 358 | } 359 | 360 | addRevDeps = (pkgs: string[]) => { 361 | if (this.opts.revRecursive) { 362 | return expandRevDeps(pkgs, this.pkgJsons) 363 | } else { 364 | return pkgs 365 | } 366 | } 367 | 368 | async run(cmd: string[], globs: string[] = ['**/*']) { 369 | let pkgs: string[] = this.pkgJsons.map(p => p.name) 370 | 371 | pkgs = await this.filterByGlobs(pkgs, globs).then(pkgs => 372 | this.filterByChangedFiles(pkgs).then(pkgs => this.addRevDeps(pkgs)) 373 | ) 374 | 375 | this.runList = new Set(pkgs) 376 | return ( 377 | Bromise.all(pkgs.map(pkg => this.lookupOrRun(cmd, pkg))) 378 | // Wait for any of them to error 379 | .then(() => Bromise.all(this.children.map(c => c.exitError))) 380 | // If any of them do, and fastExit is enabled, stop every other 381 | .catch(_err => this.opts.fastExit && this.closeAll()) 382 | // Wait for the all the processes to finish 383 | .then(() => Bromise.all(this.children.map(c => c.result))) 384 | // Generate report 385 | .then(() => this.checkResultsAndReport(cmd, pkgs)) 386 | ) 387 | } 388 | } 389 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | import * as Bromise from 'bluebird' 2 | export interface Defer { 3 | promise: Bromise 4 | resolve: (thenableOrResult?: T | PromiseLike | undefined) => void 5 | reject: (error?: any) => void 6 | } 7 | 8 | export function defer() { 9 | let d: Defer 10 | let promise = new Bromise((resolve, reject) => { 11 | d = { resolve, reject } as any 12 | }) 13 | d!.promise = promise 14 | return d! 15 | } 16 | -------------------------------------------------------------------------------- /src/workspace.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Remove me. 3 | */ 4 | 5 | import * as fs from 'fs' 6 | import * as path from 'path' 7 | import * as glob from 'glob' 8 | import { flatMap } from 'lodash' 9 | 10 | export type Dict = { [key: string]: T } 11 | 12 | export interface PkgJson { 13 | name: string 14 | dependencies?: Dict 15 | devDependencies?: Dict 16 | scripts?: { [name: string]: string } 17 | } 18 | 19 | export type Packages = Dict<{ 20 | path: string 21 | json: PkgJson 22 | }> 23 | 24 | /** 25 | * Given a path, it returns paths to package.json files of all packages, 26 | * and the package JSONs themselves. 27 | */ 28 | export function listPkgs(wsRoot: string, globs: string[]) { 29 | // based on yarn v1.18.0 workspace resolution: https://github.com/yarnpkg/yarn/blob/v1.18.0/src/config.js#L794 30 | const registryFilenames = ['package.json', 'yarn.json'] 31 | const registryFolders = ['node_modules'] 32 | const trailingPattern = `/+(${registryFilenames.join('|')})` 33 | const ignorePatterns = registryFolders.map( 34 | folder => `/${folder}/**/+(${registryFilenames.join('|')})` 35 | ) 36 | 37 | const pkgJsonPaths = flatMap(globs, (g: string) => 38 | glob.sync(g.replace(/\/?$/, trailingPattern), { 39 | cwd: wsRoot, 40 | ignore: ignorePatterns.map(ignorePattern => g.replace(/\/?$/, ignorePattern)) 41 | }) 42 | ) 43 | 44 | const packages: Packages = {} 45 | pkgJsonPaths.forEach(pkgJsonPath => { 46 | const pkgDir = path.dirname(pkgJsonPath) 47 | const pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8')) 48 | if (!pkgJson.name) 49 | throw new Error(`Package in directory ${pkgDir} has no name in ${path.basename(pkgJsonPath)}`) 50 | packages[pkgJson.name] = { 51 | path: path.join(wsRoot, pkgDir), 52 | json: pkgJson 53 | } 54 | }) 55 | return packages 56 | } 57 | -------------------------------------------------------------------------------- /tests/__snapshots__/basic.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`basic should keep its stdout and stderr stable 1`] = ` 4 | "@x/p1 5 | | $ sleep $1; echo @x/p1 $1 >> '/tmp/wsrun-test-1/echo.out' 0 Hello 6 | @x/p2 7 | | $ sleep $1; echo @x/p2 $1 >> '/tmp/wsrun-test-1/echo.out' 0 Hello 8 | " 9 | `; 10 | 11 | exports[`basic should keep its stdout and stderr stable 2`] = ` 12 | " | sleep: missing operand 13 | | Try 'sleep --help' for more information. 14 | | sleep: missing operand 15 | | Try 'sleep --help' for more information. 16 | " 17 | `; 18 | -------------------------------------------------------------------------------- /tests/basic.ts: -------------------------------------------------------------------------------- 1 | import 'jest' 2 | import { withScaffold, echo, wsrun } from './test.util' 3 | 4 | let pkgList = (errorp3: boolean = false, condition?: string) => [ 5 | echo.makePkg({ name: 'p1', dependencies: { p2: '*' } }, condition), 6 | echo.makePkg({ name: 'p2', dependencies: { p3: '*', p4: '*' } }, condition), 7 | errorp3 8 | ? echo.makePkgErr({ name: 'p3', dependencies: { p4: '*', p5: '*' } }) 9 | : echo.makePkg({ name: 'p3', dependencies: { p4: '*', p5: '*' } }, condition), 10 | echo.makePkg({ name: 'p4', dependencies: { p5: '*' } }, condition), 11 | echo.makePkg({ name: 'p5', dependencies: {} }, condition) 12 | ] 13 | 14 | describe('basic', () => { 15 | it('should run for all packages when in series', async () => { 16 | await withScaffold( 17 | { 18 | packages: pkgList() 19 | }, 20 | async () => { 21 | let tst = await wsrun('--serial doecho') 22 | expect(tst.error).toBeFalsy() 23 | let output = await echo.getOutput() 24 | expect(output).toEqual(['p5', 'p4', 'p3', 'p2', 'p1', ''].join('\n')) 25 | } 26 | ) 27 | }) 28 | 29 | it('should run for all packages when parallel', async () => { 30 | await withScaffold( 31 | { 32 | packages: pkgList() 33 | }, 34 | async () => { 35 | let wait = 0.25 36 | let tst = await wsrun(`--parallel doecho ${wait}`) 37 | expect(tst.error).toBeFalsy() 38 | let output = await echo.getOutput() 39 | expect( 40 | output 41 | .split('\n') 42 | .sort() 43 | .reverse() 44 | ).toEqual([`p5 ${wait}`, `p4 ${wait}`, `p3 ${wait}`, `p2 ${wait}`, `p1 ${wait}`, '']) 45 | } 46 | ) 47 | }) 48 | 49 | it('should run for a subset of packages in stages', async () => { 50 | await withScaffold( 51 | { 52 | packages: pkgList() 53 | }, 54 | async () => { 55 | let tst = await wsrun('-p p3 --stages -r doecho') 56 | expect(tst.error).toBeFalsy() 57 | let output = await echo.getOutput() 58 | expect(output).toEqual(['p5', 'p4', 'p3', ''].join('\n')) 59 | } 60 | ) 61 | }) 62 | 63 | it('should pass arguments to echo', async () => { 64 | await withScaffold( 65 | { 66 | packages: pkgList() 67 | }, 68 | async () => { 69 | let tst = await wsrun('-p p3 --stages -r doecho 0 hello world') 70 | expect(tst.error).toBeFalsy() 71 | let output = await echo.getOutput() 72 | expect(output).toEqual( 73 | ['p5 0 hello world', 'p4 0 hello world', 'p3 0 hello world', ''].join('\n') 74 | ) 75 | } 76 | ) 77 | }) 78 | 79 | it('should support conditional execution', async () => { 80 | await withScaffold( 81 | { 82 | packages: pkgList(false, 'pwd | grep -q p4$') 83 | }, 84 | async () => { 85 | let tst = await wsrun('--stages -r --if=condition -- doecho') 86 | expect(tst.error).toBeFalsy() 87 | let output = await echo.getOutput() 88 | expect(output).toEqual(['p4', ''].join('\n')) 89 | } 90 | ) 91 | }) 92 | 93 | it('should support dependant conditional execution', async () => { 94 | await withScaffold( 95 | { 96 | packages: pkgList(false, 'pwd | grep -q p2$') 97 | }, 98 | async () => { 99 | let tst = await wsrun('--stages -r --if=condition --ifDependency -- doecho') 100 | expect(tst.error).toBeFalsy() 101 | let output = await echo.getOutput() 102 | expect(output).toEqual(['p2', 'p1', ''].join('\n')) 103 | } 104 | ) 105 | }) 106 | 107 | it('should fast-exit for a subset of packages in stages', async () => { 108 | await withScaffold( 109 | { 110 | packages: pkgList(true) 111 | }, 112 | async () => { 113 | let tst = await wsrun('--stages -r --fast-exit doecho') 114 | expect(tst.stderr.toString()).toContain('Aborted execution due to previous error') 115 | let output = String(await echo.getOutput()) 116 | expect(output).toEqual(['p5', 'p4', ''].join('\n')) 117 | } 118 | ) 119 | }) 120 | 121 | it('should not fast-exit without fast-exit when parallel', async () => { 122 | await withScaffold( 123 | { 124 | packages: pkgList(true) 125 | }, 126 | async () => { 127 | let tst = await wsrun('doecho') 128 | expect(tst.status).toBeTruthy() 129 | let output = String(await echo.getOutput()) 130 | .split('\n') 131 | .sort() 132 | .reverse() 133 | expect(output).toEqual(['p5', 'p4', 'p2', 'p1', '']) 134 | } 135 | ) 136 | }) 137 | 138 | it('should limit parallelism with --concurrency', async () => { 139 | await withScaffold( 140 | { 141 | packages: pkgList(false) 142 | }, 143 | async () => { 144 | let tst = await wsrun('--parallel --concurrency 1 doecho') 145 | expect(tst.status).toBeFalsy() 146 | let output = String(await echo.getOutput()) 147 | // will run in order due to concurrency limit 148 | expect(output).toEqual('p5\np4\np3\np2\np1\n') 149 | } 150 | ) 151 | }) 152 | 153 | it('should support globs', async () => { 154 | await withScaffold( 155 | { 156 | packages: [ 157 | echo.makePkg({ name: 'app-x-frontend', dependencies: {} }), 158 | echo.makePkg({ name: 'app-x-backend', dependencies: {} }), 159 | echo.makePkg({ name: 'app-y-frontend', dependencies: { 'app-x-frontend': '*' } }) 160 | ] 161 | }, 162 | async () => { 163 | let tst = await wsrun('-p app-*-frontend --serial doecho') 164 | expect(tst.status).toBeFalsy() 165 | let output = String(await echo.getOutput()) 166 | expect(output).toEqual('app-x-frontend\napp-y-frontend\n') 167 | } 168 | ) 169 | }) 170 | 171 | it('should not break with namespaced pkgs', async () => { 172 | await withScaffold( 173 | { 174 | packages: [ 175 | echo.makePkg({ name: '@x/p1', path: 'packages/p1', dependencies: {} }), 176 | echo.makePkg({ name: '@x/p2', path: 'packages/p2', dependencies: { '@x/p1': '*' } }) 177 | ] 178 | }, 179 | async () => { 180 | let tst = await wsrun('--serial doecho') 181 | expect(tst.status).toBeFalsy() 182 | let output = String(await echo.getOutput()) 183 | expect(output).toEqual('@x/p1\n@x/p2\n') 184 | } 185 | ) 186 | }) 187 | 188 | it('should rewrite paths if instructed', async () => { 189 | await withScaffold( 190 | { 191 | packages: [ 192 | echo.makePkg({ name: 'app-x-frontend', dependencies: {} }, '', 'X src/index.ts testing') 193 | ] 194 | }, 195 | async () => { 196 | let tst = await wsrun('printthis', { WSRUN_REWRITE_PATHS: 'true' }) 197 | expect(tst.output.toString()).toContain('X packages/app-x-frontend/src/index.ts') 198 | } 199 | ) 200 | }) 201 | 202 | it('should not rewrite paths by default', async () => { 203 | await withScaffold( 204 | { 205 | packages: [ 206 | echo.makePkg( 207 | { name: 'app-x-frontend', dependencies: {} }, 208 | '', 209 | 'Output for path src/index.ts testing' 210 | ) 211 | ] 212 | }, 213 | async () => { 214 | let tst = await wsrun('printthis', {}) 215 | expect(tst.output.toString()).not.toContain('app-x-frontend/src/index.ts') 216 | } 217 | ) 218 | }) 219 | 220 | it('should show an error for pkgs without name', async () => { 221 | await withScaffold( 222 | { 223 | packages: [echo.makePkg({ path: 'packages/p1', dependencies: {} })] 224 | }, 225 | async () => { 226 | let tst = await wsrun('doecho') 227 | expect(tst.status).toBeTruthy() 228 | expect(String(tst.output[2])).toEqual( 229 | '\nERROR: Package in directory packages/p1 has no name in package.json\n' 230 | ) 231 | } 232 | ) 233 | }) 234 | 235 | function removePath(processOutput: string) { 236 | return processOutput.split(process.cwd()).join('') 237 | } 238 | it('should keep its stdout and stderr stable', async () => { 239 | await withScaffold( 240 | { 241 | packages: [ 242 | echo.makePkg({ name: '@x/p1', path: 'packages/p1', dependencies: {} }), 243 | echo.makePkg({ name: '@x/p2', path: 'packages/p2', dependencies: { '@x/p1': '*' } }) 244 | ] 245 | }, 246 | async () => { 247 | let tst = await wsrun('--serial doecho 0 Hello') 248 | expect(removePath(tst.stdout.toString())).toMatchSnapshot() 249 | expect(removePath(tst.stderr.toString())).toMatchSnapshot() 250 | } 251 | ) 252 | }) 253 | }) 254 | -------------------------------------------------------------------------------- /tests/runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export PATH=$PWD/node_modules/.bin:$PATH 4 | # RUNCMD=$(jq -r .scripts.$1 package.json) 5 | RUNCMD=$(node -e "console.log(JSON.parse(fs.readFileSync('package.json', 'utf8')).scripts['$1'])") 6 | 7 | if [ "xundefined" = "x$RUNCMD" ] 8 | then 9 | RUNCMD=$1 10 | fi 11 | 12 | BASHCMD="$RUNCMD ${@:2}" 13 | echo '$' $BASHCMD 14 | exec bash -c "$BASHCMD" 15 | -------------------------------------------------------------------------------- /tests/test.util.ts: -------------------------------------------------------------------------------- 1 | // scaffold takes: 2 | // * packages -> package.jsons (dependencies, devdependencies, name, scripts) 3 | // * 4 | // scaffold does: 5 | // * creates dir tmp, if exists, rim-raf 6 | // * in tmp, create packages dir and substructure for all packages 7 | // * in each package dir, create package.json and dump dependencies and scripts there. 8 | 9 | // spawn takes: 10 | // * command (wsrun etc etc) 11 | // * env var 12 | 13 | import * as fs from 'mz/fs' 14 | import * as rimraf from 'rimraf' 15 | import * as cp from 'child_process' 16 | import * as mkdirp from 'mkdirp' 17 | import * as path from 'path' 18 | 19 | import { promisify } from 'util' 20 | 21 | let rimrafAsync = promisify(rimraf) 22 | let mkdirpAsync = promisify(mkdirp) 23 | 24 | export type PackageJson = { 25 | name?: string 26 | path?: string 27 | dependencies?: { [name: string]: string } 28 | devDependencies?: { [name: string]: string } 29 | scripts?: { [name: string]: string } 30 | } 31 | 32 | let counter = process.env['JEST_WORKER_ID'] || '0' 33 | 34 | let testDir = `${process.cwd()}/tmp/wsrun-test-${counter}` 35 | 36 | export type ScaffoldOptions = { 37 | packages: PackageJson[] 38 | workspaces?: any 39 | } 40 | 41 | async function realExists(path: string) { 42 | try { 43 | return fs.exists(path) 44 | } catch (e) { 45 | return false 46 | } 47 | } 48 | 49 | export async function withScaffold(opts: ScaffoldOptions, f: () => PromiseLike) { 50 | if (await realExists(testDir)) await rimrafAsync(testDir) 51 | await mkdirpAsync(testDir) 52 | await fs.writeFile( 53 | `${testDir}/package.json`, 54 | JSON.stringify( 55 | { 56 | name: test, 57 | license: 'MIT', 58 | workspaces: opts.workspaces || { 59 | packages: ['packages/*'] 60 | } 61 | }, 62 | null, 63 | 2 64 | ) 65 | ) 66 | for (let pkg of opts.packages) { 67 | let pkgPath = pkg.path || `packages/${pkg.name}` 68 | let fullDir = `${testDir}/${pkgPath}` 69 | await mkdirpAsync(fullDir) 70 | await fs.writeFile(`${fullDir}/package.json`, JSON.stringify(pkg, null, 2)) 71 | } 72 | try { 73 | return await f() 74 | } finally { 75 | if (await realExists(testDir)) await rimrafAsync(testDir) 76 | } 77 | } 78 | 79 | export let echo = { 80 | makePkg(json: PackageJson, condition: string = '', printthis = '') { 81 | return Object.assign(json, { 82 | license: 'MIT', 83 | scripts: { 84 | doecho: `sleep $1; echo ${json.name} $1 >> '${testDir}/echo.out'`, 85 | condition, 86 | printthis: `echo ${printthis}` 87 | } 88 | }) 89 | }, 90 | makePkgErr(json: PackageJson, condition: string = '') { 91 | return Object.assign(json, { 92 | license: 'MIT', 93 | scripts: { 94 | doecho: `exit 1` 95 | } 96 | }) 97 | }, 98 | async getOutput() { 99 | return fs.readFile(`${testDir}/echo.out`, 'utf8') 100 | } 101 | } 102 | 103 | let pkgPath = path.resolve(__dirname, '..') 104 | let binPath = require('../package.json').bin.wsrun 105 | let wsrunPath = path.resolve(pkgPath, binPath) 106 | 107 | export async function wsrun(cmd: string | string[], env: { [key: string]: string } = {}) { 108 | if (typeof cmd === 'string') cmd = cmd.split(' ') 109 | return cp.spawnSync(wsrunPath, ['--bin=' + require.resolve('./runner.sh')].concat(cmd), { 110 | cwd: testDir, 111 | env: { ...process.env, ...env } 112 | }) 113 | } 114 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "include": ["src"], 3 | "compilerOptions": { 4 | "target": "es2015", 5 | "module": "commonjs", 6 | "moduleResolution": "node", 7 | "outDir": "./build", 8 | "strict": true, 9 | "experimentalDecorators": true, 10 | "sourceMap": true 11 | } 12 | } 13 | --------------------------------------------------------------------------------