├── .eslintignore ├── .eslintrc.js ├── .github └── workflows │ ├── pr.yml │ └── release.yml ├── .gitignore ├── .npmignore ├── .prettierrc ├── CHANGELOG.json ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── SECURITY.md ├── jest.config.js ├── package.json ├── src ├── __tests__ │ ├── generateTaskGraph.test.ts │ └── index.test.ts ├── generateTaskGraph.ts ├── index.ts ├── output.ts ├── pipeline.ts ├── publicInterfaces.ts ├── runAndLog.ts ├── taskId.ts └── types.ts ├── tsconfig.json └── yarn.lock /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | lib 3 | coverage 4 | 5 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | parser: "@typescript-eslint/parser", 4 | plugins: ["@typescript-eslint"], 5 | extends: [ 6 | "eslint:recommended", 7 | "plugin:@typescript-eslint/eslint-recommended", 8 | "plugin:@typescript-eslint/recommended", 9 | "plugin:import/errors", 10 | "plugin:import/warnings", 11 | "plugin:import/typescript", 12 | "prettier/@typescript-eslint", // this disables the linting error which conflict with prettier 13 | "plugin:prettier/recommended", // [Has to be last] this does prettier as part of the linting 14 | ], 15 | parserOptions: { 16 | ecmaVersion: 2018, 17 | sourceType: "module", 18 | }, 19 | rules: { 20 | "@typescript-eslint/no-non-null-assertion": "off", 21 | "@typescript-eslint/no-use-before-define": "off", 22 | "@typescript-eslint/no-var-requires": "off", 23 | "@typescript-eslint/explicit-function-return-type": "off", 24 | "import/order": "warn", 25 | }, 26 | }; 27 | -------------------------------------------------------------------------------- /.github/workflows/pr.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: PR 5 | 6 | on: 7 | pull_request: 8 | branches: [master] 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | 14 | strategy: 15 | matrix: 16 | node-version: [12.x] 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | - name: Use Node.js ${{ matrix.node-version }} 21 | uses: actions/setup-node@v1 22 | with: 23 | node-version: ${{ matrix.node-version }} 24 | - run: yarn 25 | - run: yarn checkchange 26 | - run: yarn lint 27 | - run: yarn build 28 | - run: yarn test 29 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: Release 5 | 6 | on: 7 | push: 8 | branches: [master] 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | 14 | strategy: 15 | matrix: 16 | node-version: [12.x] 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | with: 21 | token: ${{ secrets.repo_pat }} 22 | - name: Use Node.js ${{ matrix.node-version }} 23 | uses: actions/setup-node@v1 24 | with: 25 | node-version: ${{ matrix.node-version }} 26 | - run: yarn 27 | - run: yarn build 28 | - run: yarn checkchange 29 | - run: yarn lint 30 | - run: yarn test 31 | - run: | 32 | git config user.email "kchau@microsoft.com" 33 | git config user.name "Ken Chau" 34 | - run: yarn release -y -n $NPM_AUTHTOKEN --access public 35 | env: 36 | NPM_AUTHTOKEN: ${{ secrets.npm_authtoken }} 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | lib 3 | dist 4 | coverage 5 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | * 2 | !lib/**/* 3 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "endOfLine": "auto" 3 | } 4 | -------------------------------------------------------------------------------- /CHANGELOG.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@microsoft/task-scheduler", 3 | "entries": [ 4 | { 5 | "date": "Tue, 04 May 2021 06:50:40 GMT", 6 | "tag": "@microsoft/task-scheduler_v2.7.1", 7 | "version": "2.7.1", 8 | "comments": { 9 | "patch": [ 10 | { 11 | "comment": "chore(update): upgrade p-graph to latest version (#30)", 12 | "author": "cheruiyotbryan@gmail.com", 13 | "commit": "c6488d6554411681b324066dc9300447b908b31b", 14 | "package": "@microsoft/task-scheduler" 15 | } 16 | ] 17 | } 18 | }, 19 | { 20 | "date": "Thu, 22 Oct 2020 20:03:00 GMT", 21 | "tag": "@microsoft/task-scheduler_v2.7.0", 22 | "version": "2.7.0", 23 | "comments": { 24 | "minor": [ 25 | { 26 | "comment": "adds ability to continue task-scheduler even when one task has failed", 27 | "author": "kchau@microsoft.com", 28 | "commit": "82e31625c2635ebd17e40fb3bce2c42218b36d09", 29 | "package": "@microsoft/task-scheduler" 30 | } 31 | ] 32 | } 33 | }, 34 | { 35 | "date": "Thu, 22 Oct 2020 15:35:51 GMT", 36 | "tag": "@microsoft/task-scheduler_v2.6.5", 37 | "version": "2.6.5", 38 | "comments": { 39 | "patch": [ 40 | { 41 | "comment": "build: Remove bundling", 42 | "author": "olwheele@microsoft.com", 43 | "commit": "e8071db62f36f966be4e4b38d57338c4d40aca3a", 44 | "package": "@microsoft/task-scheduler" 45 | } 46 | ] 47 | } 48 | }, 49 | { 50 | "date": "Mon, 05 Oct 2020 19:11:43 GMT", 51 | "tag": "@microsoft/task-scheduler_v2.6.4", 52 | "version": "2.6.4", 53 | "comments": { 54 | "patch": [ 55 | { 56 | "comment": "bump pgraph", 57 | "author": "kchau@microsoft.com", 58 | "commit": "4156b855080820d47a3412cd56d14763435d305f", 59 | "package": "@microsoft/task-scheduler" 60 | } 61 | ] 62 | } 63 | }, 64 | { 65 | "date": "Mon, 05 Oct 2020 18:42:30 GMT", 66 | "tag": "@microsoft/task-scheduler_v2.6.3", 67 | "version": "2.6.3", 68 | "comments": { 69 | "patch": [ 70 | { 71 | "comment": "Add error message for missing pipeline config", 72 | "author": "elcraig@microsoft.com", 73 | "commit": "b845d13afbe33627eb427a978b629cefb464b97d", 74 | "package": "@microsoft/task-scheduler" 75 | } 76 | ] 77 | } 78 | }, 79 | { 80 | "date": "Wed, 26 Aug 2020 20:33:37 GMT", 81 | "tag": "@microsoft/task-scheduler_v2.6.2", 82 | "version": "2.6.2", 83 | "comments": { 84 | "patch": [ 85 | { 86 | "comment": "make sure no-deps case is actually covered", 87 | "author": "kchau@microsoft.com", 88 | "commit": "8a21c790f0e4e15750c4068e0c0478f7379ff052", 89 | "package": "@microsoft/task-scheduler" 90 | } 91 | ] 92 | } 93 | }, 94 | { 95 | "date": "Sun, 16 Aug 2020 16:27:52 GMT", 96 | "tag": "@microsoft/task-scheduler_v2.6.1", 97 | "version": "2.6.1", 98 | "comments": { 99 | "patch": [ 100 | { 101 | "comment": "using a different delimiter to make it not conflict with lots of build scripts out there", 102 | "author": "kchau@microsoft.com", 103 | "commit": "c6803ba306f92b3028d7f5e237c289c4905d1532", 104 | "package": "@microsoft/task-scheduler" 105 | } 106 | ] 107 | } 108 | }, 109 | { 110 | "date": "Fri, 14 Aug 2020 18:35:13 GMT", 111 | "tag": "@microsoft/task-scheduler_v2.6.0", 112 | "version": "2.6.0", 113 | "comments": { 114 | "minor": [ 115 | { 116 | "comment": "Adds the ability to add an individual package task dependency", 117 | "author": "kchau@microsoft.com", 118 | "commit": "da65db7e2ae37285cd52711e0b892d7cf44c2dcd", 119 | "package": "@microsoft/task-scheduler" 120 | } 121 | ] 122 | } 123 | }, 124 | { 125 | "date": "Thu, 13 Aug 2020 17:04:57 GMT", 126 | "tag": "@microsoft/task-scheduler_v2.5.0", 127 | "version": "2.5.0", 128 | "comments": { 129 | "minor": [ 130 | { 131 | "comment": "exposes generateTaskGraph as well so it can be used as a separate library util to examine the task graph", 132 | "author": "kchau@microsoft.com", 133 | "commit": "37e919c3108f2c4bb4142f342b94bbab419fa67c", 134 | "package": "@microsoft/task-scheduler" 135 | } 136 | ] 137 | } 138 | }, 139 | { 140 | "date": "Wed, 15 Jul 2020 14:37:22 GMT", 141 | "tag": "@microsoft/task-scheduler_v2.4.0", 142 | "version": "2.4.0", 143 | "comments": { 144 | "minor": [ 145 | { 146 | "comment": "Change priority API to be package specific", 147 | "author": "1581488+christiango@users.noreply.github.com", 148 | "commit": "4a7ca2427e590b9005dd356afb55537540094cc9", 149 | "package": "@microsoft/task-scheduler" 150 | } 151 | ] 152 | } 153 | }, 154 | { 155 | "date": "Tue, 14 Jul 2020 22:29:27 GMT", 156 | "tag": "@microsoft/task-scheduler_v2.3.0", 157 | "version": "2.3.0", 158 | "comments": { 159 | "minor": [ 160 | { 161 | "comment": "Upgrade to latest p-graph and add support for task prioritization and maximum concurrency limiting", 162 | "author": "1581488+christiango@users.noreply.github.com", 163 | "commit": "3f986796095998a6313c4890f54816683147fc07", 164 | "package": "@microsoft/task-scheduler" 165 | } 166 | ] 167 | } 168 | }, 169 | { 170 | "date": "Mon, 13 Jul 2020 22:28:02 GMT", 171 | "tag": "@microsoft/task-scheduler_v2.2.0", 172 | "version": "2.2.0", 173 | "comments": { 174 | "minor": [ 175 | { 176 | "comment": "adds a targets only mode", 177 | "author": "kchau@microsoft.com", 178 | "commit": "cac432b0db88649e52755b2ede8c837e2bf3ad4f", 179 | "package": "@microsoft/task-scheduler" 180 | } 181 | ] 182 | } 183 | }, 184 | { 185 | "date": "Wed, 17 Jun 2020 15:56:12 GMT", 186 | "tag": "@microsoft/task-scheduler_v2.1.3", 187 | "version": "2.1.3", 188 | "comments": { 189 | "patch": [ 190 | { 191 | "comment": "adding public access since this is microsoft scoped; fix graph generation", 192 | "author": "kchau@microsoft.com", 193 | "commit": "80db31bd1f02c191dc8626d610af56fc1035853a", 194 | "package": "@microsoft/task-scheduler" 195 | } 196 | ] 197 | } 198 | }, 199 | { 200 | "date": "Fri, 05 Jun 2020 16:18:43 GMT", 201 | "tag": "@microsoft/task-scheduler_v2.1.2", 202 | "version": "2.1.2", 203 | "comments": { 204 | "patch": [ 205 | { 206 | "comment": "restore the webpacked output", 207 | "author": "kchau@microsoft.com", 208 | "commit": "34ef65fbf110d961d26c5496eb88e8b60ef23711", 209 | "package": "@microsoft/task-scheduler" 210 | } 211 | ] 212 | } 213 | }, 214 | { 215 | "date": "Fri, 05 Jun 2020 16:06:20 GMT", 216 | "tag": "@microsoft/task-scheduler_v2.1.1", 217 | "version": "2.1.1", 218 | "comments": { 219 | "patch": [ 220 | { 221 | "comment": "fixes npmignore to include all the lib files", 222 | "author": "kchau@microsoft.com", 223 | "commit": "5c6e26133af8c513d253d1662b80783361df8d34", 224 | "package": "@microsoft/task-scheduler" 225 | } 226 | ] 227 | } 228 | }, 229 | { 230 | "date": "Fri, 05 Jun 2020 15:53:52 GMT", 231 | "tag": "@microsoft/task-scheduler_v2.1.0", 232 | "version": "2.1.0", 233 | "comments": { 234 | "minor": [ 235 | { 236 | "comment": "adding an override for exit() so consumers can handle the exits", 237 | "author": "kchau@microsoft.com", 238 | "commit": "c855019a0f8b018c02707f963f3e2d8a76bb7a54", 239 | "package": "@microsoft/task-scheduler" 240 | } 241 | ] 242 | } 243 | }, 244 | { 245 | "date": "Wed, 03 Jun 2020 23:26:58 GMT", 246 | "tag": "@microsoft/task-scheduler_v2.0.2", 247 | "version": "2.0.2", 248 | "comments": { 249 | "patch": [ 250 | { 251 | "comment": "allow override of logger in task-scheduler", 252 | "author": "kchau@microsoft.com", 253 | "commit": "8e2934371bcf8ee0241d132f845d9976c972da02", 254 | "package": "@microsoft/task-scheduler" 255 | } 256 | ] 257 | } 258 | }, 259 | { 260 | "date": "Wed, 03 Jun 2020 20:49:24 GMT", 261 | "tag": "@microsoft/task-scheduler_v2.0.1", 262 | "version": "2.0.1", 263 | "comments": { 264 | "patch": [ 265 | { 266 | "comment": "adding beachball for releases and also adding an arg to the task run function to give more info to the consumers", 267 | "author": "kchau@microsoft.com", 268 | "commit": "5256915f8152ab09fe8ea68d7629baa9c337defa", 269 | "package": "@microsoft/task-scheduler" 270 | } 271 | ] 272 | } 273 | } 274 | ] 275 | } 276 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log - @microsoft/task-scheduler 2 | 3 | This log was last generated on Tue, 04 May 2021 06:50:40 GMT and should not be manually modified. 4 | 5 | 6 | 7 | ## 2.7.1 8 | 9 | Tue, 04 May 2021 06:50:40 GMT 10 | 11 | ### Patches 12 | 13 | - chore(update): upgrade p-graph to latest version (#30) (cheruiyotbryan@gmail.com) 14 | 15 | ## 2.7.0 16 | 17 | Thu, 22 Oct 2020 20:03:00 GMT 18 | 19 | ### Minor changes 20 | 21 | - adds ability to continue task-scheduler even when one task has failed (kchau@microsoft.com) 22 | 23 | ## 2.6.5 24 | 25 | Thu, 22 Oct 2020 15:35:51 GMT 26 | 27 | ### Patches 28 | 29 | - build: Remove bundling (olwheele@microsoft.com) 30 | 31 | ## 2.6.4 32 | 33 | Mon, 05 Oct 2020 19:11:43 GMT 34 | 35 | ### Patches 36 | 37 | - bump pgraph (kchau@microsoft.com) 38 | 39 | ## 2.6.3 40 | 41 | Mon, 05 Oct 2020 18:42:30 GMT 42 | 43 | ### Patches 44 | 45 | - Add error message for missing pipeline config (elcraig@microsoft.com) 46 | 47 | ## 2.6.2 48 | 49 | Wed, 26 Aug 2020 20:33:37 GMT 50 | 51 | ### Patches 52 | 53 | - make sure no-deps case is actually covered (kchau@microsoft.com) 54 | 55 | ## 2.6.1 56 | 57 | Sun, 16 Aug 2020 16:27:52 GMT 58 | 59 | ### Patches 60 | 61 | - using a different delimiter to make it not conflict with lots of build scripts out there (kchau@microsoft.com) 62 | 63 | ## 2.6.0 64 | 65 | Fri, 14 Aug 2020 18:35:13 GMT 66 | 67 | ### Minor changes 68 | 69 | - Adds the ability to add an individual package task dependency (kchau@microsoft.com) 70 | 71 | ## 2.5.0 72 | 73 | Thu, 13 Aug 2020 17:04:57 GMT 74 | 75 | ### Minor changes 76 | 77 | - exposes generateTaskGraph as well so it can be used as a separate library util to examine the task graph (kchau@microsoft.com) 78 | 79 | ## 2.4.0 80 | 81 | Wed, 15 Jul 2020 14:37:22 GMT 82 | 83 | ### Minor changes 84 | 85 | - Change priority API to be package specific (1581488+christiango@users.noreply.github.com) 86 | 87 | ## 2.3.0 88 | 89 | Tue, 14 Jul 2020 22:29:27 GMT 90 | 91 | ### Minor changes 92 | 93 | - Upgrade to latest p-graph and add support for task prioritization and maximum concurrency limiting (1581488+christiango@users.noreply.github.com) 94 | 95 | ## 2.2.0 96 | 97 | Mon, 13 Jul 2020 22:28:02 GMT 98 | 99 | ### Minor changes 100 | 101 | - adds a targets only mode (kchau@microsoft.com) 102 | 103 | ## 2.1.3 104 | 105 | Wed, 17 Jun 2020 15:56:12 GMT 106 | 107 | ### Patches 108 | 109 | - adding public access since this is microsoft scoped; fix graph generation (kchau@microsoft.com) 110 | 111 | ## 2.1.2 112 | 113 | Fri, 05 Jun 2020 16:18:43 GMT 114 | 115 | ### Patches 116 | 117 | - restore the webpacked output (kchau@microsoft.com) 118 | 119 | ## 2.1.1 120 | 121 | Fri, 05 Jun 2020 16:06:20 GMT 122 | 123 | ### Patches 124 | 125 | - fixes npmignore to include all the lib files (kchau@microsoft.com) 126 | 127 | ## 2.1.0 128 | 129 | Fri, 05 Jun 2020 15:53:52 GMT 130 | 131 | ### Minor changes 132 | 133 | - adding an override for exit() so consumers can handle the exits (kchau@microsoft.com) 134 | 135 | ## 2.0.2 136 | 137 | Wed, 03 Jun 2020 23:26:58 GMT 138 | 139 | ### Patches 140 | 141 | - allow override of logger in task-scheduler (kchau@microsoft.com) 142 | 143 | ## 2.0.1 144 | 145 | Wed, 03 Jun 2020 20:49:24 GMT 146 | 147 | ### Patches 148 | 149 | - adding beachball for releases and also adding an arg to the task run function to give more info to the consumers (kchau@microsoft.com) 150 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # @microsoft/task-scheduler 2 | 3 | Run a sequence of steps across all the packages of a monorepo. 4 | 5 | # Why 6 | 7 | - This tool does not assume any workspace/package manager so it can be used on any JavaScript repository. 8 | - The steps run on the main thread, sparing the cost of spawning one process per step. If parallelization is needed, the implementation of the steps can spawn processes. 9 | - This tool optimizes CI builds performance by avoiding unnecessary waiting (see example below). 10 | - This tool has no dependencies and is very small. 11 | - Its interface makes it easy to compose with other tools to get fancy pipelines (eg. parallelization, profiling, throttling...) 12 | - Running the tasks on the main node process allows for cross-step in-memory memoization 13 | 14 | # Usage 15 | 16 | ```js 17 | const { createPipeline } = require("@microsoft/task-scheduler"); 18 | 19 | // this graph describes a topological graph 20 | // e.g. {foo: {location: 'packages/foo', dependencies: ['bar']}, bar: { ... }} 21 | const graph = getDependencyGraph(); 22 | 23 | const pipeline = await createPipeline(graph) 24 | // defining a task with NO task dependencies 25 | .addTask({ 26 | name: "prepare", 27 | run: prepare 28 | }) 29 | // defining a task with task dependencies as well as the topological deps 30 | .addTask({ 31 | name: "build", 32 | run: build, 33 | deps: ["prepare"], 34 | topoDeps: ["build"] 35 | }) 36 | .addTask({ 37 | name: "test", 38 | run: test, 39 | deps: ["build"] 40 | }) 41 | .addTask({ 42 | name: "bundle", 43 | run: bundle, 44 | deps: ["build"] 45 | }) 46 | // you can call go() with no parameters to target everything, or specify which packages or tasks to target 47 | .go({ 48 | packages: ["foo", "bar"], 49 | tasks: ["test", "bundle"] 50 | }); 51 | 52 | async function prepare(cwd, stdout, stderr) { 53 | ... 54 | } 55 | 56 | async function build(cwd, stdout, stderr) { 57 | ... 58 | } 59 | 60 | async function test(cwd, stdout, stderr) { 61 | ... 62 | } 63 | 64 | async function bundle(cwd, stdout, stderr) { 65 | ... 66 | } 67 | 68 | ``` 69 | 70 | A `Task` is described by this: 71 | 72 | ```ts 73 | type Task = { 74 | /** name of the task */ 75 | name: string; 76 | 77 | /** a function that gets invoked by the task-scheduler */ 78 | run: (cwd: string, stdout: Writable, stderr: Writable) => Promise; 79 | 80 | /** dependencies between tasks within the same package (e.g. `build` -> `test`) */ 81 | deps?: string[]; 82 | 83 | /** dependencies across packages within the same topological graph (e.g. parent `build` -> child `build`) */ 84 | topoDeps?: string[]; 85 | 86 | /** An optional priority to set for packages that have this task. Unblocked tasks with a higher priority will be scheduled before lower priority tasks. */ 87 | priorities?: { 88 | [packageName: string]: number; 89 | }; 90 | }; 91 | ``` 92 | 93 | Here is how the tasks defined above would run on a repo which has two packages A and B, A depending on B: 94 | 95 | ``` 96 | 97 | A: [-prepare-] [------build------] [----test----] 98 | [-----bundle-----] 99 | B: [-prepare-] [------build------] [----test----] 100 | [-----bundle-----] 101 | ----------> time 102 | ``` 103 | 104 | Here is how the same workflow would be executed by using lerna: 105 | 106 | ``` 107 | 108 | A: [-prepare-] [------build------] [----test----][-----bundle-----] 109 | 110 | B: [-prepare-] [------build------] [----test----][-----bundle-----] 111 | 112 | ----------> time 113 | ``` 114 | 115 | # Contributing 116 | 117 | ## Development 118 | 119 | This repo uses `beachball` for automated releases and semver. Please include a change file by running: 120 | 121 | ``` 122 | $ yarn change 123 | ``` 124 | 125 | ## CLA 126 | 127 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 128 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 129 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 130 | 131 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 132 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 133 | provided by the bot. You will only need to do this once across all repos using our CLA. 134 | 135 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 136 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 137 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 138 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets Microsoft's [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)) of a security vulnerability, please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). 40 | 41 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | }; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@microsoft/task-scheduler", 3 | "version": "2.7.1", 4 | "description": "Schedule tasks in a monorepo", 5 | "repository": "git@github.com:microsoft/task-scheduler.git", 6 | "license": "MIT", 7 | "author": "Vincent Bailly ", 8 | "main": "lib/index.js", 9 | "typings": "lib/index.d.ts", 10 | "scripts": { 11 | "build": "tsc", 12 | "change": "beachball change", 13 | "checkchange": "beachball check", 14 | "lint": "eslint . --ext .ts", 15 | "lint:fix": "eslint --fix . --ext .ts", 16 | "release": "beachball publish -y", 17 | "start": "tsc -w --preserveWatchOutput", 18 | "test": "jest" 19 | }, 20 | "husky": { 21 | "hooks": { 22 | "pre-commit": "lint-staged" 23 | } 24 | }, 25 | "lint-staged": { 26 | "**/*.{js,json,md}": [ 27 | "prettier --write", 28 | "git add" 29 | ], 30 | "**/*.{ts}": [ 31 | "eslint --fix", 32 | "git add" 33 | ] 34 | }, 35 | "dependencies": { 36 | "memory-streams": "^0.1.3", 37 | "p-graph": "^1.1.0" 38 | }, 39 | "devDependencies": { 40 | "@types/jest": "^25.1.4", 41 | "@types/node": "12.x.x", 42 | "@typescript-eslint/eslint-plugin": "^2.26.0", 43 | "@typescript-eslint/parser": "^2.26.0", 44 | "beachball": "^1.31.2", 45 | "eslint": "^6.8.0", 46 | "eslint-config-prettier": "^6.10.1", 47 | "eslint-plugin-import": "^2.20.2", 48 | "eslint-plugin-prettier": "^3.1.2", 49 | "husk": "^0.5.3", 50 | "jest": "^25.2.4", 51 | "lint-staged": "^10.1.1", 52 | "prettier": "^2.0.2", 53 | "ts-jest": "^25.3.0", 54 | "typescript": "^3.8.3" 55 | }, 56 | "publishConfig": { 57 | "access": "public" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/__tests__/generateTaskGraph.test.ts: -------------------------------------------------------------------------------- 1 | import { generateTaskGraph } from "../generateTaskGraph"; 2 | import { Tasks } from "../types"; 3 | import { getPackageTaskFromId } from "../taskId"; 4 | 5 | describe("generateTaskGraph", () => { 6 | const graph = { 7 | A: { location: "a", dependencies: ["B"] }, 8 | B: { location: "b", dependencies: [] }, 9 | }; 10 | 11 | test("targetOnly mode includes only tasks listed in targets array", async () => { 12 | const scope = ["A", "B"]; 13 | const targets = ["test", "bundle"]; 14 | const tasks: Tasks = new Map([ 15 | [ 16 | "build", 17 | { 18 | name: "build", 19 | run: () => Promise.resolve(true), 20 | deps: [], 21 | topoDeps: ["build"], 22 | }, 23 | ], 24 | [ 25 | "test", 26 | { 27 | name: "test", 28 | run: () => Promise.resolve(true), 29 | deps: [], 30 | topoDeps: [], 31 | }, 32 | ], 33 | [ 34 | "bundle", 35 | { 36 | name: "bundle", 37 | run: () => Promise.resolve(true), 38 | deps: ["build"], 39 | topoDeps: [], 40 | }, 41 | ], 42 | ]); 43 | 44 | const taskGraph = generateTaskGraph(scope, targets, tasks, graph, [], true); 45 | expect(taskGraph).toHaveLength(4); 46 | 47 | // None of the "from" taskId should contain "build" task 48 | expect( 49 | !taskGraph.find((entry) => getPackageTaskFromId(entry[0])[1] === "build") 50 | ).toBeTruthy(); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /src/__tests__/index.test.ts: -------------------------------------------------------------------------------- 1 | import { EOL } from "os"; 2 | import { Writable } from "stream"; 3 | import { createPipelineInternal } from "../pipeline"; 4 | import { Task, Globals } from "../publicInterfaces"; 5 | 6 | describe("task scheduling", () => { 7 | const graph = { 8 | A: { location: "a", dependencies: ["B"] }, 9 | B: { location: "b", dependencies: [] }, 10 | }; 11 | 12 | test("tological steps wait for dependencies to be done", async () => { 13 | const tracingContext = makeTestEnvironment(); 14 | const task = tracingContext.makeTask(); 15 | 16 | task.topoDeps = [task.name]; 17 | 18 | await createPipelineInternal(graph, getGlobals()).addTask(task).go(); 19 | 20 | const expected = [ 21 | task.started("/b"), 22 | task.finished("/b"), 23 | task.started("/a"), 24 | task.finished("/a"), 25 | ]; 26 | 27 | expected.forEach((e, i) => expect(e).toBe(tracingContext.logs[i])); 28 | }); 29 | 30 | test("parallel steps dont wait for dependencies to be done", async () => { 31 | const tracingContext = makeTestEnvironment(); 32 | const task = tracingContext.makeTask(); 33 | 34 | await createPipelineInternal(graph, getGlobals()).addTask(task).go(); 35 | 36 | const expected = [ 37 | task.started("/a"), 38 | task.started("/b"), 39 | task.finished("/a"), 40 | task.finished("/b"), 41 | ]; 42 | 43 | expected.forEach((e, i) => expect(e).toBe(tracingContext.logs[i])); 44 | }); 45 | 46 | test("tological steps wait for the previous step", async () => { 47 | const tracingContext = makeTestEnvironment(); 48 | const task1 = tracingContext.makeTask(); 49 | const task2 = tracingContext.makeTask(); 50 | 51 | task2.deps = [task1.name]; 52 | 53 | await createPipelineInternal(graph, getGlobals()) 54 | .addTask(task1) 55 | .addTask(task2) 56 | .go(); 57 | 58 | const expected = [ 59 | task1.started("/b"), 60 | task1.finished("/b"), 61 | task2.started("/b"), 62 | task2.finished("/b"), 63 | ]; 64 | 65 | expected.forEach((e, i) => 66 | expect(e).toBe( 67 | tracingContext.logs.filter((line) => line.includes("/b"))[i] 68 | ) 69 | ); 70 | }); 71 | 72 | test("parallel steps run in parallel for same package", async () => { 73 | const tracingContext = makeTestEnvironment(); 74 | const task1 = tracingContext.makeTask(); 75 | const task2 = tracingContext.makeTask(); 76 | 77 | await createPipelineInternal(graph, getGlobals()) 78 | .addTask(task1) 79 | .addTask(task2) 80 | .go(); 81 | 82 | const expected = [ 83 | task1.started("/b"), 84 | task2.started("/b"), 85 | task1.finished("/b"), 86 | task2.finished("/b"), 87 | ]; 88 | 89 | expected.forEach((e, i) => 90 | expect(e).toBe( 91 | tracingContext.logs.filter((line) => line.includes("/b"))[i] 92 | ) 93 | ); 94 | }); 95 | }); 96 | 97 | describe("failing steps", () => { 98 | test("a failing step fails the entire process", async () => { 99 | const graph = { 100 | A: { location: "a", dependencies: [] }, 101 | }; 102 | 103 | const tracingContext = makeTestEnvironment(); 104 | const step = tracingContext.makeTask({ success: false }); 105 | const globals = getGlobals(); 106 | 107 | await createPipelineInternal(graph, globals).addTask(step).go(); 108 | 109 | expect(globals.exitCode).toBe(1); 110 | }); 111 | 112 | test("the second step is not run if the first one fails", async () => { 113 | const graph = { 114 | A: { location: "a", dependencies: [] }, 115 | }; 116 | 117 | const tracingContext = makeTestEnvironment(); 118 | const task1 = tracingContext.makeTask({ success: false }); 119 | const task2 = tracingContext.makeTask(); 120 | 121 | task2.deps = [task1.name]; 122 | 123 | await createPipelineInternal(graph, getGlobals()) 124 | .addTask(task1) 125 | .addTask(task2) 126 | .go(); 127 | 128 | expect( 129 | tracingContext.logs.filter((l) => l.includes(task1.started("/a"))).length 130 | ).toBe(1); 131 | expect( 132 | tracingContext.logs.filter((l) => l.includes(task2.started("/a"))).length 133 | ).toBe(0); 134 | }); 135 | }); 136 | 137 | describe("output", () => { 138 | test("validating step output", async () => { 139 | const graph = { 140 | A: { location: "a", dependencies: [] }, 141 | }; 142 | 143 | const tracingContext = makeTestEnvironment(); 144 | const task = tracingContext.makeTask({ 145 | stdout: "task stdout", 146 | stderr: "task stderr", 147 | }); 148 | 149 | const globals = getGlobals(); 150 | await createPipelineInternal(graph, globals).addTask(task).go(); 151 | 152 | const expectedStdout: string[] = [ 153 | ` / Done ${task.name} in A`, 154 | ` | STDOUT`, 155 | ` | | task stdout`, 156 | ` | STDERR`, 157 | ` | | task stderr`, 158 | ` \\ Done ${task.name} in A`, 159 | ``, 160 | ]; 161 | const expectedStderr: string[] = []; 162 | 163 | globals.validateOuput(expectedStdout, expectedStderr); 164 | }); 165 | 166 | test("validating step output with nothing written to console", async () => { 167 | const graph = { 168 | A: { location: "a", dependencies: [] }, 169 | }; 170 | 171 | const tracingContext = makeTestEnvironment(); 172 | const task = tracingContext.makeTask(); 173 | 174 | const globals = getGlobals(); 175 | await createPipelineInternal(graph, globals).addTask(task).go(); 176 | 177 | const expectedStdout: string[] = [`Done ${task.name} in A`, ""]; 178 | const expectedStderr: string[] = []; 179 | 180 | globals.validateOuput(expectedStdout, expectedStderr); 181 | }); 182 | 183 | test("validating failing step output with nothing written to console", async () => { 184 | const graph = { 185 | A: { location: "a", dependencies: [] }, 186 | }; 187 | 188 | const tracingContext = makeTestEnvironment(); 189 | const task = tracingContext.makeTask({ success: false }); 190 | 191 | const globals = getGlobals(); 192 | await createPipelineInternal(graph, globals).addTask(task).go(); 193 | 194 | const expectedStdout: string[] = []; 195 | const expectedStderr: string[] = [`Failed ${task.name} in A`, ``]; 196 | 197 | globals.validateOuput(expectedStdout, expectedStderr); 198 | }); 199 | 200 | test("validating throwing step output", async () => { 201 | const graph = { 202 | A: { location: "a", dependencies: [] }, 203 | }; 204 | 205 | const tracingContext = makeTestEnvironment(); 206 | const task = tracingContext.makeTask({ 207 | success: new Error("failing miserably"), 208 | stderr: "task stderr", 209 | stdout: "task stdout", 210 | }); 211 | 212 | const globals = getGlobals(); 213 | await createPipelineInternal(graph, globals).addTask(task).go(); 214 | 215 | const expectedStderr: string[] = [ 216 | ` / Failed ${task.name} in A`, 217 | ` | STDOUT`, 218 | ` | | task stdout`, 219 | ` | STDERR`, 220 | ` | | task stderr`, 221 | ` | | stack trace for following error: failing miserably`, 222 | ` \\ Failed ${task.name} in A`, 223 | ``, 224 | ]; 225 | const expectedStdout: string[] = []; 226 | 227 | globals.validateOuput(expectedStdout, expectedStderr); 228 | }); 229 | 230 | test("validate output with two steps", async () => { 231 | const graph = { 232 | A: { location: "a", dependencies: [] }, 233 | }; 234 | 235 | const tracingContext = makeTestEnvironment(); 236 | const task1 = tracingContext.makeTask({ 237 | stdout: "task1 stdout", 238 | }); 239 | const task2 = tracingContext.makeTask({ 240 | stdout: "task2 stdout", 241 | }); 242 | 243 | task2.deps = [task1.name]; 244 | 245 | const globals = getGlobals(); 246 | await createPipelineInternal(graph, globals) 247 | .addTask(task1) 248 | .addTask(task2) 249 | .go(); 250 | 251 | const expectedStdout: string[] = [ 252 | ` / Done ${task1.name} in A`, 253 | ` | STDOUT`, 254 | ` | | task1 stdout`, 255 | ` \\ Done ${task1.name} in A`, 256 | ``, 257 | ` / Done ${task2.name} in A`, 258 | ` | STDOUT`, 259 | ` | | task2 stdout`, 260 | ` \\ Done ${task2.name} in A`, 261 | ``, 262 | ]; 263 | const expectedStderr: string[] = []; 264 | 265 | globals.validateOuput(expectedStdout, expectedStderr); 266 | }); 267 | 268 | test("the message of the failing step is output at the end", async () => { 269 | const graph = { 270 | A: { location: "a", dependencies: ["B"] }, 271 | B: { location: "b", dependencies: [] }, 272 | }; 273 | 274 | const run = async ( 275 | cwd: string, 276 | stdout: Writable, 277 | stderr: Writable 278 | ): Promise => { 279 | if (cwd.replace(/\\/g, "/") === "/a") { 280 | stdout.write(`task1 stdout${EOL}`); 281 | return true; 282 | } else { 283 | stderr.write(`task1 failed${EOL}`); 284 | return false; 285 | } 286 | }; 287 | 288 | const globals = getGlobals(true); 289 | 290 | await createPipelineInternal(graph, globals) 291 | .addTask({ name: "task1", run }) 292 | .go(); 293 | 294 | const expectedStdout: string[] = [ 295 | ` / Done task1 in A`, 296 | ` | STDOUT`, 297 | ` | | task1 stdout`, 298 | ` \\ Done task1 in A`, 299 | ``, 300 | ` / Failed task1 in B`, 301 | ` | STDERR`, 302 | ` | | task1 failed`, 303 | ` \\ Failed task1 in B`, 304 | ``, 305 | ]; 306 | 307 | globals.validateOuput(expectedStdout, expectedStdout); 308 | }); 309 | }); 310 | 311 | type TestingGlobals = Globals & { 312 | validateOuput(expectedStdout: string[], expectedStderr: string[]): void; 313 | stdout: string[]; 314 | stderr: string[]; 315 | exitCode: number; 316 | }; 317 | 318 | function getGlobals(stdoutAsStderr = false): TestingGlobals { 319 | const _stdout: string[] = []; 320 | const _stderr: string[] = stdoutAsStderr ? _stdout : []; 321 | let _exitCode = 0; 322 | 323 | return { 324 | validateOuput(expectedStdout: string[], expectedStderr: string[]): void { 325 | expect(_stderr.length).toBe(expectedStderr.length); 326 | expect(_stdout.length).toBe(expectedStdout.length); 327 | expectedStdout.forEach((m, i) => expect(_stdout[i]).toBe(m)); 328 | expectedStderr.forEach((m, i) => expect(_stderr[i]).toBe(m)); 329 | }, 330 | logger: { 331 | log(message: string): void { 332 | message.split(EOL).forEach((m) => _stdout.push(m)); 333 | }, 334 | error(message: string): void { 335 | message.split(EOL).forEach((m) => _stderr.push(m)); 336 | }, 337 | }, 338 | cwd(): string { 339 | return "/"; 340 | }, 341 | exit(int: number): void { 342 | _exitCode = int; 343 | }, 344 | get stdout(): string[] { 345 | return _stdout; 346 | }, 347 | get stderr(): string[] { 348 | return _stderr; 349 | }, 350 | get exitCode(): number { 351 | return _exitCode; 352 | }, 353 | errorFormatter(err: Error): string { 354 | return `stack trace for following error: ${err.message}`; 355 | }, 356 | targetsOnly: false, 357 | }; 358 | } 359 | 360 | type TaskResult = { 361 | success: true | false | Error; 362 | stdout: string; 363 | stderr: string; 364 | }; 365 | 366 | type TaskResultOverride = { 367 | success?: true | false | Error; 368 | stdout?: string; 369 | stderr?: string; 370 | }; 371 | 372 | type TaskMock = Task & { 373 | started: (cwd: string) => string; 374 | finished: (cwd: string) => string; 375 | }; 376 | 377 | async function wait(): Promise { 378 | return new Promise((resolve) => setTimeout(resolve, 50)); 379 | } 380 | 381 | function makeTestEnvironment(): { 382 | logs: string[]; 383 | makeTask: (desiredResult?: TaskResultOverride) => TaskMock; 384 | } { 385 | const logs: string[] = []; 386 | return { 387 | logs, 388 | makeTask(desiredResult?: TaskResultOverride): TaskMock { 389 | const name = Math.random().toString(36); 390 | const defaultResult: TaskResult = { 391 | success: true, 392 | stdout: "", 393 | stderr: "", 394 | }; 395 | 396 | const result = desiredResult 397 | ? { ...defaultResult, ...desiredResult } 398 | : defaultResult; 399 | 400 | const messages = { 401 | started(cwd: string): string { 402 | return `called ${name} for ${cwd.replace(/\\/g, "/")}`; 403 | }, 404 | finished(cwd: string): string { 405 | return `finished ${name} for ${cwd.replace(/\\/g, "/")}`; 406 | }, 407 | }; 408 | 409 | const run = async ( 410 | cwd: string, 411 | stdout: Writable, 412 | stderr: Writable 413 | ): Promise => { 414 | logs.push(messages.started(cwd)); 415 | stdout.write(result.stdout); 416 | stderr.write(result.stderr); 417 | await wait(); 418 | if (typeof result.success === "object") { 419 | logs.push(messages.finished(cwd)); 420 | throw result.success; 421 | } else { 422 | logs.push(messages.finished(cwd)); 423 | return result.success; 424 | } 425 | }; 426 | 427 | return { run, name, ...messages }; 428 | }, 429 | }; 430 | } 431 | -------------------------------------------------------------------------------- /src/generateTaskGraph.ts: -------------------------------------------------------------------------------- 1 | import { getTaskId, getPackageTaskFromId } from "./taskId"; 2 | import { TopologicalGraph, Tasks, TaskId, PackageTaskDeps } from "./types"; 3 | 4 | export function generateTaskGraph( 5 | scope: string[], 6 | targets: string[], 7 | tasks: Tasks, 8 | graph: TopologicalGraph, 9 | packageTaskDeps: PackageTaskDeps = [], 10 | targetsOnly = false 11 | ): PackageTaskDeps { 12 | const taskDeps: PackageTaskDeps = []; 13 | 14 | // These are the manually added package task dependencies from "addDep()" API 15 | const packageTaskDepsMap = getPackageTaskDepsMap(packageTaskDeps); 16 | 17 | const traversalQueue: TaskId[] = []; 18 | 19 | for (const pkg of scope) { 20 | for (const target of targets) { 21 | traversalQueue.push(getTaskId(pkg, target)); 22 | } 23 | } 24 | 25 | const visited = new Set(); 26 | 27 | while (traversalQueue.length > 0) { 28 | const taskId = traversalQueue.shift()!; 29 | const [pkg, taskName] = getPackageTaskFromId(taskId); 30 | 31 | if (!visited.has(taskId) && tasks.has(taskName)) { 32 | visited.add(taskId); 33 | const task = tasks.get(taskName)!; 34 | 35 | // If we're in targetsOnly mode, make sure none of the non-targets are filtered out of task.deps 36 | task.deps = targetsOnly 37 | ? task.deps?.filter((d) => targets.indexOf(d) > -1) 38 | : task.deps; 39 | 40 | const toTaskId = getTaskId(pkg, taskName); 41 | 42 | const hasTopoDeps = 43 | task.topoDeps && 44 | task.topoDeps.length > 0 && 45 | typeof graph[pkg].dependencies !== "undefined" && 46 | Object.keys(graph[pkg].dependencies).length > 0; 47 | const hasDeps = task.deps && task.deps.length > 0; 48 | const hasPackagetTaskDeps = packageTaskDepsMap.has(taskId); 49 | 50 | if (hasTopoDeps) { 51 | for (const from of task.topoDeps!) { 52 | const depPkgs = graph[pkg].dependencies; 53 | 54 | // add task dep from all the package deps within repo 55 | for (const depPkg of depPkgs) { 56 | const fromTaskId = getTaskId(depPkg, from); 57 | taskDeps.push([fromTaskId, toTaskId]); 58 | traversalQueue.push(fromTaskId); 59 | } 60 | } 61 | } 62 | 63 | if (hasDeps) { 64 | for (const from of task.deps!) { 65 | const fromTaskId = getTaskId(pkg, from); 66 | taskDeps.push([fromTaskId, toTaskId]); 67 | traversalQueue.push(fromTaskId); 68 | } 69 | } 70 | 71 | if (hasPackagetTaskDeps) { 72 | for (const fromTaskId of packageTaskDepsMap.get(taskId)!) { 73 | taskDeps.push([fromTaskId, toTaskId]); 74 | traversalQueue.push(fromTaskId); 75 | } 76 | } 77 | 78 | if (!hasDeps && !hasTopoDeps && !hasPackagetTaskDeps) { 79 | const fromTaskId = getTaskId(pkg, ""); 80 | taskDeps.push([fromTaskId, toTaskId]); 81 | } 82 | } 83 | } 84 | 85 | return taskDeps; 86 | } 87 | 88 | function getPackageTaskDepsMap(packageTaskDeps: PackageTaskDeps) { 89 | const depMap = new Map(); 90 | for (const [from, to] of packageTaskDeps) { 91 | if (!depMap.has(to)) { 92 | depMap.set(to, []); 93 | } 94 | depMap.get(to)!.push(from); 95 | } 96 | return depMap; 97 | } 98 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export { 2 | TopologicalGraph, 3 | Task, 4 | Pipeline, 5 | PackageTask, 6 | } from "./publicInterfaces"; 7 | export { createPipeline } from "./pipeline"; 8 | export { generateTaskGraph } from "./generateTaskGraph"; 9 | export { getTaskId, getPackageTaskFromId } from "./taskId"; 10 | -------------------------------------------------------------------------------- /src/output.ts: -------------------------------------------------------------------------------- 1 | import { EOL } from "os"; 2 | import { Logger, TaskResult } from "./types"; 3 | 4 | /* 5 | * Ouptut to console the result of a task. 6 | */ 7 | export function outputResult( 8 | message: string, 9 | p: string, 10 | name: string, 11 | result: "success" | "failure", 12 | logger: Logger 13 | ): void { 14 | const state = result === "success" ? "Done" : "Failed"; 15 | const log = result === "success" ? logger.log : logger.error; 16 | if (message === "") { 17 | log(`${state} ${name} in ${p}${EOL}`); 18 | } else { 19 | log(` / ${state} ${name} in ${p}`); 20 | log(prefix(message, " | ")); 21 | log(` \\ ${state} ${name} in ${p}${EOL}`); 22 | } 23 | } 24 | 25 | /* 26 | * Take a block of text and add a prefix in front of each line. 27 | */ 28 | function prefix(message: string, prefix: string): string { 29 | return ( 30 | prefix + 31 | message 32 | .split(EOL) 33 | .filter((m) => m !== "") 34 | .join(`${EOL}${prefix}`) 35 | ); 36 | } 37 | 38 | /* 39 | * format stdout and stderr in the following format: 40 | * 41 | * ``` 42 | * STDOUT: 43 | * | some output 44 | * STDERR: 45 | * | some stderr output 46 | * ``` 47 | */ 48 | export function formatOutput(result: TaskResult): string { 49 | let message = ""; 50 | if (result.stdout.length !== 0) { 51 | message += `STDOUT${EOL}`; 52 | message += prefix(result.stdout, " | "); 53 | message += `${EOL}`; 54 | } 55 | if (result.stderr.length !== 0) { 56 | message += `STDERR${EOL}`; 57 | message += prefix(result.stderr, " | "); 58 | message += `${EOL}`; 59 | } 60 | return message; 61 | } 62 | -------------------------------------------------------------------------------- /src/pipeline.ts: -------------------------------------------------------------------------------- 1 | import pGraph, { PGraphNodeMap } from "p-graph"; 2 | import { generateTaskGraph } from "./generateTaskGraph"; 3 | import { outputResult } from "./output"; 4 | import { Globals, Pipeline, Options } from "./publicInterfaces"; 5 | import { runAndLog } from "./runAndLog"; 6 | import { getPackageTaskFromId, getTaskId } from "./taskId"; 7 | import { 8 | Task, 9 | Tasks, 10 | TopologicalGraph, 11 | PackageTaskDeps, 12 | PackageTask, 13 | } from "./types"; 14 | 15 | const defaultGlobals: Globals = { 16 | logger: console, 17 | cwd() { 18 | return process.cwd(); 19 | }, 20 | exit(int: number) { 21 | process.exit(int); 22 | }, 23 | errorFormatter(err: Error): string { 24 | return err.stack || err.message || err.toString(); 25 | }, 26 | targetsOnly: false, 27 | }; 28 | 29 | async function execute( 30 | globals: Globals, 31 | graph: TopologicalGraph, 32 | task: Task, 33 | pkg: string, 34 | shouldBail: () => boolean 35 | ): Promise { 36 | if (shouldBail()) { 37 | return; 38 | } 39 | 40 | try { 41 | return await runAndLog(task, graph, pkg, globals); 42 | } catch (error) { 43 | throw { 44 | task: task.name, 45 | package: pkg, 46 | message: error, 47 | }; 48 | } 49 | } 50 | 51 | export function createPipelineInternal( 52 | graph: TopologicalGraph, 53 | globals: Globals, 54 | tasks: Tasks = new Map(), 55 | packageTaskDeps: PackageTaskDeps = [] 56 | ): Pipeline { 57 | const pipeline: Pipeline = { 58 | addTask(task) { 59 | tasks.set(task.name, task); 60 | return pipeline; 61 | }, 62 | addDep(from: PackageTask, to: PackageTask) { 63 | packageTaskDeps.push([ 64 | getTaskId(from.package, from.task), 65 | getTaskId(to.package, to.task), 66 | ]); 67 | return pipeline; 68 | }, 69 | async go(targets = {}) { 70 | if (typeof targets.packages === "undefined") { 71 | targets.packages = Object.keys(graph); 72 | } 73 | 74 | if (typeof targets.tasks === "undefined") { 75 | targets.tasks = [...tasks.keys()]; 76 | } 77 | 78 | const taskDeps = generateTaskGraph( 79 | targets.packages, 80 | targets.tasks, 81 | tasks, 82 | graph, 83 | packageTaskDeps, 84 | globals.targetsOnly 85 | ); 86 | const failures: { 87 | task: string; 88 | package: string; 89 | message: string; 90 | }[] = []; 91 | 92 | let bail = false; 93 | 94 | const packageTasks: PGraphNodeMap = new Map(); 95 | 96 | for (const [from, to] of taskDeps) { 97 | for (const taskId of [from, to]) { 98 | if (!packageTasks.has(taskId)) { 99 | const [pkg, taskName] = getPackageTaskFromId(taskId); 100 | 101 | if (taskName === "") { 102 | packageTasks.set(taskId, { run: () => Promise.resolve() }); 103 | } else { 104 | const task = tasks.get(taskName); 105 | if (!task) { 106 | throw new Error(`Missing pipeline config for "${taskName}"`); 107 | } 108 | packageTasks.set(taskId, { 109 | priority: task.priorities && task.priorities[pkg], 110 | run: () => 111 | execute(globals, graph, task, pkg, () => bail).catch( 112 | (error: { 113 | task: string; 114 | package: string; 115 | message: string; 116 | }) => { 117 | bail = true; 118 | failures.push(error); 119 | } 120 | ), 121 | }); 122 | } 123 | } 124 | } 125 | } 126 | 127 | await pGraph(packageTasks, taskDeps).run({ 128 | concurrency: globals.concurrency, 129 | continue: globals.continue, 130 | }); 131 | 132 | if (failures.length > 0) { 133 | failures.forEach((err) => 134 | outputResult( 135 | err.message, 136 | err.package, 137 | err.task, 138 | "failure", 139 | globals.logger 140 | ) 141 | ); 142 | 143 | globals.exit(1); 144 | } 145 | }, 146 | }; 147 | 148 | return pipeline; 149 | } 150 | 151 | export function createPipeline( 152 | graph: TopologicalGraph, 153 | options: Options = {} 154 | ): Pipeline { 155 | const fullOptions: Globals = { ...defaultGlobals, ...options }; 156 | return createPipelineInternal(graph, fullOptions, new Map(), []); 157 | } 158 | -------------------------------------------------------------------------------- /src/publicInterfaces.ts: -------------------------------------------------------------------------------- 1 | import { Task, Logger, PackageTask } from "./types"; 2 | 3 | export { Task, TopologicalGraph, Tasks, PackageTask } from "./types"; 4 | 5 | export type Pipeline = { 6 | addTask: (task: Task) => Pipeline; 7 | addDep: (from: PackageTask, to: PackageTask) => Pipeline; 8 | go: (targets?: { packages?: string[]; tasks?: string[] }) => Promise; 9 | }; 10 | 11 | export type Globals = { 12 | logger: Logger; 13 | cwd(): string; 14 | exit(int: number): void; 15 | errorFormatter(err: Error): string; 16 | targetsOnly: boolean; 17 | /** The maximum number of tasks that can be running simultaneously running for the given pipeline. By default, concurrency is not limited */ 18 | concurrency?: number; 19 | continue?: boolean; 20 | }; 21 | 22 | export type Options = Partial< 23 | Pick 24 | >; 25 | -------------------------------------------------------------------------------- /src/runAndLog.ts: -------------------------------------------------------------------------------- 1 | import * as path from "path"; 2 | import { EOL } from "os"; 3 | import { Writable } from "stream"; 4 | import * as streams from "memory-streams"; 5 | 6 | import { Task, TopologicalGraph } from "./types"; 7 | import { formatOutput, outputResult } from "./output"; 8 | 9 | import { Globals } from "./publicInterfaces"; 10 | 11 | type TaskResult = { 12 | success: boolean; 13 | stderr: string; 14 | stdout: string; 15 | }; 16 | 17 | async function runAndCollectLogs( 18 | runner: (stdout: Writable, stderr: Writable) => Promise, 19 | globals: Globals 20 | ): Promise { 21 | const stdout = new streams.WritableStream(); 22 | const stderr = new streams.WritableStream(); 23 | 24 | try { 25 | const success = await runner(stdout, stderr); 26 | 27 | return { success, stdout: stdout.toString(), stderr: stderr.toString() }; 28 | } catch (error) { 29 | if (error) { 30 | const exceptionMessage = globals.errorFormatter(error); 31 | stderr.write(EOL + exceptionMessage); 32 | } 33 | 34 | return { 35 | success: false, 36 | stdout: stdout.toString(), 37 | stderr: stderr.toString(), 38 | }; 39 | } 40 | } 41 | 42 | async function wait(): Promise { 43 | return new Promise((resolve) => setTimeout(resolve, 50)); 44 | } 45 | 46 | export async function runAndLog( 47 | task: Task, 48 | graph: TopologicalGraph, 49 | p: string, 50 | globals: Globals 51 | ): Promise { 52 | const result = await runAndCollectLogs( 53 | (stdout: Writable, stderr: Writable) => 54 | task.run(path.join(globals.cwd(), graph[p].location), stdout, stderr, p), 55 | globals 56 | ); 57 | await wait(); 58 | 59 | const message = formatOutput(result); 60 | if (result.success) { 61 | outputResult(message, p, task.name, "success", globals.logger); 62 | } else { 63 | throw message; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/taskId.ts: -------------------------------------------------------------------------------- 1 | import { TaskId } from "./types"; 2 | 3 | const DELIMITER = "#"; 4 | 5 | export function getTaskId(pkg: string, taskName: string): string { 6 | return `${pkg}${DELIMITER}${taskName}`; 7 | } 8 | 9 | export function getPackageTaskFromId(taskId: TaskId): string[] { 10 | return taskId.split(DELIMITER); 11 | } 12 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | import { Writable } from "stream"; 2 | 3 | export type Tasks = Map; 4 | 5 | export type TopologicalGraph = { 6 | [name: string]: { 7 | location: string; 8 | dependencies: string[]; 9 | }; 10 | }; 11 | 12 | export type Task = { 13 | /** name of the task */ 14 | name: string; 15 | 16 | /** a function that gets invoked by the task-scheduler */ 17 | run: ( 18 | cwd: string, 19 | stdout: Writable, 20 | stderr: Writable, 21 | packageName: string 22 | ) => Promise; 23 | 24 | /** dependencies between tasks within the same package (e.g. `build` -> `test`) */ 25 | deps?: string[]; 26 | 27 | /** dependencies across packages within the same topological graph (e.g. parent `build` -> child `build`) */ 28 | topoDeps?: string[]; 29 | 30 | /** An optional priority to set for packages that have this task. Unblocked tasks with a higher priority will be scheduled before lower priority tasks. */ 31 | priorities?: { 32 | [packageName: string]: number; 33 | }; 34 | }; 35 | 36 | export type PackageTask = { package: string; task: string }; 37 | export type PackageTaskDeps = [string, string][]; 38 | export type TaskId = string; 39 | 40 | export type Logger = { 41 | log(message: string): void; 42 | error(message: string): void; 43 | }; 44 | export type TaskResult = { 45 | success: boolean; 46 | stderr: string; 47 | stdout: string; 48 | }; 49 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": 3 | { 4 | "module": "commonjs", 5 | "strict": true, 6 | "declaration": true, 7 | "target": "es2018", 8 | "outDir": "./lib", 9 | "lib": ["es2018"] 10 | }, 11 | "files": ["./src/index.ts"] 12 | 13 | } 14 | --------------------------------------------------------------------------------