├── .github ├── dependabot.yml └── workflows │ ├── CI.yml │ └── release.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── screenshots ├── dark_background_progress_bar.png ├── light_background_progress_bar.png └── simple_progress_bar.png ├── scripts ├── benchmark.sh └── generate_manual.sh ├── src ├── command.rs ├── command │ ├── metrics.rs │ └── path_cache.rs ├── command_line_args.rs ├── common.rs ├── input.rs ├── input │ ├── buffered_reader.rs │ └── task.rs ├── main.rs ├── output.rs ├── output │ └── task.rs ├── parser.rs ├── parser │ ├── buffered.rs │ ├── command_line.rs │ └── regex.rs ├── process.rs ├── progress.rs └── progress │ └── style.rs └── tests ├── csv_file.txt ├── csv_file_badline.txt ├── dummy_shell.sh ├── file.txt └── integration_tests.rs /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "cargo" 9 | directory: "/" 10 | schedule: 11 | interval: "weekly" 12 | allow: 13 | - dependency-type: "all" 14 | 15 | - package-ecosystem: "github-actions" 16 | directory: "/" 17 | schedule: 18 | interval: "weekly" 19 | allow: 20 | - dependency-type: "all" 21 | -------------------------------------------------------------------------------- /.github/workflows/CI.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Build 20 | run: cargo build --verbose 21 | - name: Run tests 22 | run: cargo test --verbose 23 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | permissions: 4 | contents: write 5 | 6 | on: 7 | push: 8 | tags: 9 | - v[0-9]+.* 10 | 11 | jobs: 12 | create-release: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | - uses: taiki-e/create-gh-release-action@v1 17 | with: 18 | # (optional) Path to changelog. 19 | #changelog: CHANGELOG.md 20 | # (required) GitHub token for creating GitHub Releases. 21 | token: ${{ secrets.GITHUB_TOKEN }} 22 | 23 | upload-assets: 24 | strategy: 25 | matrix: 26 | include: 27 | - target: aarch64-unknown-linux-gnu 28 | os: ubuntu-latest 29 | - target: aarch64-apple-darwin 30 | os: macos-latest 31 | - target: x86_64-unknown-linux-gnu 32 | os: ubuntu-latest 33 | - target: x86_64-apple-darwin 34 | os: macos-latest 35 | # Universal macOS binary is supported as universal-apple-darwin. 36 | - target: universal-apple-darwin 37 | os: macos-latest 38 | - target: x86_64-pc-windows-msvc 39 | os: windows-latest 40 | runs-on: ${{ matrix.os }} 41 | steps: 42 | - uses: actions/checkout@v4 43 | - uses: taiki-e/upload-rust-binary-action@v1 44 | with: 45 | # (required) Comma-separated list of binary names (non-extension portion of filename) to build and upload. 46 | # Note that glob pattern is not supported yet. 47 | bin: rust-parallel 48 | tar: unix 49 | zip: windows 50 | # (optional) Target triple, default is host triple. 51 | target: ${{ matrix.target }} 52 | # (required) GitHub token for uploading assets to GitHub Releases. 53 | token: ${{ secrets.GITHUB_TOKEN }} 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 4 4 | 5 | [[package]] 6 | name = "addr2line" 7 | version = "0.24.2" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" 10 | dependencies = [ 11 | "gimli", 12 | ] 13 | 14 | [[package]] 15 | name = "adler2" 16 | version = "2.0.0" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" 19 | 20 | [[package]] 21 | name = "aho-corasick" 22 | version = "1.1.3" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" 25 | dependencies = [ 26 | "memchr", 27 | ] 28 | 29 | [[package]] 30 | name = "anstream" 31 | version = "0.6.18" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" 34 | dependencies = [ 35 | "anstyle", 36 | "anstyle-parse", 37 | "anstyle-query", 38 | "anstyle-wincon", 39 | "colorchoice", 40 | "is_terminal_polyfill", 41 | "utf8parse", 42 | ] 43 | 44 | [[package]] 45 | name = "anstyle" 46 | version = "1.0.10" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" 49 | 50 | [[package]] 51 | name = "anstyle-parse" 52 | version = "0.2.6" 53 | source = "registry+https://github.com/rust-lang/crates.io-index" 54 | checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" 55 | dependencies = [ 56 | "utf8parse", 57 | ] 58 | 59 | [[package]] 60 | name = "anstyle-query" 61 | version = "1.1.2" 62 | source = "registry+https://github.com/rust-lang/crates.io-index" 63 | checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" 64 | dependencies = [ 65 | "windows-sys 0.59.0", 66 | ] 67 | 68 | [[package]] 69 | name = "anstyle-wincon" 70 | version = "3.0.8" 71 | source = "registry+https://github.com/rust-lang/crates.io-index" 72 | checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" 73 | dependencies = [ 74 | "anstyle", 75 | "once_cell_polyfill", 76 | "windows-sys 0.59.0", 77 | ] 78 | 79 | [[package]] 80 | name = "anyhow" 81 | version = "1.0.98" 82 | source = "registry+https://github.com/rust-lang/crates.io-index" 83 | checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" 84 | 85 | [[package]] 86 | name = "assert_cmd" 87 | version = "2.0.17" 88 | source = "registry+https://github.com/rust-lang/crates.io-index" 89 | checksum = "2bd389a4b2970a01282ee455294913c0a43724daedcd1a24c3eb0ec1c1320b66" 90 | dependencies = [ 91 | "anstyle", 92 | "bstr", 93 | "doc-comment", 94 | "libc", 95 | "predicates", 96 | "predicates-core", 97 | "predicates-tree", 98 | "wait-timeout", 99 | ] 100 | 101 | [[package]] 102 | name = "autocfg" 103 | version = "1.4.0" 104 | source = "registry+https://github.com/rust-lang/crates.io-index" 105 | checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" 106 | 107 | [[package]] 108 | name = "backtrace" 109 | version = "0.3.75" 110 | source = "registry+https://github.com/rust-lang/crates.io-index" 111 | checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" 112 | dependencies = [ 113 | "addr2line", 114 | "cfg-if", 115 | "libc", 116 | "miniz_oxide", 117 | "object", 118 | "rustc-demangle", 119 | "windows-targets", 120 | ] 121 | 122 | [[package]] 123 | name = "bitflags" 124 | version = "2.9.1" 125 | source = "registry+https://github.com/rust-lang/crates.io-index" 126 | checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" 127 | 128 | [[package]] 129 | name = "bstr" 130 | version = "1.12.0" 131 | source = "registry+https://github.com/rust-lang/crates.io-index" 132 | checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" 133 | dependencies = [ 134 | "memchr", 135 | "regex-automata", 136 | "serde", 137 | ] 138 | 139 | [[package]] 140 | name = "bumpalo" 141 | version = "3.17.0" 142 | source = "registry+https://github.com/rust-lang/crates.io-index" 143 | checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" 144 | 145 | [[package]] 146 | name = "bytes" 147 | version = "1.10.1" 148 | source = "registry+https://github.com/rust-lang/crates.io-index" 149 | checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" 150 | 151 | [[package]] 152 | name = "cfg-if" 153 | version = "1.0.0" 154 | source = "registry+https://github.com/rust-lang/crates.io-index" 155 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 156 | 157 | [[package]] 158 | name = "clap" 159 | version = "4.5.39" 160 | source = "registry+https://github.com/rust-lang/crates.io-index" 161 | checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" 162 | dependencies = [ 163 | "clap_builder", 164 | "clap_derive", 165 | ] 166 | 167 | [[package]] 168 | name = "clap_builder" 169 | version = "4.5.39" 170 | source = "registry+https://github.com/rust-lang/crates.io-index" 171 | checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" 172 | dependencies = [ 173 | "anstream", 174 | "anstyle", 175 | "clap_lex", 176 | "strsim", 177 | ] 178 | 179 | [[package]] 180 | name = "clap_derive" 181 | version = "4.5.32" 182 | source = "registry+https://github.com/rust-lang/crates.io-index" 183 | checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" 184 | dependencies = [ 185 | "heck", 186 | "proc-macro2", 187 | "quote", 188 | "syn", 189 | ] 190 | 191 | [[package]] 192 | name = "clap_lex" 193 | version = "0.7.4" 194 | source = "registry+https://github.com/rust-lang/crates.io-index" 195 | checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" 196 | 197 | [[package]] 198 | name = "colorchoice" 199 | version = "1.0.3" 200 | source = "registry+https://github.com/rust-lang/crates.io-index" 201 | checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" 202 | 203 | [[package]] 204 | name = "console" 205 | version = "0.15.11" 206 | source = "registry+https://github.com/rust-lang/crates.io-index" 207 | checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" 208 | dependencies = [ 209 | "encode_unicode", 210 | "libc", 211 | "once_cell", 212 | "unicode-width", 213 | "windows-sys 0.59.0", 214 | ] 215 | 216 | [[package]] 217 | name = "difflib" 218 | version = "0.4.0" 219 | source = "registry+https://github.com/rust-lang/crates.io-index" 220 | checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" 221 | 222 | [[package]] 223 | name = "doc-comment" 224 | version = "0.3.3" 225 | source = "registry+https://github.com/rust-lang/crates.io-index" 226 | checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" 227 | 228 | [[package]] 229 | name = "either" 230 | version = "1.15.0" 231 | source = "registry+https://github.com/rust-lang/crates.io-index" 232 | checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" 233 | 234 | [[package]] 235 | name = "encode_unicode" 236 | version = "1.0.0" 237 | source = "registry+https://github.com/rust-lang/crates.io-index" 238 | checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" 239 | 240 | [[package]] 241 | name = "env_home" 242 | version = "0.1.0" 243 | source = "registry+https://github.com/rust-lang/crates.io-index" 244 | checksum = "c7f84e12ccf0a7ddc17a6c41c93326024c42920d7ee630d04950e6926645c0fe" 245 | 246 | [[package]] 247 | name = "errno" 248 | version = "0.3.12" 249 | source = "registry+https://github.com/rust-lang/crates.io-index" 250 | checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" 251 | dependencies = [ 252 | "libc", 253 | "windows-sys 0.59.0", 254 | ] 255 | 256 | [[package]] 257 | name = "float-cmp" 258 | version = "0.10.0" 259 | source = "registry+https://github.com/rust-lang/crates.io-index" 260 | checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" 261 | dependencies = [ 262 | "num-traits", 263 | ] 264 | 265 | [[package]] 266 | name = "gimli" 267 | version = "0.31.1" 268 | source = "registry+https://github.com/rust-lang/crates.io-index" 269 | checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" 270 | 271 | [[package]] 272 | name = "heck" 273 | version = "0.5.0" 274 | source = "registry+https://github.com/rust-lang/crates.io-index" 275 | checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" 276 | 277 | [[package]] 278 | name = "hermit-abi" 279 | version = "0.5.1" 280 | source = "registry+https://github.com/rust-lang/crates.io-index" 281 | checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" 282 | 283 | [[package]] 284 | name = "indicatif" 285 | version = "0.17.11" 286 | source = "registry+https://github.com/rust-lang/crates.io-index" 287 | checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" 288 | dependencies = [ 289 | "console", 290 | "number_prefix", 291 | "portable-atomic", 292 | "unicode-width", 293 | "web-time", 294 | ] 295 | 296 | [[package]] 297 | name = "is_terminal_polyfill" 298 | version = "1.70.1" 299 | source = "registry+https://github.com/rust-lang/crates.io-index" 300 | checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" 301 | 302 | [[package]] 303 | name = "itertools" 304 | version = "0.14.0" 305 | source = "registry+https://github.com/rust-lang/crates.io-index" 306 | checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" 307 | dependencies = [ 308 | "either", 309 | ] 310 | 311 | [[package]] 312 | name = "js-sys" 313 | version = "0.3.77" 314 | source = "registry+https://github.com/rust-lang/crates.io-index" 315 | checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" 316 | dependencies = [ 317 | "once_cell", 318 | "wasm-bindgen", 319 | ] 320 | 321 | [[package]] 322 | name = "lazy_static" 323 | version = "1.5.0" 324 | source = "registry+https://github.com/rust-lang/crates.io-index" 325 | checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" 326 | 327 | [[package]] 328 | name = "libc" 329 | version = "0.2.172" 330 | source = "registry+https://github.com/rust-lang/crates.io-index" 331 | checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" 332 | 333 | [[package]] 334 | name = "linux-raw-sys" 335 | version = "0.9.4" 336 | source = "registry+https://github.com/rust-lang/crates.io-index" 337 | checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" 338 | 339 | [[package]] 340 | name = "lock_api" 341 | version = "0.4.13" 342 | source = "registry+https://github.com/rust-lang/crates.io-index" 343 | checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" 344 | dependencies = [ 345 | "autocfg", 346 | "scopeguard", 347 | ] 348 | 349 | [[package]] 350 | name = "log" 351 | version = "0.4.27" 352 | source = "registry+https://github.com/rust-lang/crates.io-index" 353 | checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" 354 | 355 | [[package]] 356 | name = "memchr" 357 | version = "2.7.4" 358 | source = "registry+https://github.com/rust-lang/crates.io-index" 359 | checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" 360 | 361 | [[package]] 362 | name = "miniz_oxide" 363 | version = "0.8.8" 364 | source = "registry+https://github.com/rust-lang/crates.io-index" 365 | checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" 366 | dependencies = [ 367 | "adler2", 368 | ] 369 | 370 | [[package]] 371 | name = "mio" 372 | version = "1.0.4" 373 | source = "registry+https://github.com/rust-lang/crates.io-index" 374 | checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" 375 | dependencies = [ 376 | "libc", 377 | "wasi", 378 | "windows-sys 0.59.0", 379 | ] 380 | 381 | [[package]] 382 | name = "normalize-line-endings" 383 | version = "0.3.0" 384 | source = "registry+https://github.com/rust-lang/crates.io-index" 385 | checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" 386 | 387 | [[package]] 388 | name = "nu-ansi-term" 389 | version = "0.46.0" 390 | source = "registry+https://github.com/rust-lang/crates.io-index" 391 | checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" 392 | dependencies = [ 393 | "overload", 394 | "winapi", 395 | ] 396 | 397 | [[package]] 398 | name = "num-traits" 399 | version = "0.2.19" 400 | source = "registry+https://github.com/rust-lang/crates.io-index" 401 | checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" 402 | dependencies = [ 403 | "autocfg", 404 | ] 405 | 406 | [[package]] 407 | name = "num_cpus" 408 | version = "1.17.0" 409 | source = "registry+https://github.com/rust-lang/crates.io-index" 410 | checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" 411 | dependencies = [ 412 | "hermit-abi", 413 | "libc", 414 | ] 415 | 416 | [[package]] 417 | name = "number_prefix" 418 | version = "0.4.0" 419 | source = "registry+https://github.com/rust-lang/crates.io-index" 420 | checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" 421 | 422 | [[package]] 423 | name = "object" 424 | version = "0.36.7" 425 | source = "registry+https://github.com/rust-lang/crates.io-index" 426 | checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" 427 | dependencies = [ 428 | "memchr", 429 | ] 430 | 431 | [[package]] 432 | name = "once_cell" 433 | version = "1.21.3" 434 | source = "registry+https://github.com/rust-lang/crates.io-index" 435 | checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" 436 | 437 | [[package]] 438 | name = "once_cell_polyfill" 439 | version = "1.70.1" 440 | source = "registry+https://github.com/rust-lang/crates.io-index" 441 | checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" 442 | 443 | [[package]] 444 | name = "overload" 445 | version = "0.1.1" 446 | source = "registry+https://github.com/rust-lang/crates.io-index" 447 | checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" 448 | 449 | [[package]] 450 | name = "parking_lot" 451 | version = "0.12.4" 452 | source = "registry+https://github.com/rust-lang/crates.io-index" 453 | checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" 454 | dependencies = [ 455 | "lock_api", 456 | "parking_lot_core", 457 | ] 458 | 459 | [[package]] 460 | name = "parking_lot_core" 461 | version = "0.9.11" 462 | source = "registry+https://github.com/rust-lang/crates.io-index" 463 | checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" 464 | dependencies = [ 465 | "cfg-if", 466 | "libc", 467 | "redox_syscall", 468 | "smallvec", 469 | "windows-targets", 470 | ] 471 | 472 | [[package]] 473 | name = "pin-project-lite" 474 | version = "0.2.16" 475 | source = "registry+https://github.com/rust-lang/crates.io-index" 476 | checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" 477 | 478 | [[package]] 479 | name = "portable-atomic" 480 | version = "1.11.0" 481 | source = "registry+https://github.com/rust-lang/crates.io-index" 482 | checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" 483 | 484 | [[package]] 485 | name = "predicates" 486 | version = "3.1.3" 487 | source = "registry+https://github.com/rust-lang/crates.io-index" 488 | checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" 489 | dependencies = [ 490 | "anstyle", 491 | "difflib", 492 | "float-cmp", 493 | "normalize-line-endings", 494 | "predicates-core", 495 | "regex", 496 | ] 497 | 498 | [[package]] 499 | name = "predicates-core" 500 | version = "1.0.9" 501 | source = "registry+https://github.com/rust-lang/crates.io-index" 502 | checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" 503 | 504 | [[package]] 505 | name = "predicates-tree" 506 | version = "1.0.12" 507 | source = "registry+https://github.com/rust-lang/crates.io-index" 508 | checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" 509 | dependencies = [ 510 | "predicates-core", 511 | "termtree", 512 | ] 513 | 514 | [[package]] 515 | name = "proc-macro2" 516 | version = "1.0.95" 517 | source = "registry+https://github.com/rust-lang/crates.io-index" 518 | checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" 519 | dependencies = [ 520 | "unicode-ident", 521 | ] 522 | 523 | [[package]] 524 | name = "quote" 525 | version = "1.0.40" 526 | source = "registry+https://github.com/rust-lang/crates.io-index" 527 | checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" 528 | dependencies = [ 529 | "proc-macro2", 530 | ] 531 | 532 | [[package]] 533 | name = "redox_syscall" 534 | version = "0.5.12" 535 | source = "registry+https://github.com/rust-lang/crates.io-index" 536 | checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" 537 | dependencies = [ 538 | "bitflags", 539 | ] 540 | 541 | [[package]] 542 | name = "regex" 543 | version = "1.11.1" 544 | source = "registry+https://github.com/rust-lang/crates.io-index" 545 | checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" 546 | dependencies = [ 547 | "aho-corasick", 548 | "memchr", 549 | "regex-automata", 550 | "regex-syntax", 551 | ] 552 | 553 | [[package]] 554 | name = "regex-automata" 555 | version = "0.4.9" 556 | source = "registry+https://github.com/rust-lang/crates.io-index" 557 | checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" 558 | dependencies = [ 559 | "aho-corasick", 560 | "memchr", 561 | "regex-syntax", 562 | ] 563 | 564 | [[package]] 565 | name = "regex-syntax" 566 | version = "0.8.5" 567 | source = "registry+https://github.com/rust-lang/crates.io-index" 568 | checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" 569 | 570 | [[package]] 571 | name = "rust-parallel" 572 | version = "1.18.1" 573 | dependencies = [ 574 | "anyhow", 575 | "assert_cmd", 576 | "clap", 577 | "indicatif", 578 | "itertools", 579 | "num_cpus", 580 | "predicates", 581 | "regex", 582 | "thiserror", 583 | "tokio", 584 | "tracing", 585 | "tracing-subscriber", 586 | "which", 587 | ] 588 | 589 | [[package]] 590 | name = "rustc-demangle" 591 | version = "0.1.24" 592 | source = "registry+https://github.com/rust-lang/crates.io-index" 593 | checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" 594 | 595 | [[package]] 596 | name = "rustix" 597 | version = "1.0.7" 598 | source = "registry+https://github.com/rust-lang/crates.io-index" 599 | checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" 600 | dependencies = [ 601 | "bitflags", 602 | "errno", 603 | "libc", 604 | "linux-raw-sys", 605 | "windows-sys 0.59.0", 606 | ] 607 | 608 | [[package]] 609 | name = "scopeguard" 610 | version = "1.2.0" 611 | source = "registry+https://github.com/rust-lang/crates.io-index" 612 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 613 | 614 | [[package]] 615 | name = "serde" 616 | version = "1.0.219" 617 | source = "registry+https://github.com/rust-lang/crates.io-index" 618 | checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" 619 | dependencies = [ 620 | "serde_derive", 621 | ] 622 | 623 | [[package]] 624 | name = "serde_derive" 625 | version = "1.0.219" 626 | source = "registry+https://github.com/rust-lang/crates.io-index" 627 | checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" 628 | dependencies = [ 629 | "proc-macro2", 630 | "quote", 631 | "syn", 632 | ] 633 | 634 | [[package]] 635 | name = "sharded-slab" 636 | version = "0.1.7" 637 | source = "registry+https://github.com/rust-lang/crates.io-index" 638 | checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" 639 | dependencies = [ 640 | "lazy_static", 641 | ] 642 | 643 | [[package]] 644 | name = "signal-hook-registry" 645 | version = "1.4.5" 646 | source = "registry+https://github.com/rust-lang/crates.io-index" 647 | checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" 648 | dependencies = [ 649 | "libc", 650 | ] 651 | 652 | [[package]] 653 | name = "smallvec" 654 | version = "1.15.0" 655 | source = "registry+https://github.com/rust-lang/crates.io-index" 656 | checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" 657 | 658 | [[package]] 659 | name = "socket2" 660 | version = "0.5.10" 661 | source = "registry+https://github.com/rust-lang/crates.io-index" 662 | checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" 663 | dependencies = [ 664 | "libc", 665 | "windows-sys 0.52.0", 666 | ] 667 | 668 | [[package]] 669 | name = "strsim" 670 | version = "0.11.1" 671 | source = "registry+https://github.com/rust-lang/crates.io-index" 672 | checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" 673 | 674 | [[package]] 675 | name = "syn" 676 | version = "2.0.101" 677 | source = "registry+https://github.com/rust-lang/crates.io-index" 678 | checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" 679 | dependencies = [ 680 | "proc-macro2", 681 | "quote", 682 | "unicode-ident", 683 | ] 684 | 685 | [[package]] 686 | name = "termtree" 687 | version = "0.5.1" 688 | source = "registry+https://github.com/rust-lang/crates.io-index" 689 | checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" 690 | 691 | [[package]] 692 | name = "thiserror" 693 | version = "2.0.12" 694 | source = "registry+https://github.com/rust-lang/crates.io-index" 695 | checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" 696 | dependencies = [ 697 | "thiserror-impl", 698 | ] 699 | 700 | [[package]] 701 | name = "thiserror-impl" 702 | version = "2.0.12" 703 | source = "registry+https://github.com/rust-lang/crates.io-index" 704 | checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" 705 | dependencies = [ 706 | "proc-macro2", 707 | "quote", 708 | "syn", 709 | ] 710 | 711 | [[package]] 712 | name = "thread_local" 713 | version = "1.1.8" 714 | source = "registry+https://github.com/rust-lang/crates.io-index" 715 | checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" 716 | dependencies = [ 717 | "cfg-if", 718 | "once_cell", 719 | ] 720 | 721 | [[package]] 722 | name = "tokio" 723 | version = "1.45.1" 724 | source = "registry+https://github.com/rust-lang/crates.io-index" 725 | checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" 726 | dependencies = [ 727 | "backtrace", 728 | "bytes", 729 | "libc", 730 | "mio", 731 | "parking_lot", 732 | "pin-project-lite", 733 | "signal-hook-registry", 734 | "socket2", 735 | "tokio-macros", 736 | "windows-sys 0.52.0", 737 | ] 738 | 739 | [[package]] 740 | name = "tokio-macros" 741 | version = "2.5.0" 742 | source = "registry+https://github.com/rust-lang/crates.io-index" 743 | checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" 744 | dependencies = [ 745 | "proc-macro2", 746 | "quote", 747 | "syn", 748 | ] 749 | 750 | [[package]] 751 | name = "tracing" 752 | version = "0.1.41" 753 | source = "registry+https://github.com/rust-lang/crates.io-index" 754 | checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" 755 | dependencies = [ 756 | "pin-project-lite", 757 | "tracing-attributes", 758 | "tracing-core", 759 | ] 760 | 761 | [[package]] 762 | name = "tracing-attributes" 763 | version = "0.1.28" 764 | source = "registry+https://github.com/rust-lang/crates.io-index" 765 | checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" 766 | dependencies = [ 767 | "proc-macro2", 768 | "quote", 769 | "syn", 770 | ] 771 | 772 | [[package]] 773 | name = "tracing-core" 774 | version = "0.1.33" 775 | source = "registry+https://github.com/rust-lang/crates.io-index" 776 | checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" 777 | dependencies = [ 778 | "once_cell", 779 | "valuable", 780 | ] 781 | 782 | [[package]] 783 | name = "tracing-log" 784 | version = "0.2.0" 785 | source = "registry+https://github.com/rust-lang/crates.io-index" 786 | checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" 787 | dependencies = [ 788 | "log", 789 | "once_cell", 790 | "tracing-core", 791 | ] 792 | 793 | [[package]] 794 | name = "tracing-subscriber" 795 | version = "0.3.19" 796 | source = "registry+https://github.com/rust-lang/crates.io-index" 797 | checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" 798 | dependencies = [ 799 | "nu-ansi-term", 800 | "sharded-slab", 801 | "smallvec", 802 | "thread_local", 803 | "tracing-core", 804 | "tracing-log", 805 | ] 806 | 807 | [[package]] 808 | name = "unicode-ident" 809 | version = "1.0.18" 810 | source = "registry+https://github.com/rust-lang/crates.io-index" 811 | checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" 812 | 813 | [[package]] 814 | name = "unicode-width" 815 | version = "0.2.0" 816 | source = "registry+https://github.com/rust-lang/crates.io-index" 817 | checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" 818 | 819 | [[package]] 820 | name = "utf8parse" 821 | version = "0.2.2" 822 | source = "registry+https://github.com/rust-lang/crates.io-index" 823 | checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" 824 | 825 | [[package]] 826 | name = "valuable" 827 | version = "0.1.1" 828 | source = "registry+https://github.com/rust-lang/crates.io-index" 829 | checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" 830 | 831 | [[package]] 832 | name = "wait-timeout" 833 | version = "0.2.1" 834 | source = "registry+https://github.com/rust-lang/crates.io-index" 835 | checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" 836 | dependencies = [ 837 | "libc", 838 | ] 839 | 840 | [[package]] 841 | name = "wasi" 842 | version = "0.11.0+wasi-snapshot-preview1" 843 | source = "registry+https://github.com/rust-lang/crates.io-index" 844 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 845 | 846 | [[package]] 847 | name = "wasm-bindgen" 848 | version = "0.2.100" 849 | source = "registry+https://github.com/rust-lang/crates.io-index" 850 | checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" 851 | dependencies = [ 852 | "cfg-if", 853 | "once_cell", 854 | "wasm-bindgen-macro", 855 | ] 856 | 857 | [[package]] 858 | name = "wasm-bindgen-backend" 859 | version = "0.2.100" 860 | source = "registry+https://github.com/rust-lang/crates.io-index" 861 | checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" 862 | dependencies = [ 863 | "bumpalo", 864 | "log", 865 | "proc-macro2", 866 | "quote", 867 | "syn", 868 | "wasm-bindgen-shared", 869 | ] 870 | 871 | [[package]] 872 | name = "wasm-bindgen-macro" 873 | version = "0.2.100" 874 | source = "registry+https://github.com/rust-lang/crates.io-index" 875 | checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" 876 | dependencies = [ 877 | "quote", 878 | "wasm-bindgen-macro-support", 879 | ] 880 | 881 | [[package]] 882 | name = "wasm-bindgen-macro-support" 883 | version = "0.2.100" 884 | source = "registry+https://github.com/rust-lang/crates.io-index" 885 | checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" 886 | dependencies = [ 887 | "proc-macro2", 888 | "quote", 889 | "syn", 890 | "wasm-bindgen-backend", 891 | "wasm-bindgen-shared", 892 | ] 893 | 894 | [[package]] 895 | name = "wasm-bindgen-shared" 896 | version = "0.2.100" 897 | source = "registry+https://github.com/rust-lang/crates.io-index" 898 | checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" 899 | dependencies = [ 900 | "unicode-ident", 901 | ] 902 | 903 | [[package]] 904 | name = "web-time" 905 | version = "1.1.0" 906 | source = "registry+https://github.com/rust-lang/crates.io-index" 907 | checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" 908 | dependencies = [ 909 | "js-sys", 910 | "wasm-bindgen", 911 | ] 912 | 913 | [[package]] 914 | name = "which" 915 | version = "7.0.3" 916 | source = "registry+https://github.com/rust-lang/crates.io-index" 917 | checksum = "24d643ce3fd3e5b54854602a080f34fb10ab75e0b813ee32d00ca2b44fa74762" 918 | dependencies = [ 919 | "either", 920 | "env_home", 921 | "rustix", 922 | "winsafe", 923 | ] 924 | 925 | [[package]] 926 | name = "winapi" 927 | version = "0.3.9" 928 | source = "registry+https://github.com/rust-lang/crates.io-index" 929 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 930 | dependencies = [ 931 | "winapi-i686-pc-windows-gnu", 932 | "winapi-x86_64-pc-windows-gnu", 933 | ] 934 | 935 | [[package]] 936 | name = "winapi-i686-pc-windows-gnu" 937 | version = "0.4.0" 938 | source = "registry+https://github.com/rust-lang/crates.io-index" 939 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 940 | 941 | [[package]] 942 | name = "winapi-x86_64-pc-windows-gnu" 943 | version = "0.4.0" 944 | source = "registry+https://github.com/rust-lang/crates.io-index" 945 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 946 | 947 | [[package]] 948 | name = "windows-sys" 949 | version = "0.52.0" 950 | source = "registry+https://github.com/rust-lang/crates.io-index" 951 | checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" 952 | dependencies = [ 953 | "windows-targets", 954 | ] 955 | 956 | [[package]] 957 | name = "windows-sys" 958 | version = "0.59.0" 959 | source = "registry+https://github.com/rust-lang/crates.io-index" 960 | checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" 961 | dependencies = [ 962 | "windows-targets", 963 | ] 964 | 965 | [[package]] 966 | name = "windows-targets" 967 | version = "0.52.6" 968 | source = "registry+https://github.com/rust-lang/crates.io-index" 969 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 970 | dependencies = [ 971 | "windows_aarch64_gnullvm", 972 | "windows_aarch64_msvc", 973 | "windows_i686_gnu", 974 | "windows_i686_gnullvm", 975 | "windows_i686_msvc", 976 | "windows_x86_64_gnu", 977 | "windows_x86_64_gnullvm", 978 | "windows_x86_64_msvc", 979 | ] 980 | 981 | [[package]] 982 | name = "windows_aarch64_gnullvm" 983 | version = "0.52.6" 984 | source = "registry+https://github.com/rust-lang/crates.io-index" 985 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 986 | 987 | [[package]] 988 | name = "windows_aarch64_msvc" 989 | version = "0.52.6" 990 | source = "registry+https://github.com/rust-lang/crates.io-index" 991 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 992 | 993 | [[package]] 994 | name = "windows_i686_gnu" 995 | version = "0.52.6" 996 | source = "registry+https://github.com/rust-lang/crates.io-index" 997 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 998 | 999 | [[package]] 1000 | name = "windows_i686_gnullvm" 1001 | version = "0.52.6" 1002 | source = "registry+https://github.com/rust-lang/crates.io-index" 1003 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 1004 | 1005 | [[package]] 1006 | name = "windows_i686_msvc" 1007 | version = "0.52.6" 1008 | source = "registry+https://github.com/rust-lang/crates.io-index" 1009 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 1010 | 1011 | [[package]] 1012 | name = "windows_x86_64_gnu" 1013 | version = "0.52.6" 1014 | source = "registry+https://github.com/rust-lang/crates.io-index" 1015 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 1016 | 1017 | [[package]] 1018 | name = "windows_x86_64_gnullvm" 1019 | version = "0.52.6" 1020 | source = "registry+https://github.com/rust-lang/crates.io-index" 1021 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 1022 | 1023 | [[package]] 1024 | name = "windows_x86_64_msvc" 1025 | version = "0.52.6" 1026 | source = "registry+https://github.com/rust-lang/crates.io-index" 1027 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 1028 | 1029 | [[package]] 1030 | name = "winsafe" 1031 | version = "0.0.19" 1032 | source = "registry+https://github.com/rust-lang/crates.io-index" 1033 | checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" 1034 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rust-parallel" 3 | version = "1.18.1" 4 | authors = ["Aaron Riekenberg "] 5 | edition = "2024" 6 | categories = ["asynchronous", "command-line-interface", "concurrency"] 7 | description = "Fast command line app in rust/tokio to run commands in parallel. Similar interface to GNU parallel or xargs." 8 | keywords = ["cli", "parallel", "tokio"] 9 | license = "MIT" 10 | readme = "README.md" 11 | repository = "https://github.com/aaronriekenberg/rust-parallel" 12 | 13 | 14 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 15 | 16 | [dependencies] 17 | anyhow = "1" 18 | clap = { version = "4", features = ["derive"] } 19 | indicatif = "0.17" 20 | itertools = "0.14" 21 | num_cpus = "1" 22 | regex = "1" 23 | thiserror = "2" 24 | tokio = { version = "1", features = ["full"] } 25 | tracing = "0.1" 26 | tracing-subscriber = "0.3" 27 | which = "7" 28 | 29 | [dev-dependencies] 30 | assert_cmd = "2" 31 | predicates = "3" 32 | 33 | [lints.rust] 34 | unsafe_code = "forbid" 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Aaron Riekenberg 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rust-parallel 2 | 3 | [crates-badge]: https://img.shields.io/crates/v/rust-parallel.svg 4 | [crates-url]: https://crates.io/crates/rust-parallel 5 | [homebrew-badge]: https://img.shields.io/homebrew/v/rust-parallel.svg 6 | [homebrew-url]: https://formulae.brew.sh/formula/rust-parallel 7 | 8 | [ci-badge]: https://github.com/aaronriekenberg/rust-parallel/actions/workflows/CI.yml/badge.svg 9 | [ci-url]: https://github.com/aaronriekenberg/rust-parallel/actions/workflows/CI.yml 10 | 11 | [![Crates.io][crates-badge]][crates-url] [![Homebrew][homebrew-badge]][homebrew-url] [![CI workflow][ci-badge]][ci-url] 12 | 13 | Run commands in parallel and aggregate outputs. Async application using [tokio](https://tokio.rs). 14 | 15 | [Example commands](https://github.com/aaronriekenberg/rust-parallel/wiki/Examples) and [detailed manual](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual). 16 | 17 | Listed in [Awesome Rust - utilities](https://github.com/rust-unofficial/awesome-rust#utilities) 18 | 19 | Similar interface to [GNU Parallel](https://www.gnu.org/software/parallel/parallel_examples.html) or [xargs](https://man7.org/linux/man-pages/man1/xargs.1.html) plus useful features: 20 | * More than 10x faster than GNU Parallel [in benchmarks](https://github.com/aaronriekenberg/rust-parallel/wiki/Benchmarks) 21 | * Run commands from [stdin](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#commands-from-stdin), [input files](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#reading-multiple-inputs), or [`:::` arguments](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#commands-from-arguments) 22 | * Automatic parallelism to all cpus, or [configure manually](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#parallelism) 23 | * Transform inputs with [variables](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#automatic-variables) or [regular expressions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#regular-expression) 24 | * Prevent [output interleaving](https://github.com/aaronriekenberg/rust-parallel/wiki/Output-Interleaving) 25 | * Shell mode to run [bash functions](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#bash-function) and [commands](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#shell-commands) 26 | * [TUI progress bar](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#progress-bar) using [indicatif](https://github.com/console-rs/indicatif) 27 | * [Path cache](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#path-cache) 28 | * [Command timeouts](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#timeout) 29 | * [Structured debug logging](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#debug-logging) 30 | * [Dry run mode](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#dry-run) 31 | * [Configurable error handling](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual#error-handling) 32 | 33 | ## Contents: 34 | * [Installation](#installation) 35 | * [Documents](#documents) 36 | * [Tech Stack](#tech-stack) 37 | 38 | ## Installation 39 | 40 | ### Homebrew 41 | 42 | With [Homebrew](https://brew.sh) installed, run 43 | 44 | ```sh 45 | brew install rust-parallel 46 | ``` 47 | 48 | ### Pre-built release 49 | 50 | 1. Download a pre-built release from [Github Releases](https://github.com/aaronriekenberg/rust-parallel/releases) for Linux, MacOS, or Windows. 51 | 2. Extract the executable and put it somewhere in your $PATH. 52 | 53 | ### From Crates.io via Cargo 54 | 55 | 1. [Install Rust](https://www.rust-lang.org/learn/get-started) 56 | 2. Install the latest version of this app from [crates.io](https://crates.io/crates/rust-parallel): 57 | 58 | ``` 59 | $ cargo install rust-parallel 60 | ``` 61 | 62 | The same `cargo install rust-parallel` command will also update to the latest version after initial installation. 63 | 64 | ## Documents: 65 | 1. [Examples](https://github.com/aaronriekenberg/rust-parallel/wiki/Examples) - complete runnable commands to give an idea of overall features. 66 | 1. [Manual](https://github.com/aaronriekenberg/rust-parallel/wiki/Manual) - more detailed manual on how to use individual features. 67 | 1. [Benchmarks](https://github.com/aaronriekenberg/rust-parallel/wiki/Benchmarks) 68 | 1. [Output Interleaving](https://github.com/aaronriekenberg/rust-parallel/wiki/Output-Interleaving) - output interleaving in rust-parallel compared with other commands. 69 | 70 | ## Tech Stack: 71 | * [anyhow](https://github.com/dtolnay/anyhow) used for application error handling to propogate and format fatal errors. 72 | * [clap](https://docs.rs/clap/latest/clap/) command line argument parser. 73 | * [itertools](https://docs.rs/itertools/latest/itertools/) using [`multi_cartesian_product`](https://docs.rs/itertools/latest/itertools/trait.Itertools.html#method.multi_cartesian_product) to process `:::` command line inputs. 74 | * [indicatif](https://github.com/console-rs/indicatif) optional TUI progress bar. 75 | * [regex](https://github.com/rust-lang/regex) optional regular expression capture groups processing for `-r`/`--regex` option. 76 | * [tokio](https://tokio.rs/) asynchronous runtime for rust. From tokio this app uses: 77 | * `async` / `await` functions (aka coroutines) 78 | * Singleton `CommandLineArgs` instance using [`tokio::sync::OnceCell`](https://docs.rs/tokio/latest/tokio/sync/struct.OnceCell.html). 79 | * Asynchronous command execution using [`tokio::process::Command`](https://docs.rs/tokio/latest/tokio/process/struct.Command.html) 80 | * [`tokio::sync::Semaphore`](https://docs.rs/tokio/latest/tokio/sync/struct.Semaphore.html) used to limit number of commands that run concurrently. 81 | * [`tokio::sync::mpsc::channel`](https://docs.rs/tokio/latest/tokio/sync/mpsc/fn.channel.html) used to receive inputs from input task, and to send command outputs to an output writer task. To await command completions, use the elegant property that when all `Senders` are dropped the channel is closed. 82 | * [tracing](https://docs.rs/tracing/latest/tracing/) structured debug and warning logs. 83 | * [`tracing::Instrument`](https://docs.rs/tracing/latest/tracing/attr.instrument.html) is used to provide structured debug logs. 84 | * [which](https://github.com/harryfei/which-rs) used to resolve command paths for path cache. 85 | -------------------------------------------------------------------------------- /screenshots/dark_background_progress_bar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aaronriekenberg/rust-parallel/7358f22710fc1073a255940e1151c77fdbd50f5d/screenshots/dark_background_progress_bar.png -------------------------------------------------------------------------------- /screenshots/light_background_progress_bar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aaronriekenberg/rust-parallel/7358f22710fc1073a255940e1151c77fdbd50f5d/screenshots/light_background_progress_bar.png -------------------------------------------------------------------------------- /screenshots/simple_progress_bar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aaronriekenberg/rust-parallel/7358f22710fc1073a255940e1151c77fdbd50f5d/screenshots/simple_progress_bar.png -------------------------------------------------------------------------------- /scripts/benchmark.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | hyperfine --warmup 3 \ 4 | 'seq 1 1000 | rust-parallel echo' \ 5 | 'seq 1 1000 | xargs -P8 -L1 echo' \ 6 | 'seq 1 1000 | parallel echo' 7 | -------------------------------------------------------------------------------- /scripts/generate_manual.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | export NO_COLOR=1 6 | RUST_PARALLEL="./target/debug/rust-parallel" 7 | VERSION=$($RUST_PARALLEL -V | cut -f2 -d' ') 8 | 9 | echo "## Manual for rust-parallel $VERSION" 10 | 11 | echo ' 12 | 1. [Command line options](#command-line-options) 13 | 1. [Commands from arguments](#commands-from-arguments) 14 | 1. [Automatic variables](#automatic-variables) 15 | 1. [Commands from stdin](#commands-from-stdin) 16 | 1. [Command and initial arguments on command line](#command-and-initial-arguments-on-command-line) 17 | 1. [Reading multiple inputs](#reading-multiple-inputs) 18 | 1. [Parallelism](#parallelism) 19 | 1. [Dry run](#dry-run) 20 | 1. [Debug logging](#debug-logging) 21 | 1. [Error handling](#error-handling) 22 | 1. [Timeout](#timeout) 23 | 1. [Path cache](#path-cache) 24 | 1. [Progress bar](#progress-bar) 25 | 1. [Regular Expression](#regular-expression) 26 | 1. [Named Capture Groups](#named-capture-groups) 27 | 1. [Numbered Capture Groups](#numbered-capture-groups) 28 | 1. [Capture Group Special Characters](#capture-group-special-characters) 29 | 1. [Shell Commands](#shell-commands) 30 | 1. [Bash Function](#bash-function) 31 | 1. [Function Setup](#function-setup) 32 | 1. [Demo of command line arguments](#demo-of-command-line-arguments) 33 | 1. [Demo of function and command line arguments from stdin](#demo-of-function-and-command-line-arguments-from-stdin) 34 | 1. [Demo of function and initial arguments on command line, additional arguments from stdin](#demo-of-function-and-initial-arguments-on-command-line-additional-arguments-from-stdin) 35 | ' 36 | 37 | echo '## Command line options' 38 | 39 | echo '``` 40 | $ rust-parallel --help' 41 | $RUST_PARALLEL --help 42 | echo '```' 43 | 44 | echo '## Commands from arguments 45 | 46 | The `:::` separator can be used to run the [Cartesian Product](https://en.wikipedia.org/wiki/Cartesian_product) of command line arguments. This is similar to the `:::` behavior in GNU Parallel. 47 | ' 48 | 49 | echo '``` 50 | $ rust-parallel echo ::: A B ::: C D ::: E F G' 51 | $RUST_PARALLEL echo ::: A B ::: C D ::: E F G 52 | 53 | echo ' 54 | $ rust-parallel echo hello ::: larry curly moe' 55 | $RUST_PARALLEL echo hello ::: larry curly moe 56 | 57 | echo ' 58 | # run gzip -k on all *.html files in current directory 59 | $ rust-parallel gzip -k ::: *.html 60 | ```' 61 | 62 | echo '### Automatic Variables' 63 | 64 | echo 'When using commands from arguments, numbered variables `{0}`, `{1}`, etc are automatically available based on the number of arguments. `{0}` will be replaced by the entire input line, and other groups match individual argument groups. This is useful for building more complex command lines. For example: 65 | ' 66 | 67 | echo '``` 68 | $ rust-parallel echo group0={0} group1={1} group2={2} group3={3} group2again={2} ::: A B ::: C D ::: E F G' 69 | $RUST_PARALLEL echo group0={0} group1={1} group2={2} group3={3} group2again={2} ::: A B ::: C D ::: E F G 70 | echo '```' 71 | 72 | echo 'Internally these variables are implemented using an auto-generated [regular expression](#regular-expression). If a regular expression is manually specified this will override the auto-generated one.' 73 | 74 | echo '## Commands from stdin 75 | 76 | Run complete commands from stdin. 77 | 78 | ' 79 | echo '``` 80 | $ cat >./test <./test<./test <./test <&1 212 | RET_VAL=$? 213 | set -e 214 | 215 | echo ' 216 | $ echo $?' 217 | echo $RET_VAL 218 | echo '```' 219 | 220 | echo 'The `--exit-on-error` option can be used to exit after one command fails. 221 | 222 | rust-parallel waits for in-progress commands to finish before exiting and then exits with status 1.' 223 | echo '``` 224 | $ head -100 /usr/share/dict/words | rust-parallel --exit-on-error cat' 225 | set +e 226 | head -100 /usr/share/dict/words | $RUST_PARALLEL --exit-on-error cat 2>&1 227 | RET_VAL=$? 228 | set -e 229 | 230 | echo ' 231 | $ echo $?' 232 | echo $RET_VAL 233 | echo '```' 234 | 235 | echo ' 236 | ## Timeout 237 | 238 | The `-t`/`--timeout-seconds` option can be used to specify a command timeout in seconds. If any command times out this is considered a command failure (see [error handling](#error-handling)). 239 | ' 240 | 241 | echo '```' 242 | 243 | echo '$ rust-parallel -t 0.5 sleep ::: 0 3 5' 244 | set +e 245 | $RUST_PARALLEL -t 0.5 sleep ::: 0 3 5 246 | RET_VAL=$? 247 | set -e 248 | 249 | echo ' 250 | $ echo $?' 251 | echo $RET_VAL 252 | echo '```' 253 | 254 | echo ' 255 | ## Path Cache 256 | 257 | By default as commands are run the full paths are resolved using [which](https://github.com/harryfei/which-rs). Resolved paths are stored in a cache to prevent duplicate resolutions. This is generally [good for performance](https://github.com/aaronriekenberg/rust-parallel/wiki/Benchmarks). 258 | 259 | The path cache can be disabled using the `--disable-path-cache` option. 260 | ' 261 | 262 | echo '## Progress bar 263 | 264 | The `-p`/`--progress-bar` option can be used to enable a graphical progress bar. 265 | 266 | This is best used for commands which are running for at least a few seconds, and which do not produce output to stdout or stderr. In the below commands `-d all` is used to discard all output from commands run. 267 | 268 | Progress styles can be chosen with the `PROGRESS_STYLE` environment variable. If `PROGRESS_STYLE` is not set it defaults to `light_bg`. 269 | 270 | The following progress styles are available: 271 | * `PROGRESS_STYLE=light_bg` good for light terminal background with colors, spinner, and steady tick enabled: 272 | ![light_bg](https://github.com/aaronriekenberg/rust-parallel/blob/main/screenshots/light_background_progress_bar.png) 273 | 274 | * `PROGRESS_STYLE=dark_bg` good for dark terminal background with colors, spinner, and steady tick enabled: 275 | ![dark_bg](https://github.com/aaronriekenberg/rust-parallel/blob/main/screenshots/dark_background_progress_bar.png) 276 | 277 | * `PROGRESS_STYLE=simple` good for simple or non-ansi terminals/jobs with colors, spinner, and steady tick disabled: 278 | ![simple](https://github.com/aaronriekenberg/rust-parallel/blob/main/screenshots/simple_progress_bar.png) 279 | 280 | ## Regular Expression 281 | 282 | Regular expressions can be specified by the `-r` or `--regex` command line argument. 283 | 284 | [Named or numbered capture groups](https://docs.rs/regex/latest/regex/#grouping-and-flags) are expanded with data values from the current input before the command is executed. 285 | 286 | ### Named Capture Groups 287 | 288 | In these examples using command line arguments `{url}` and `{filename}` are named capture groups. `{0}` is a numbered capture group. 289 | ' 290 | 291 | echo '```' 292 | echo -e '$ rust-parallel -r \x27(?P.*),(?P.*)\x27 echo got url={url} filename={filename} ::: URL1,filename1 URL2,filename2' 293 | $RUST_PARALLEL -r '(?P.*),(?P.*)' echo got url={url} filename={filename} ::: URL1,filename1 URL2,filename2 294 | 295 | echo 296 | echo -e '$ rust-parallel -r \x27(?P.*) (?P.*)\x27 echo got url={url} filename={filename} full input={0} ::: URL1 URL2 ::: filename1 filename2' 297 | $RUST_PARALLEL -r '(?P.*) (?P.*)' echo got url={url} filename={filename} full input={0} ::: URL1 URL2 ::: filename1 filename2 298 | 299 | echo '```' 300 | 301 | echo '### Numbered Capture Groups 302 | 303 | In the next example input file arguments `{0}` `{1}` `{2}` `{3}` are numbered capture groups, and the input is a csv file:' 304 | 305 | echo '```' 306 | echo '$ cat >./test <./test <./test <./test <.*) (?P.*)\x27 \x27FOO={arg1}; BAR={arg2}; echo "FOO = ${FOO}, BAR = ${BAR}, shell pid = $$, date = $(date)"\x27 ::: A B ::: CAT DOG' 361 | $RUST_PARALLEL -s -r '(?P.*) (?P.*)' 'FOO={arg1}; BAR={arg2}; echo "FOO = ${FOO}, BAR = ${BAR}, shell pid = $$, date = $(date)"' ::: A B ::: CAT DOG 362 | echo '```' 363 | 364 | echo '## Bash Function 365 | 366 | `-s` shell mode can be used to invoke an arbitrary bash function. 367 | 368 | Similar to normal commands bash functions can be called using stdin, input files, or from command line arguments.' 369 | 370 | echo '### Function Setup 371 | 372 | Define a bash fuction `logargs` that logs all arguments and make visible with `export -f`: 373 | ' 374 | 375 | echo '```' 376 | 377 | echo '$ logargs() { 378 | echo "logargs got $@" 379 | }' 380 | logargs() { 381 | echo "logargs got $@" 382 | } 383 | 384 | echo ' 385 | $ export -f logargs' 386 | export -f logargs 387 | 388 | echo '```' 389 | 390 | echo '### Demo of command line arguments: 391 | ' 392 | 393 | echo '``` 394 | $ rust-parallel -s logargs ::: A B C ::: D E F' 395 | $RUST_PARALLEL -s logargs ::: A B C ::: D E F 396 | 397 | echo '```' 398 | 399 | echo '### Demo of function and command line arguments from stdin:' 400 | 401 | echo '``` 402 | $ cat >./test <./test <./test <./test < { 55 | error!("spawn error command: {}: {}", self, e); 56 | command_metrics.increment_spawn_errors(); 57 | return; 58 | } 59 | Ok(child_process) => child_process, 60 | }; 61 | 62 | if span_enabled!(Level::DEBUG) { 63 | let child_pid = child_process.id(); 64 | Span::current().record("child_pid", child_pid); 65 | 66 | debug!("spawned child process, awaiting completion"); 67 | } 68 | 69 | match child_process.await_completion().await { 70 | Err(e) => { 71 | error!("child process error command: {} error: {}", self, e); 72 | command_metrics.handle_child_process_execution_error(e); 73 | } 74 | Ok(output) => { 75 | debug!("command exit status = {}", output.status); 76 | if !output.status.success() { 77 | command_metrics.increment_exit_status_errors(); 78 | } 79 | 80 | output_sender 81 | .send(output, self.command_and_args, self.input_line_number) 82 | .await; 83 | } 84 | }; 85 | 86 | debug!("end run"); 87 | } 88 | } 89 | 90 | impl std::fmt::Display for Command { 91 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 92 | write!( 93 | f, 94 | "cmd={:?},args={:?},line={}", 95 | self.command_and_args.command_path, self.command_and_args.args, self.input_line_number, 96 | ) 97 | } 98 | } 99 | 100 | pub struct CommandService { 101 | command_line_args: &'static CommandLineArgs, 102 | command_path_cache: CommandPathCache, 103 | command_semaphore: Arc, 104 | context: Arc, 105 | output_writer: OutputWriter, 106 | } 107 | 108 | impl CommandService { 109 | pub fn new(command_line_args: &'static CommandLineArgs, progress: Arc) -> Self { 110 | let context = Arc::new(CommandRunContext { 111 | child_process_factory: ChildProcessFactory::new(command_line_args), 112 | command_metrics: CommandMetrics::default(), 113 | progress, 114 | }); 115 | Self { 116 | command_line_args, 117 | command_path_cache: CommandPathCache::new(command_line_args), 118 | command_semaphore: Arc::new(Semaphore::new(command_line_args.jobs)), 119 | context, 120 | output_writer: OutputWriter::new(command_line_args), 121 | } 122 | } 123 | 124 | async fn spawn_command( 125 | &self, 126 | command_and_args: OwnedCommandAndArgs, 127 | input_line_number: InputLineNumber, 128 | ) -> anyhow::Result<()> { 129 | let command = Command { 130 | command_and_args, 131 | input_line_number, 132 | }; 133 | 134 | if self.command_line_args.dry_run { 135 | info!("{}", command); 136 | return Ok(()); 137 | } 138 | 139 | if self.command_line_args.exit_on_error && self.context.command_metrics.error_occurred() { 140 | trace!("return from spawn_command due to exit_on_error"); 141 | return Ok(()); 142 | } 143 | 144 | let context_clone = Arc::clone(&self.context); 145 | 146 | let output_sender = self.output_writer.sender(); 147 | 148 | let permit = Arc::clone(&self.command_semaphore) 149 | .acquire_owned() 150 | .await 151 | .context("command_semaphore.acquire_owned error")?; 152 | 153 | tokio::spawn(async move { 154 | command.run(&context_clone, output_sender).await; 155 | 156 | drop(permit); 157 | 158 | context_clone.progress.command_finished(); 159 | }); 160 | 161 | Ok(()) 162 | } 163 | 164 | async fn process_input_message(&self, input_message: InputMessage) -> anyhow::Result<()> { 165 | let InputMessage { 166 | command_and_args, 167 | input_line_number, 168 | } = input_message; 169 | 170 | let Some(command_and_args) = self 171 | .command_path_cache 172 | .resolve_command_path(command_and_args) 173 | .await? 174 | else { 175 | return Ok(()); 176 | }; 177 | 178 | self.spawn_command(command_and_args, input_line_number) 179 | .await?; 180 | 181 | Ok(()) 182 | } 183 | 184 | async fn process_inputs(&self) -> anyhow::Result<()> { 185 | let mut input_producer = 186 | InputProducer::new(self.command_line_args, &self.context.progress)?; 187 | 188 | while let Some(input_message) = input_producer.receiver().recv().await { 189 | self.process_input_message(input_message).await?; 190 | } 191 | 192 | input_producer.wait_for_completion().await?; 193 | 194 | Ok(()) 195 | } 196 | 197 | #[instrument(name = "CommandService::run_commands", skip_all, level = "debug")] 198 | pub async fn run_commands(self) -> anyhow::Result<()> { 199 | debug!("begin run_commands"); 200 | 201 | self.process_inputs().await?; 202 | 203 | debug!("before output_writer.wait_for_completion",); 204 | 205 | self.output_writer.wait_for_completion().await?; 206 | 207 | self.context.progress.finish(); 208 | 209 | if self.context.command_metrics.error_occurred() { 210 | anyhow::bail!("command failures: {}", self.context.command_metrics); 211 | } 212 | 213 | debug!( 214 | "end run_commands command_metrics = {}", 215 | self.context.command_metrics 216 | ); 217 | 218 | Ok(()) 219 | } 220 | } 221 | 222 | struct CommandRunContext { 223 | child_process_factory: ChildProcessFactory, 224 | command_metrics: CommandMetrics, 225 | progress: Arc, 226 | } 227 | -------------------------------------------------------------------------------- /src/command/metrics.rs: -------------------------------------------------------------------------------- 1 | use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; 2 | 3 | use crate::process::ChildProcessExecutionError; 4 | 5 | const ORDERING: Ordering = Ordering::SeqCst; 6 | 7 | #[derive(Debug, Default)] 8 | pub struct CommandMetrics { 9 | commands_run: AtomicU64, 10 | error_occurred: AtomicBool, 11 | spawn_errors: AtomicU64, 12 | timeouts: AtomicU64, 13 | io_errors: AtomicU64, 14 | exit_status_errors: AtomicU64, 15 | } 16 | 17 | impl CommandMetrics { 18 | pub fn increment_commands_run(&self) { 19 | self.commands_run.fetch_add(1, ORDERING); 20 | } 21 | 22 | fn commands_run(&self) -> u64 { 23 | self.commands_run.load(ORDERING) 24 | } 25 | 26 | pub fn error_occurred(&self) -> bool { 27 | self.error_occurred.load(ORDERING) 28 | } 29 | 30 | fn set_error_occurred(&self) { 31 | self.error_occurred.store(true, ORDERING); 32 | } 33 | 34 | fn total_failures(&self) -> u64 { 35 | self.spawn_errors() + self.timeouts() + self.io_errors() + self.exit_status_errors() 36 | } 37 | 38 | pub fn increment_spawn_errors(&self) { 39 | self.set_error_occurred(); 40 | self.spawn_errors.fetch_add(1, ORDERING); 41 | } 42 | 43 | fn spawn_errors(&self) -> u64 { 44 | self.spawn_errors.load(ORDERING) 45 | } 46 | 47 | pub fn handle_child_process_execution_error(&self, error: ChildProcessExecutionError) { 48 | match error { 49 | ChildProcessExecutionError::IOError(_) => self.increment_io_errors(), 50 | ChildProcessExecutionError::Timeout(_) => self.increment_timeouts(), 51 | } 52 | } 53 | 54 | fn increment_timeouts(&self) { 55 | self.set_error_occurred(); 56 | self.timeouts.fetch_add(1, ORDERING); 57 | } 58 | 59 | fn timeouts(&self) -> u64 { 60 | self.timeouts.load(ORDERING) 61 | } 62 | 63 | fn increment_io_errors(&self) { 64 | self.set_error_occurred(); 65 | self.io_errors.fetch_add(1, ORDERING); 66 | } 67 | 68 | fn io_errors(&self) -> u64 { 69 | self.io_errors.load(ORDERING) 70 | } 71 | 72 | pub fn increment_exit_status_errors(&self) { 73 | self.set_error_occurred(); 74 | self.exit_status_errors.fetch_add(1, ORDERING); 75 | } 76 | 77 | fn exit_status_errors(&self) -> u64 { 78 | self.exit_status_errors.load(ORDERING) 79 | } 80 | } 81 | 82 | impl std::fmt::Display for CommandMetrics { 83 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 84 | write!( 85 | f, 86 | "commands_run={} total_failures={} spawn_errors={} timeouts={} io_errors={} exit_status_errors={}", 87 | self.commands_run(), 88 | self.total_failures(), 89 | self.spawn_errors(), 90 | self.timeouts(), 91 | self.io_errors(), 92 | self.exit_status_errors(), 93 | ) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/command/path_cache.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Context; 2 | 3 | use tokio::sync::Mutex; 4 | 5 | use tracing::warn; 6 | 7 | use std::{collections::HashMap, path::PathBuf}; 8 | 9 | use crate::{command_line_args::CommandLineArgs, common::OwnedCommandAndArgs}; 10 | 11 | enum CacheValue { 12 | NotResolvable, 13 | 14 | Resolved(PathBuf), 15 | } 16 | 17 | pub struct CommandPathCache { 18 | enabled: bool, 19 | cache: Mutex>, 20 | } 21 | 22 | impl CommandPathCache { 23 | pub fn new(command_line_args: &CommandLineArgs) -> Self { 24 | Self { 25 | enabled: !command_line_args.disable_path_cache, 26 | cache: Mutex::new(HashMap::new()), 27 | } 28 | } 29 | 30 | pub async fn resolve_command_path( 31 | &self, 32 | command_and_args: OwnedCommandAndArgs, 33 | ) -> anyhow::Result> { 34 | if !self.enabled { 35 | return Ok(Some(command_and_args)); 36 | } 37 | 38 | let mut command_and_args = command_and_args; 39 | 40 | let command_path = &command_and_args.command_path; 41 | 42 | let mut cache = self.cache.lock().await; 43 | 44 | if let Some(cached_value) = cache.get(command_path) { 45 | return Ok(match cached_value { 46 | CacheValue::NotResolvable => None, 47 | CacheValue::Resolved(cached_path) => { 48 | command_and_args.command_path.clone_from(cached_path); 49 | Some(command_and_args) 50 | } 51 | }); 52 | } 53 | 54 | let command_path_clone = command_path.clone(); 55 | 56 | let which_result = tokio::task::spawn_blocking(move || which::which(command_path_clone)) 57 | .await 58 | .context("spawn_blocking error")?; 59 | 60 | let full_path = match which_result { 61 | Ok(path) => path, 62 | Err(e) => { 63 | warn!("error resolving path {:?}: {}", command_path, e); 64 | cache.insert(command_path.clone(), CacheValue::NotResolvable); 65 | return Ok(None); 66 | } 67 | }; 68 | 69 | cache.insert( 70 | command_path.clone(), 71 | CacheValue::Resolved(full_path.clone()), 72 | ); 73 | 74 | command_and_args.command_path = full_path; 75 | 76 | Ok(Some(command_and_args)) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/command_line_args.rs: -------------------------------------------------------------------------------- 1 | use clap::{Parser, ValueEnum}; 2 | 3 | use tokio::sync::OnceCell; 4 | 5 | use tracing::debug; 6 | 7 | pub const COMMANDS_FROM_ARGS_SEPARATOR: &str = ":::"; 8 | 9 | /// Execute commands in parallel 10 | /// 11 | /// By Aaron Riekenberg 12 | /// 13 | /// https://github.com/aaronriekenberg/rust-parallel 14 | /// https://crates.io/crates/rust-parallel 15 | #[derive(Parser, Debug, Default)] 16 | #[command(verbatim_doc_comment, version)] 17 | pub struct CommandLineArgs { 18 | /// Discard output for commands 19 | #[arg(short, long)] 20 | pub discard_output: Option, 21 | 22 | /// Input file or - for stdin. Defaults to stdin if no inputs are specified. 23 | #[arg(short, long)] 24 | pub input_file: Vec, 25 | 26 | /// Maximum number of commands to run in parallel, defauts to num cpus 27 | #[arg(short, long, default_value_t = num_cpus::get(), value_parser = Self::parse_semaphore_permits)] 28 | pub jobs: usize, 29 | 30 | /// Use null separator for reading input files instead of newline. 31 | #[arg(short('0'), long)] 32 | pub null_separator: bool, 33 | 34 | /// Display progress bar. 35 | #[arg(short, long)] 36 | pub progress_bar: bool, 37 | 38 | /// Progress bar style. 39 | #[arg(long)] 40 | pub progress_bar_style: Option, 41 | 42 | /// Apply regex pattern to inputs. 43 | #[arg(short, long)] 44 | pub regex: Option, 45 | 46 | /// Use shell mode for running commands. 47 | /// 48 | /// Each command line is passed to " " as a single argument. 49 | #[arg(short, long)] 50 | pub shell: bool, 51 | 52 | /// Timeout seconds for running commands. Defaults to infinite timeout if not specified. 53 | #[arg(short, long, value_parser = Self::parse_timeout_seconds)] 54 | pub timeout_seconds: Option, 55 | 56 | /// Input and output channel capacity, defaults to num cpus * 2 57 | #[arg(long, default_value_t = num_cpus::get() * 2, value_parser = Self::parse_semaphore_permits)] 58 | pub channel_capacity: usize, 59 | 60 | /// Disable command path cache 61 | #[arg(long)] 62 | pub disable_path_cache: bool, 63 | 64 | /// Dry run mode 65 | /// 66 | /// Do not actually run commands just log. 67 | #[arg(long)] 68 | pub dry_run: bool, 69 | 70 | /// Exit on error mode 71 | /// 72 | /// Exit immediately when a command fails. 73 | #[arg(long)] 74 | pub exit_on_error: bool, 75 | 76 | /// Do not run commands for empty buffered input lines. 77 | #[arg(long)] 78 | pub no_run_if_empty: bool, 79 | 80 | /// Path to shell to use for shell mode 81 | #[arg(long, default_value = Self::default_shell())] 82 | pub shell_path: String, 83 | 84 | /// Argument to shell for shell mode 85 | #[arg(long, default_value = Self::default_shell_argument())] 86 | pub shell_argument: String, 87 | 88 | /// Optional command and initial arguments. 89 | /// 90 | /// If this contains 1 or more ::: delimiters the cartesian product 91 | /// of arguments from all groups are run. 92 | #[arg(trailing_var_arg(true))] 93 | pub command_and_initial_arguments: Vec, 94 | } 95 | 96 | impl CommandLineArgs { 97 | pub async fn instance() -> &'static Self { 98 | static INSTANCE: OnceCell = OnceCell::const_new(); 99 | 100 | INSTANCE 101 | .get_or_init(|| async move { 102 | let command_line_args = CommandLineArgs::parse(); 103 | 104 | debug!("command_line_args = {:?}", command_line_args); 105 | 106 | command_line_args 107 | }) 108 | .await 109 | } 110 | 111 | pub fn commands_from_args_mode(&self) -> bool { 112 | self.command_and_initial_arguments 113 | .iter() 114 | .any(|s| s == COMMANDS_FROM_ARGS_SEPARATOR) 115 | } 116 | 117 | fn parse_semaphore_permits(s: &str) -> Result { 118 | let range = 1..=tokio::sync::Semaphore::MAX_PERMITS; 119 | 120 | let value: usize = s.parse().map_err(|_| format!("`{s}` isn't a number"))?; 121 | if range.contains(&value) { 122 | Ok(value) 123 | } else { 124 | Err(format!("value not in range {:?}", range)) 125 | } 126 | } 127 | 128 | fn parse_timeout_seconds(s: &str) -> Result { 129 | let value: f64 = s.parse().map_err(|_| format!("`{s}` isn't a number"))?; 130 | if value > 0f64 { 131 | Ok(value) 132 | } else { 133 | Err("value not greater than 0".to_string()) 134 | } 135 | } 136 | 137 | fn default_shell() -> &'static str { 138 | if cfg!(unix) { 139 | "/bin/bash" 140 | } else if cfg!(windows) { 141 | "cmd" 142 | } else { 143 | unreachable!() 144 | } 145 | } 146 | 147 | fn default_shell_argument() -> &'static str { 148 | if cfg!(unix) { 149 | "-c" 150 | } else if cfg!(windows) { 151 | "/c" 152 | } else { 153 | unreachable!() 154 | } 155 | } 156 | } 157 | 158 | #[derive(Clone, Copy, Debug, ValueEnum)] 159 | pub enum DiscardOutput { 160 | /// Redirect stdout for commands to /dev/null 161 | Stdout, 162 | /// Redirect stderr for commands to /dev/null 163 | Stderr, 164 | /// Redirect stdout and stderr for commands to /dev/null 165 | All, 166 | } 167 | 168 | #[cfg(test)] 169 | mod test { 170 | use super::*; 171 | 172 | #[test] 173 | fn test_clap_configuation() { 174 | use clap::CommandFactory; 175 | 176 | CommandLineArgs::command().debug_assert() 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /src/common.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::VecDeque, path::PathBuf}; 2 | 3 | #[derive(Debug, Eq, PartialEq)] 4 | pub struct OwnedCommandAndArgs { 5 | pub command_path: PathBuf, 6 | pub args: Vec, 7 | } 8 | 9 | impl std::fmt::Display for OwnedCommandAndArgs { 10 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 11 | write!(f, "cmd={:?},args={:?}", self.command_path, self.args) 12 | } 13 | } 14 | 15 | #[derive(thiserror::Error, Debug)] 16 | pub enum OwnedCommandAndArgsConversionError { 17 | #[error("empty input")] 18 | EmptyInput, 19 | } 20 | 21 | impl TryFrom> for OwnedCommandAndArgs { 22 | type Error = OwnedCommandAndArgsConversionError; 23 | 24 | fn try_from(mut deque: VecDeque) -> Result { 25 | let command = deque 26 | .pop_front() 27 | .ok_or(OwnedCommandAndArgsConversionError::EmptyInput)?; 28 | 29 | Ok(Self { 30 | command_path: PathBuf::from(command), 31 | args: deque.into(), 32 | }) 33 | } 34 | } 35 | 36 | impl TryFrom> for OwnedCommandAndArgs { 37 | type Error = OwnedCommandAndArgsConversionError; 38 | 39 | fn try_from(vec: Vec) -> Result { 40 | Self::try_from(VecDeque::from(vec)) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/input.rs: -------------------------------------------------------------------------------- 1 | mod buffered_reader; 2 | mod task; 3 | 4 | use anyhow::Context; 5 | 6 | use tokio::{ 7 | sync::mpsc::{Receiver, channel}, 8 | task::JoinHandle, 9 | }; 10 | 11 | use tracing::debug; 12 | 13 | use std::sync::Arc; 14 | 15 | use crate::{command_line_args::CommandLineArgs, common::OwnedCommandAndArgs, progress::Progress}; 16 | 17 | #[derive(Debug, Clone, Copy)] 18 | pub enum BufferedInput { 19 | Stdin, 20 | 21 | File { file_name: &'static str }, 22 | } 23 | 24 | impl std::fmt::Display for BufferedInput { 25 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 26 | match self { 27 | Self::Stdin => write!(f, "stdin"), 28 | Self::File { file_name } => write!(f, "{}", file_name), 29 | } 30 | } 31 | } 32 | 33 | #[derive(Debug, Clone, Copy)] 34 | pub enum Input { 35 | Buffered(BufferedInput), 36 | 37 | CommandLineArgs, 38 | } 39 | 40 | impl std::fmt::Display for Input { 41 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 42 | match self { 43 | Self::Buffered(b) => write!(f, "{}", b), 44 | Self::CommandLineArgs => write!(f, "command_line_args"), 45 | } 46 | } 47 | } 48 | 49 | #[derive(Debug)] 50 | pub struct InputLineNumber { 51 | pub input: Input, 52 | pub line_number: usize, 53 | } 54 | 55 | impl std::fmt::Display for InputLineNumber { 56 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 57 | write!(f, "{}:{}", self.input, self.line_number) 58 | } 59 | } 60 | 61 | enum InputList { 62 | BufferedInputList(Vec), 63 | 64 | CommandLineArgs, 65 | } 66 | 67 | fn build_input_list(command_line_args: &'static CommandLineArgs) -> InputList { 68 | if command_line_args.commands_from_args_mode() { 69 | InputList::CommandLineArgs 70 | } else if command_line_args.input_file.is_empty() { 71 | InputList::BufferedInputList(vec![BufferedInput::Stdin]) 72 | } else { 73 | InputList::BufferedInputList( 74 | command_line_args 75 | .input_file 76 | .iter() 77 | .map(|input_name| { 78 | if input_name == "-" { 79 | BufferedInput::Stdin 80 | } else { 81 | BufferedInput::File { 82 | file_name: input_name, 83 | } 84 | } 85 | }) 86 | .collect(), 87 | ) 88 | } 89 | } 90 | 91 | #[derive(Debug)] 92 | pub struct InputMessage { 93 | pub command_and_args: OwnedCommandAndArgs, 94 | pub input_line_number: InputLineNumber, 95 | } 96 | 97 | pub struct InputProducer { 98 | input_task_join_handle: JoinHandle<()>, 99 | receiver: Receiver, 100 | } 101 | 102 | impl InputProducer { 103 | pub fn new( 104 | command_line_args: &'static CommandLineArgs, 105 | progress: &Arc, 106 | ) -> anyhow::Result { 107 | let (sender, receiver) = channel(command_line_args.channel_capacity); 108 | debug!( 109 | "created input channel with capacity {}", 110 | command_line_args.channel_capacity 111 | ); 112 | 113 | let input_sender_task = task::InputTask::new(command_line_args, sender, progress)?; 114 | 115 | let input_task_join_handle = tokio::spawn(input_sender_task.run()); 116 | 117 | Ok(Self { 118 | input_task_join_handle, 119 | receiver, 120 | }) 121 | } 122 | 123 | pub fn receiver(&mut self) -> &mut Receiver { 124 | &mut self.receiver 125 | } 126 | 127 | pub async fn wait_for_completion(self) -> anyhow::Result<()> { 128 | self.input_task_join_handle 129 | .await 130 | .context("InputProducer::wait_for_completion: input_task_join_handle.await error")?; 131 | 132 | Ok(()) 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/input/buffered_reader.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Context; 2 | 3 | use tokio::io::{AsyncBufRead, AsyncBufReadExt, BufReader, Split}; 4 | 5 | use crate::command_line_args::CommandLineArgs; 6 | 7 | use super::{BufferedInput, Input, InputLineNumber}; 8 | 9 | type AsyncBufReadBox = Box; 10 | 11 | pub struct BufferedInputReader { 12 | buffered_input: BufferedInput, 13 | split: Split, 14 | next_line_number: usize, 15 | } 16 | 17 | impl BufferedInputReader { 18 | pub async fn new( 19 | buffered_input: BufferedInput, 20 | command_line_args: &CommandLineArgs, 21 | ) -> anyhow::Result { 22 | let buf_reader = Self::create_buf_reader(buffered_input).await?; 23 | 24 | let line_separator = if command_line_args.null_separator { 25 | 0u8 26 | } else { 27 | b'\n' 28 | }; 29 | 30 | let split = buf_reader.split(line_separator); 31 | 32 | Ok(Self { 33 | buffered_input, 34 | split, 35 | next_line_number: 0, 36 | }) 37 | } 38 | 39 | async fn create_buf_reader(buffered_input: BufferedInput) -> anyhow::Result { 40 | match buffered_input { 41 | BufferedInput::Stdin => { 42 | let buf_reader = BufReader::new(tokio::io::stdin()); 43 | 44 | Ok(Box::new(buf_reader)) 45 | } 46 | BufferedInput::File { file_name } => { 47 | let file = tokio::fs::File::open(file_name).await.with_context(|| { 48 | format!("error opening input file file_name = '{}'", file_name) 49 | })?; 50 | let buf_reader = BufReader::new(file); 51 | 52 | Ok(Box::new(buf_reader)) 53 | } 54 | } 55 | } 56 | 57 | pub async fn next_segment(&mut self) -> anyhow::Result)>> { 58 | let segment = self.split.next_segment().await?; 59 | 60 | match segment { 61 | None => Ok(None), 62 | Some(segment) => { 63 | self.next_line_number += 1; 64 | 65 | let input_line_number = InputLineNumber { 66 | input: Input::Buffered(self.buffered_input), 67 | line_number: self.next_line_number, 68 | }; 69 | 70 | Ok(Some((input_line_number, segment))) 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/input/task.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Context; 2 | 3 | use tokio::sync::mpsc::Sender; 4 | 5 | use tracing::{debug, instrument, warn}; 6 | 7 | use std::sync::Arc; 8 | 9 | use crate::{ 10 | command_line_args::CommandLineArgs, 11 | parser::{Parsers, buffered::BufferedInputLineParser, command_line::CommandLineArgsParser}, 12 | progress::Progress, 13 | }; 14 | 15 | use super::{ 16 | BufferedInput, Input, InputLineNumber, InputList, InputMessage, 17 | buffered_reader::BufferedInputReader, 18 | }; 19 | 20 | pub struct InputTask { 21 | sender: Sender, 22 | command_line_args: &'static CommandLineArgs, 23 | progress: Arc, 24 | parsers: Parsers, 25 | } 26 | 27 | impl InputTask { 28 | pub fn new( 29 | command_line_args: &'static CommandLineArgs, 30 | sender: Sender, 31 | progress: &Arc, 32 | ) -> anyhow::Result { 33 | let parsers = Parsers::new(command_line_args)?; 34 | Ok(Self { 35 | sender, 36 | command_line_args, 37 | progress: Arc::clone(progress), 38 | parsers, 39 | }) 40 | } 41 | 42 | async fn send(&self, input_message: InputMessage) { 43 | self.progress.increment_total_commands(1); 44 | 45 | if let Err(e) = self.sender.send(input_message).await { 46 | warn!("input sender send error: {}", e); 47 | } 48 | } 49 | 50 | #[instrument( 51 | skip_all, 52 | fields( 53 | line=%input_line_number, 54 | ) 55 | name = "process_buffered_input_line", 56 | )] 57 | async fn process_buffered_input_line( 58 | &self, 59 | parser: &BufferedInputLineParser, 60 | input_line_number: InputLineNumber, 61 | segment: Vec, 62 | ) { 63 | if let Some(command_and_args) = parser.parse_segment(segment) { 64 | self.send(InputMessage { 65 | command_and_args, 66 | input_line_number, 67 | }) 68 | .await 69 | } 70 | } 71 | 72 | async fn process_buffered_input(&self, buffered_input: BufferedInput) -> anyhow::Result<()> { 73 | debug!( 74 | "begin process_buffered_input buffered_input {}", 75 | buffered_input 76 | ); 77 | 78 | let mut input_reader = 79 | BufferedInputReader::new(buffered_input, self.command_line_args).await?; 80 | 81 | let parser = self.parsers.buffered_input_line_parser().await; 82 | 83 | loop { 84 | match input_reader 85 | .next_segment() 86 | .await 87 | .context("next_segment error")? 88 | { 89 | Some((input_line_number, segment)) => { 90 | self.process_buffered_input_line(parser, input_line_number, segment) 91 | .await 92 | } 93 | None => { 94 | debug!("input_reader.next_segment EOF"); 95 | break; 96 | } 97 | } 98 | } 99 | 100 | Ok(()) 101 | } 102 | 103 | #[instrument( 104 | skip_all, 105 | fields( 106 | line=%input_line_number, 107 | ) 108 | name = "process_next_command_line_arg", 109 | )] 110 | async fn process_next_command_line_arg( 111 | &self, 112 | parser: &mut CommandLineArgsParser, 113 | input_line_number: InputLineNumber, 114 | ) { 115 | if let Some(command_and_args) = parser.parse_next_argument_group() { 116 | self.send(InputMessage { 117 | command_and_args, 118 | input_line_number, 119 | }) 120 | .await 121 | }; 122 | } 123 | 124 | async fn process_command_line_args_input(self) { 125 | debug!("begin process_command_line_args_input"); 126 | 127 | let mut parser = self.parsers.command_line_args_parser(); 128 | 129 | let mut line_number = 0; 130 | 131 | while parser.has_remaining_argument_groups() { 132 | line_number += 1; 133 | 134 | let input_line_number = InputLineNumber { 135 | input: Input::CommandLineArgs, 136 | line_number, 137 | }; 138 | 139 | self.process_next_command_line_arg(&mut parser, input_line_number) 140 | .await; 141 | } 142 | } 143 | 144 | #[instrument(skip_all, name = "InputTask::run", level = "debug")] 145 | pub async fn run(self) { 146 | debug!("begin run"); 147 | 148 | match super::build_input_list(self.command_line_args) { 149 | InputList::BufferedInputList(buffered_inputs) => { 150 | for buffered_input in buffered_inputs { 151 | if let Err(e) = self.process_buffered_input(buffered_input).await { 152 | warn!( 153 | "process_buffered_input error buffered_input = {}: {}", 154 | buffered_input, e 155 | ); 156 | } 157 | } 158 | } 159 | InputList::CommandLineArgs => self.process_command_line_args_input().await, 160 | } 161 | 162 | debug!("end run"); 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use tracing::{debug, error, instrument}; 2 | 3 | use crate::command_line_args::CommandLineArgs; 4 | 5 | mod command; 6 | mod command_line_args; 7 | mod common; 8 | mod input; 9 | mod output; 10 | mod parser; 11 | mod process; 12 | mod progress; 13 | 14 | #[instrument(skip_all, name = "try_main", level = "debug")] 15 | async fn try_main() -> anyhow::Result<()> { 16 | debug!("begin try_main"); 17 | 18 | let command_line_args = CommandLineArgs::instance().await; 19 | 20 | let progress = progress::Progress::new(command_line_args)?; 21 | 22 | let command_service = command::CommandService::new(command_line_args, progress); 23 | 24 | command_service.run_commands().await?; 25 | 26 | debug!("end try_main"); 27 | 28 | Ok(()) 29 | } 30 | 31 | #[tokio::main] 32 | async fn main() { 33 | tracing_subscriber::fmt::init(); 34 | 35 | if let Err(err) = try_main().await { 36 | error!("fatal error in main: {:#}", err); 37 | std::process::exit(1); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/output.rs: -------------------------------------------------------------------------------- 1 | mod task; 2 | 3 | use anyhow::Context; 4 | 5 | use tokio::{ 6 | sync::mpsc::{Sender, channel}, 7 | task::JoinHandle, 8 | }; 9 | 10 | use tracing::{debug, warn}; 11 | 12 | use std::process::{ExitStatus, Output}; 13 | 14 | use crate::{ 15 | command_line_args::CommandLineArgs, common::OwnedCommandAndArgs, input::InputLineNumber, 16 | }; 17 | 18 | #[derive(Debug)] 19 | struct OutputMessage { 20 | exit_status: ExitStatus, 21 | stdout: Vec, 22 | stderr: Vec, 23 | command_and_args: OwnedCommandAndArgs, 24 | input_line_number: InputLineNumber, 25 | } 26 | 27 | pub struct OutputSender { 28 | sender: Sender, 29 | } 30 | 31 | impl OutputSender { 32 | pub async fn send( 33 | self, 34 | output: Output, 35 | command_and_args: OwnedCommandAndArgs, 36 | input_line_number: InputLineNumber, 37 | ) { 38 | if output.status.success() && output.stdout.is_empty() && output.stderr.is_empty() { 39 | return; 40 | } 41 | 42 | let output_message = OutputMessage { 43 | exit_status: output.status, 44 | stdout: output.stdout, 45 | stderr: output.stderr, 46 | command_and_args, 47 | input_line_number, 48 | }; 49 | 50 | if let Err(e) = self.sender.send(output_message).await { 51 | warn!("sender.send error: {}", e); 52 | } 53 | } 54 | } 55 | 56 | pub struct OutputWriter { 57 | sender: Sender, 58 | output_task_join_handle: JoinHandle<()>, 59 | } 60 | 61 | impl OutputWriter { 62 | pub fn new(command_line_args: &CommandLineArgs) -> Self { 63 | let (sender, receiver) = channel(command_line_args.channel_capacity); 64 | debug!( 65 | "created output channel with capacity {}", 66 | command_line_args.channel_capacity, 67 | ); 68 | 69 | let output_task_join_handle = tokio::spawn(task::OutputTask::new(receiver).run()); 70 | 71 | Self { 72 | sender, 73 | output_task_join_handle, 74 | } 75 | } 76 | 77 | pub fn sender(&self) -> OutputSender { 78 | OutputSender { 79 | sender: self.sender.clone(), 80 | } 81 | } 82 | 83 | pub async fn wait_for_completion(self) -> anyhow::Result<()> { 84 | drop(self.sender); 85 | 86 | self.output_task_join_handle 87 | .await 88 | .context("OutputWriter::wait_for_completion: output_task_join_handle.await error")?; 89 | 90 | Ok(()) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/output/task.rs: -------------------------------------------------------------------------------- 1 | use tokio::{io::AsyncWrite, sync::mpsc::Receiver}; 2 | 3 | use tracing::{debug, error, instrument, trace}; 4 | 5 | use super::OutputMessage; 6 | 7 | pub struct OutputTask { 8 | receiver: Receiver, 9 | } 10 | 11 | impl OutputTask { 12 | pub fn new(receiver: Receiver) -> Self { 13 | Self { receiver } 14 | } 15 | 16 | #[instrument(skip_all, name = "OutputTask::run", level = "debug")] 17 | pub async fn run(self) { 18 | debug!("begin run"); 19 | 20 | async fn copy(mut buffer: &[u8], output_stream: &mut (impl AsyncWrite + Unpin)) { 21 | let result = tokio::io::copy(&mut buffer, &mut *output_stream).await; 22 | trace!("copy result = {:?}", result); 23 | } 24 | 25 | let mut stdout = tokio::io::stdout(); 26 | let mut stderr = tokio::io::stderr(); 27 | 28 | let mut receiver = self.receiver; 29 | 30 | while let Some(output_message) = receiver.recv().await { 31 | if !output_message.stdout.is_empty() { 32 | copy(&output_message.stdout, &mut stdout).await; 33 | } 34 | if !output_message.stderr.is_empty() { 35 | copy(&output_message.stderr, &mut stderr).await; 36 | } 37 | if !output_message.exit_status.success() { 38 | error!( 39 | "command failed: {},line={} exit_status={}", 40 | output_message.command_and_args, 41 | output_message.input_line_number, 42 | output_message.exit_status.code().unwrap_or_default(), 43 | ); 44 | } 45 | } 46 | 47 | debug!("end run"); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/parser.rs: -------------------------------------------------------------------------------- 1 | pub mod buffered; 2 | pub mod command_line; 3 | mod regex; 4 | 5 | use tokio::sync::OnceCell; 6 | 7 | use std::sync::Arc; 8 | 9 | use crate::{command_line_args::CommandLineArgs, common::OwnedCommandAndArgs}; 10 | 11 | use self::{ 12 | buffered::BufferedInputLineParser, command_line::CommandLineArgsParser, regex::RegexProcessor, 13 | }; 14 | 15 | struct ShellCommandAndArgs(Option>); 16 | 17 | impl ShellCommandAndArgs { 18 | fn new(command_line_args: &CommandLineArgs) -> Self { 19 | Self(if command_line_args.shell { 20 | Some(vec![ 21 | command_line_args.shell_path.clone(), 22 | command_line_args.shell_argument.clone(), 23 | ]) 24 | } else { 25 | None 26 | }) 27 | } 28 | } 29 | 30 | fn build_owned_command_and_args( 31 | shell_command_and_args: &ShellCommandAndArgs, 32 | command_and_args: Vec, 33 | ) -> Option { 34 | match &shell_command_and_args.0 { 35 | None => OwnedCommandAndArgs::try_from(command_and_args).ok(), 36 | Some(shell_command_and_args) => { 37 | let mut result = Vec::with_capacity(shell_command_and_args.len() + 1); 38 | 39 | result.extend(shell_command_and_args.iter().cloned()); 40 | result.push(command_and_args.join(" ")); 41 | 42 | OwnedCommandAndArgs::try_from(result).ok() 43 | } 44 | } 45 | } 46 | 47 | pub struct Parsers { 48 | buffered_input_line_parser: OnceCell, 49 | regex_processor: Arc, 50 | command_line_args: &'static CommandLineArgs, 51 | } 52 | 53 | impl Parsers { 54 | pub fn new(command_line_args: &'static CommandLineArgs) -> anyhow::Result { 55 | let regex_processor = RegexProcessor::new(command_line_args)?; 56 | 57 | Ok(Self { 58 | buffered_input_line_parser: OnceCell::new(), 59 | regex_processor, 60 | command_line_args, 61 | }) 62 | } 63 | 64 | pub async fn buffered_input_line_parser(&self) -> &BufferedInputLineParser { 65 | self.buffered_input_line_parser 66 | .get_or_init(|| async move { 67 | BufferedInputLineParser::new(self.command_line_args, &self.regex_processor) 68 | }) 69 | .await 70 | } 71 | 72 | pub fn command_line_args_parser(&self) -> CommandLineArgsParser { 73 | CommandLineArgsParser::new(self.command_line_args, &self.regex_processor) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/parser/buffered.rs: -------------------------------------------------------------------------------- 1 | use itertools::Itertools; 2 | 3 | use std::sync::Arc; 4 | 5 | use crate::{ 6 | command_line_args::CommandLineArgs, 7 | common::OwnedCommandAndArgs, 8 | parser::{ShellCommandAndArgs, regex::RegexProcessor}, 9 | }; 10 | 11 | pub struct BufferedInputLineParser { 12 | no_run_if_empty: bool, 13 | split_whitespace: bool, 14 | shell_command_and_args: ShellCommandAndArgs, 15 | command_and_initial_arguments: Vec, 16 | regex_processor: Arc, 17 | } 18 | 19 | impl BufferedInputLineParser { 20 | pub fn new(command_line_args: &CommandLineArgs, regex_processor: &Arc) -> Self { 21 | let split_whitespace = !command_line_args.null_separator; 22 | 23 | let command_and_initial_arguments = command_line_args.command_and_initial_arguments.clone(); 24 | 25 | let shell_command_and_args = ShellCommandAndArgs::new(command_line_args); 26 | 27 | Self { 28 | no_run_if_empty: command_line_args.no_run_if_empty, 29 | split_whitespace, 30 | shell_command_and_args, 31 | command_and_initial_arguments, 32 | regex_processor: Arc::clone(regex_processor), 33 | } 34 | } 35 | 36 | pub fn parse_segment(&self, segment: Vec) -> Option { 37 | if let Ok(input_line) = std::str::from_utf8(&segment) { 38 | self.parse_line(input_line) 39 | } else { 40 | None 41 | } 42 | } 43 | 44 | pub fn parse_line(&self, input_line: &str) -> Option { 45 | if self.no_run_if_empty && input_line.trim().is_empty() { 46 | return None; 47 | } 48 | 49 | let cmd_and_args = if !self.regex_processor.regex_mode() { 50 | let mut cmd_and_args = if self.split_whitespace { 51 | input_line.split_whitespace().map_into().collect() 52 | } else { 53 | vec![input_line.into()] 54 | }; 55 | 56 | if !self.command_and_initial_arguments.is_empty() { 57 | cmd_and_args = [self.command_and_initial_arguments.clone(), cmd_and_args].concat(); 58 | } 59 | 60 | cmd_and_args 61 | } else { 62 | let apply_regex_result = self 63 | .regex_processor 64 | .apply_regex_to_arguments(&self.command_and_initial_arguments, input_line)?; 65 | apply_regex_result.arguments 66 | }; 67 | 68 | super::build_owned_command_and_args(&self.shell_command_and_args, cmd_and_args) 69 | } 70 | } 71 | 72 | #[cfg(test)] 73 | mod test { 74 | use super::*; 75 | 76 | use std::{default::Default, path::PathBuf}; 77 | 78 | #[test] 79 | fn test_split_whitespace() { 80 | let command_line_args = CommandLineArgs { 81 | null_separator: false, 82 | shell: false, 83 | command_and_initial_arguments: vec![], 84 | ..Default::default() 85 | }; 86 | 87 | let parser = BufferedInputLineParser::new( 88 | &command_line_args, 89 | &RegexProcessor::new(&command_line_args).unwrap(), 90 | ); 91 | 92 | let result = parser.parse_line("echo hi there"); 93 | 94 | assert_eq!( 95 | result, 96 | Some(OwnedCommandAndArgs { 97 | command_path: PathBuf::from("echo"), 98 | args: vec!["hi", "there"].into_iter().map_into().collect(), 99 | }) 100 | ); 101 | 102 | let result = parser.parse_line(" echo hi there "); 103 | 104 | assert_eq!( 105 | result, 106 | Some(OwnedCommandAndArgs { 107 | command_path: PathBuf::from("echo"), 108 | args: vec!["hi", "there"].into_iter().map_into().collect(), 109 | }) 110 | ); 111 | 112 | let result = parser.parse_line(" /bin/echo "); 113 | 114 | assert_eq!( 115 | result, 116 | Some(OwnedCommandAndArgs { 117 | command_path: PathBuf::from("/bin/echo"), 118 | args: vec![], 119 | }) 120 | ); 121 | 122 | let result = parser.parse_line(""); 123 | 124 | assert_eq!(result, None); 125 | } 126 | 127 | #[test] 128 | fn test_null_separator() { 129 | let command_line_args = CommandLineArgs { 130 | null_separator: true, 131 | shell: false, 132 | command_and_initial_arguments: vec!["gzip".to_owned(), "-k".to_owned()], 133 | ..Default::default() 134 | }; 135 | 136 | let parser = BufferedInputLineParser::new( 137 | &command_line_args, 138 | &RegexProcessor::new(&command_line_args).unwrap(), 139 | ); 140 | 141 | let result = parser.parse_line("file with spaces"); 142 | 143 | assert_eq!( 144 | result, 145 | Some(OwnedCommandAndArgs { 146 | command_path: PathBuf::from("gzip"), 147 | args: vec!["-k", "file with spaces"] 148 | .into_iter() 149 | .map_into() 150 | .collect(), 151 | }) 152 | ); 153 | } 154 | 155 | #[test] 156 | fn test_shell() { 157 | let command_line_args = CommandLineArgs { 158 | null_separator: false, 159 | shell: true, 160 | command_and_initial_arguments: vec![], 161 | shell_path: "/bin/bash".to_owned(), 162 | shell_argument: "-c".to_owned(), 163 | ..Default::default() 164 | }; 165 | 166 | let parser = BufferedInputLineParser::new( 167 | &command_line_args, 168 | &RegexProcessor::new(&command_line_args).unwrap(), 169 | ); 170 | 171 | let result = parser.parse_line("awesomebashfunction 1 2 3"); 172 | 173 | assert_eq!( 174 | result, 175 | Some(OwnedCommandAndArgs { 176 | command_path: PathBuf::from("/bin/bash"), 177 | args: vec!["-c", "awesomebashfunction 1 2 3"] 178 | .into_iter() 179 | .map_into() 180 | .collect(), 181 | }) 182 | ); 183 | 184 | let command_line_args = CommandLineArgs { 185 | null_separator: false, 186 | shell: true, 187 | command_and_initial_arguments: vec![], 188 | shell_path: "/bin/zsh".to_owned(), 189 | shell_argument: "-c".to_owned(), 190 | ..Default::default() 191 | }; 192 | 193 | let parser = BufferedInputLineParser::new( 194 | &command_line_args, 195 | &RegexProcessor::new(&command_line_args).unwrap(), 196 | ); 197 | 198 | let result = parser.parse_line(" awesomebashfunction 1 2 3 "); 199 | 200 | assert_eq!( 201 | result, 202 | Some(OwnedCommandAndArgs { 203 | command_path: PathBuf::from("/bin/zsh"), 204 | args: vec!["-c", "awesomebashfunction 1 2 3"] 205 | .into_iter() 206 | .map_into() 207 | .collect(), 208 | }) 209 | ); 210 | } 211 | 212 | #[test] 213 | fn test_no_run_if_empty() { 214 | let command_line_args = CommandLineArgs { 215 | null_separator: false, 216 | shell: false, 217 | command_and_initial_arguments: vec!["echo".into()], 218 | no_run_if_empty: true, 219 | ..Default::default() 220 | }; 221 | 222 | let parser = BufferedInputLineParser::new( 223 | &command_line_args, 224 | &RegexProcessor::new(&command_line_args).unwrap(), 225 | ); 226 | 227 | let result = parser.parse_line(""); 228 | 229 | assert_eq!(result, None); 230 | 231 | let result = parser.parse_line(" \n\r\t "); 232 | 233 | assert_eq!(result, None); 234 | } 235 | 236 | #[test] 237 | fn test_command_and_initial_arguments() { 238 | let command_line_args = CommandLineArgs { 239 | null_separator: false, 240 | shell: false, 241 | command_and_initial_arguments: vec!["md5".to_owned(), "-s".to_owned()], 242 | ..Default::default() 243 | }; 244 | 245 | let parser = BufferedInputLineParser::new( 246 | &command_line_args, 247 | &RegexProcessor::new(&command_line_args).unwrap(), 248 | ); 249 | 250 | let result = parser.parse_line("stuff"); 251 | 252 | assert_eq!( 253 | result, 254 | Some(OwnedCommandAndArgs { 255 | command_path: PathBuf::from("md5"), 256 | args: vec!["-s", "stuff"].into_iter().map_into().collect(), 257 | }) 258 | ); 259 | 260 | let result = parser.parse_line(" stuff things "); 261 | 262 | assert_eq!( 263 | result, 264 | Some(OwnedCommandAndArgs { 265 | command_path: PathBuf::from("md5"), 266 | args: vec!["-s", "stuff", "things"] 267 | .into_iter() 268 | .map_into() 269 | .collect(), 270 | }) 271 | ); 272 | } 273 | 274 | #[test] 275 | fn test_regex_named_groups() { 276 | let command_line_args = CommandLineArgs { 277 | command_and_initial_arguments: vec![ 278 | "echo".to_owned(), 279 | "got arg1={arg1} arg2={arg2}".to_owned(), 280 | ], 281 | regex: Some("(?P.*),(?P.*)".to_owned()), 282 | ..Default::default() 283 | }; 284 | 285 | let parser = BufferedInputLineParser::new( 286 | &command_line_args, 287 | &RegexProcessor::new(&command_line_args).unwrap(), 288 | ); 289 | 290 | let result = parser.parse_line("foo,bar"); 291 | 292 | assert_eq!( 293 | result, 294 | Some(OwnedCommandAndArgs { 295 | command_path: PathBuf::from("echo"), 296 | args: vec!["got arg1=foo arg2=bar"] 297 | .into_iter() 298 | .map_into() 299 | .collect(), 300 | }) 301 | ); 302 | } 303 | 304 | #[test] 305 | fn test_regex_numbered_groups() { 306 | let command_line_args = CommandLineArgs { 307 | command_and_initial_arguments: vec![ 308 | "echo".to_owned(), 309 | "got arg1={2} arg2={1} arg3={0}".to_owned(), 310 | ], 311 | regex: Some("(.*),(.*)".to_owned()), 312 | ..Default::default() 313 | }; 314 | 315 | let parser = BufferedInputLineParser::new( 316 | &command_line_args, 317 | &RegexProcessor::new(&command_line_args).unwrap(), 318 | ); 319 | 320 | let result = parser.parse_line("foo,bar"); 321 | 322 | assert_eq!( 323 | result, 324 | Some(OwnedCommandAndArgs { 325 | command_path: PathBuf::from("echo"), 326 | args: vec!["got arg1=bar arg2=foo arg3=foo,bar"] 327 | .into_iter() 328 | .map_into() 329 | .collect(), 330 | }) 331 | ); 332 | } 333 | } 334 | -------------------------------------------------------------------------------- /src/parser/command_line.rs: -------------------------------------------------------------------------------- 1 | use itertools::Itertools; 2 | 3 | use std::{collections::VecDeque, sync::Arc}; 4 | 5 | use crate::{ 6 | command_line_args::{COMMANDS_FROM_ARGS_SEPARATOR, CommandLineArgs}, 7 | common::OwnedCommandAndArgs, 8 | parser::{ShellCommandAndArgs, regex::RegexProcessor}, 9 | }; 10 | 11 | #[derive(Debug)] 12 | struct ArgumentGroups { 13 | first_command_and_args: Vec, 14 | all_argument_groups: VecDeque>, 15 | } 16 | 17 | pub struct CommandLineArgsParser { 18 | argument_groups: ArgumentGroups, 19 | shell_command_and_args: ShellCommandAndArgs, 20 | regex_processor: Arc, 21 | } 22 | 23 | impl CommandLineArgsParser { 24 | pub fn new(command_line_args: &CommandLineArgs, regex_processor: &Arc) -> Self { 25 | let argument_groups = Self::build_argument_groups(command_line_args); 26 | 27 | let shell_command_and_args = ShellCommandAndArgs::new(command_line_args); 28 | 29 | Self { 30 | argument_groups, 31 | shell_command_and_args, 32 | regex_processor: Arc::clone(regex_processor), 33 | } 34 | } 35 | 36 | fn build_argument_groups(command_line_args: &CommandLineArgs) -> ArgumentGroups { 37 | let command_and_initial_arguments = &command_line_args.command_and_initial_arguments; 38 | 39 | let mut remaining_argument_groups = Vec::with_capacity(command_and_initial_arguments.len()); 40 | 41 | let mut first = true; 42 | 43 | let mut first_command_and_args = vec![]; 44 | 45 | for (separator, group) in &command_and_initial_arguments 46 | .iter() 47 | .chunk_by(|arg| *arg == COMMANDS_FROM_ARGS_SEPARATOR) 48 | { 49 | let group_vec = group.cloned().collect(); 50 | 51 | if first { 52 | if !separator { 53 | first_command_and_args = group_vec; 54 | } 55 | first = false; 56 | } else if !separator { 57 | remaining_argument_groups.push(group_vec); 58 | } 59 | } 60 | 61 | let all_argument_groups = remaining_argument_groups 62 | .into_iter() 63 | .multi_cartesian_product() 64 | .collect(); 65 | 66 | ArgumentGroups { 67 | first_command_and_args, 68 | all_argument_groups, 69 | } 70 | } 71 | 72 | fn parse_argument_group(&self, argument_group: Vec) -> Option { 73 | let first_command_and_args = &self.argument_groups.first_command_and_args; 74 | 75 | let cmd_and_args = if !self.regex_processor.regex_mode() { 76 | [first_command_and_args.clone(), argument_group].concat() 77 | } else { 78 | let input_line = argument_group.join(" "); 79 | 80 | let apply_regex_result = self 81 | .regex_processor 82 | .apply_regex_to_arguments(first_command_and_args, &input_line)?; 83 | 84 | if apply_regex_result.modified_arguments { 85 | apply_regex_result.arguments 86 | } else { 87 | [first_command_and_args.clone(), argument_group].concat() 88 | } 89 | }; 90 | 91 | super::build_owned_command_and_args(&self.shell_command_and_args, cmd_and_args) 92 | } 93 | 94 | pub fn has_remaining_argument_groups(&self) -> bool { 95 | !self.argument_groups.all_argument_groups.is_empty() 96 | } 97 | 98 | pub fn parse_next_argument_group(&mut self) -> Option { 99 | let argument_group = self.argument_groups.all_argument_groups.pop_front()?; 100 | self.parse_argument_group(argument_group) 101 | } 102 | } 103 | 104 | #[cfg(test)] 105 | mod test { 106 | use super::*; 107 | 108 | use std::{default::Default, path::PathBuf}; 109 | 110 | fn collect_into_vec(mut parser: CommandLineArgsParser) -> Vec { 111 | let mut result = vec![]; 112 | 113 | while parser.has_remaining_argument_groups() { 114 | let Some(cmd_and_args) = parser.parse_next_argument_group() else { 115 | continue; 116 | }; 117 | 118 | result.push(cmd_and_args); 119 | } 120 | 121 | result 122 | } 123 | 124 | #[test] 125 | fn test_parse_command_line_args_with_intial_command() { 126 | let command_line_args = CommandLineArgs { 127 | shell: false, 128 | command_and_initial_arguments: vec![ 129 | "echo", "-n", ":::", "A", "B", ":::", "C", "D", "E", 130 | ] 131 | .into_iter() 132 | .map_into() 133 | .collect(), 134 | ..Default::default() 135 | }; 136 | 137 | let parser = CommandLineArgsParser::new( 138 | &command_line_args, 139 | &RegexProcessor::new(&command_line_args).unwrap(), 140 | ); 141 | 142 | let result = collect_into_vec(parser); 143 | 144 | assert_eq!( 145 | result, 146 | vec![ 147 | OwnedCommandAndArgs { 148 | command_path: PathBuf::from("echo"), 149 | args: vec!["-n", "A", "C"].into_iter().map_into().collect(), 150 | }, 151 | OwnedCommandAndArgs { 152 | command_path: PathBuf::from("echo"), 153 | args: vec!["-n", "A", "D"].into_iter().map_into().collect(), 154 | }, 155 | OwnedCommandAndArgs { 156 | command_path: PathBuf::from("echo"), 157 | args: vec!["-n", "A", "E"].into_iter().map_into().collect(), 158 | }, 159 | OwnedCommandAndArgs { 160 | command_path: PathBuf::from("echo"), 161 | args: vec!["-n", "B", "C"].into_iter().map_into().collect(), 162 | }, 163 | OwnedCommandAndArgs { 164 | command_path: PathBuf::from("echo"), 165 | args: vec!["-n", "B", "D"].into_iter().map_into().collect(), 166 | }, 167 | OwnedCommandAndArgs { 168 | command_path: PathBuf::from("echo"), 169 | args: vec!["-n", "B", "E"].into_iter().map_into().collect(), 170 | }, 171 | ] 172 | ); 173 | } 174 | 175 | #[test] 176 | fn test_parse_command_line_args_no_intial_command() { 177 | let command_line_args = CommandLineArgs { 178 | shell: false, 179 | command_and_initial_arguments: vec![ 180 | ":::", "echo", "say", ":::", "arg1", "arg2", "arg3", 181 | ] 182 | .into_iter() 183 | .map_into() 184 | .collect(), 185 | ..Default::default() 186 | }; 187 | 188 | let parser = CommandLineArgsParser::new( 189 | &command_line_args, 190 | &RegexProcessor::new(&command_line_args).unwrap(), 191 | ); 192 | 193 | let result = collect_into_vec(parser); 194 | 195 | assert_eq!( 196 | result, 197 | vec![ 198 | OwnedCommandAndArgs { 199 | command_path: PathBuf::from("echo"), 200 | args: vec!["arg1"].into_iter().map_into().collect(), 201 | }, 202 | OwnedCommandAndArgs { 203 | command_path: PathBuf::from("echo"), 204 | args: vec!["arg2"].into_iter().map_into().collect(), 205 | }, 206 | OwnedCommandAndArgs { 207 | command_path: PathBuf::from("echo"), 208 | args: vec!["arg3"].into_iter().map_into().collect(), 209 | }, 210 | OwnedCommandAndArgs { 211 | command_path: PathBuf::from("say"), 212 | args: vec!["arg1"].into_iter().map_into().collect(), 213 | }, 214 | OwnedCommandAndArgs { 215 | command_path: PathBuf::from("say"), 216 | args: vec!["arg2"].into_iter().map_into().collect(), 217 | }, 218 | OwnedCommandAndArgs { 219 | command_path: PathBuf::from("say"), 220 | args: vec!["arg3"].into_iter().map_into().collect(), 221 | }, 222 | ] 223 | ); 224 | } 225 | 226 | #[test] 227 | fn test_parse_command_line_args_empty() { 228 | let command_line_args = CommandLineArgs { 229 | shell: false, 230 | command_and_initial_arguments: vec![], 231 | ..Default::default() 232 | }; 233 | 234 | let parser = CommandLineArgsParser::new( 235 | &command_line_args, 236 | &RegexProcessor::new(&command_line_args).unwrap(), 237 | ); 238 | 239 | let result = collect_into_vec(parser); 240 | 241 | assert_eq!(result, vec![]); 242 | } 243 | 244 | #[test] 245 | fn test_parse_command_line_args_invalid() { 246 | let command_line_args = CommandLineArgs { 247 | shell: false, 248 | command_and_initial_arguments: vec![":::", ":::"].into_iter().map_into().collect(), 249 | ..Default::default() 250 | }; 251 | 252 | let parser = CommandLineArgsParser::new( 253 | &command_line_args, 254 | &RegexProcessor::new(&command_line_args).unwrap(), 255 | ); 256 | 257 | let result = collect_into_vec(parser); 258 | 259 | assert_eq!(result, vec![]); 260 | } 261 | 262 | #[test] 263 | fn test_parse_command_line_args_shell_mode_with_initial_command() { 264 | let command_line_args = CommandLineArgs { 265 | shell: true, 266 | command_and_initial_arguments: vec![ 267 | "echo", "-n", ":::", "A", "B", ":::", "C", "D", "E", 268 | ] 269 | .into_iter() 270 | .map_into() 271 | .collect(), 272 | shell_path: "/bin/bash".to_owned(), 273 | shell_argument: "-c".to_owned(), 274 | ..Default::default() 275 | }; 276 | 277 | let parser = CommandLineArgsParser::new( 278 | &command_line_args, 279 | &RegexProcessor::new(&command_line_args).unwrap(), 280 | ); 281 | 282 | let result = collect_into_vec(parser); 283 | 284 | assert_eq!( 285 | result, 286 | vec![ 287 | OwnedCommandAndArgs { 288 | command_path: PathBuf::from("/bin/bash"), 289 | args: vec!["-c", "echo -n A C"].into_iter().map_into().collect(), 290 | }, 291 | OwnedCommandAndArgs { 292 | command_path: PathBuf::from("/bin/bash"), 293 | args: vec!["-c", "echo -n A D"].into_iter().map_into().collect(), 294 | }, 295 | OwnedCommandAndArgs { 296 | command_path: PathBuf::from("/bin/bash"), 297 | args: vec!["-c", "echo -n A E"].into_iter().map_into().collect(), 298 | }, 299 | OwnedCommandAndArgs { 300 | command_path: PathBuf::from("/bin/bash"), 301 | args: vec!["-c", "echo -n B C"].into_iter().map_into().collect(), 302 | }, 303 | OwnedCommandAndArgs { 304 | command_path: PathBuf::from("/bin/bash"), 305 | args: vec!["-c", "echo -n B D"].into_iter().map_into().collect(), 306 | }, 307 | OwnedCommandAndArgs { 308 | command_path: PathBuf::from("/bin/bash"), 309 | args: vec!["-c", "echo -n B E"].into_iter().map_into().collect(), 310 | }, 311 | ] 312 | ); 313 | } 314 | 315 | #[test] 316 | fn test_parse_command_line_args_shell_mode_no_initial_command() { 317 | let command_line_args = CommandLineArgs { 318 | shell: true, 319 | command_and_initial_arguments: vec![":::", "say", "echo", ":::", "C", "D", "E"] 320 | .into_iter() 321 | .map_into() 322 | .collect(), 323 | shell_path: "/bin/bash".to_owned(), 324 | shell_argument: "-c".to_owned(), 325 | ..Default::default() 326 | }; 327 | 328 | let parser = CommandLineArgsParser::new( 329 | &command_line_args, 330 | &RegexProcessor::new(&command_line_args).unwrap(), 331 | ); 332 | 333 | let result = collect_into_vec(parser); 334 | 335 | assert_eq!( 336 | result, 337 | vec![ 338 | OwnedCommandAndArgs { 339 | command_path: PathBuf::from("/bin/bash"), 340 | args: vec!["-c", "say C"].into_iter().map_into().collect(), 341 | }, 342 | OwnedCommandAndArgs { 343 | command_path: PathBuf::from("/bin/bash"), 344 | args: vec!["-c", "say D"].into_iter().map_into().collect(), 345 | }, 346 | OwnedCommandAndArgs { 347 | command_path: PathBuf::from("/bin/bash"), 348 | args: vec!["-c", "say E"].into_iter().map_into().collect(), 349 | }, 350 | OwnedCommandAndArgs { 351 | command_path: PathBuf::from("/bin/bash"), 352 | args: vec!["-c", "echo C"].into_iter().map_into().collect(), 353 | }, 354 | OwnedCommandAndArgs { 355 | command_path: PathBuf::from("/bin/bash"), 356 | args: vec!["-c", "echo D"].into_iter().map_into().collect(), 357 | }, 358 | OwnedCommandAndArgs { 359 | command_path: PathBuf::from("/bin/bash"), 360 | args: vec!["-c", "echo E"].into_iter().map_into().collect(), 361 | }, 362 | ] 363 | ); 364 | } 365 | 366 | #[test] 367 | fn test_regex_named_groups() { 368 | let command_line_args = CommandLineArgs { 369 | command_and_initial_arguments: vec![ 370 | "echo", 371 | "got", 372 | "arg1={arg1}", 373 | "arg2={arg2}", 374 | "arg3={arg3}", 375 | ":::", 376 | "foo,bar,baz", 377 | "foo2,bar2,baz2", 378 | ] 379 | .into_iter() 380 | .map_into() 381 | .collect(), 382 | regex: Some("(?P.*),(?P.*),(?P.*)".to_owned()), 383 | ..Default::default() 384 | }; 385 | 386 | let parser = CommandLineArgsParser::new( 387 | &command_line_args, 388 | &RegexProcessor::new(&command_line_args).unwrap(), 389 | ); 390 | 391 | let result = collect_into_vec(parser); 392 | 393 | assert_eq!( 394 | result, 395 | vec![ 396 | OwnedCommandAndArgs { 397 | command_path: PathBuf::from("echo"), 398 | args: vec!["got", "arg1=foo", "arg2=bar", "arg3=baz"] 399 | .into_iter() 400 | .map_into() 401 | .collect(), 402 | }, 403 | OwnedCommandAndArgs { 404 | command_path: PathBuf::from("echo"), 405 | args: vec!["got", "arg1=foo2", "arg2=bar2", "arg3=baz2"] 406 | .into_iter() 407 | .map_into() 408 | .collect(), 409 | }, 410 | ] 411 | ); 412 | } 413 | 414 | #[test] 415 | fn test_regex_numbered_groups() { 416 | let command_line_args = CommandLineArgs { 417 | command_and_initial_arguments: vec![ 418 | "echo", 419 | "got", 420 | "arg1={0}", 421 | "arg2={1}", 422 | "arg3={2}", 423 | ":::", 424 | "foo,bar,baz", 425 | "foo2,bar2,baz2", 426 | ] 427 | .into_iter() 428 | .map_into() 429 | .collect(), 430 | regex: Some("(.*),(.*),(.*)".to_owned()), 431 | ..Default::default() 432 | }; 433 | 434 | let parser = CommandLineArgsParser::new( 435 | &command_line_args, 436 | &RegexProcessor::new(&command_line_args).unwrap(), 437 | ); 438 | 439 | let result = collect_into_vec(parser); 440 | 441 | assert_eq!( 442 | result, 443 | vec![ 444 | OwnedCommandAndArgs { 445 | command_path: PathBuf::from("echo"), 446 | args: vec!["got", "arg1=foo,bar,baz", "arg2=foo", "arg3=bar"] 447 | .into_iter() 448 | .map_into() 449 | .collect(), 450 | }, 451 | OwnedCommandAndArgs { 452 | command_path: PathBuf::from("echo"), 453 | args: vec!["got", "arg1=foo2,bar2,baz2", "arg2=foo2", "arg3=bar2"] 454 | .into_iter() 455 | .map_into() 456 | .collect(), 457 | }, 458 | ] 459 | ); 460 | } 461 | 462 | #[test] 463 | fn test_auto_regex() { 464 | let command_line_args = CommandLineArgs { 465 | command_and_initial_arguments: [ 466 | "echo", "got", "arg1={1}", "arg2={2}", ":::", "foo", "bar", ":::", "baz", "qux", 467 | ] 468 | .into_iter() 469 | .map_into() 470 | .collect(), 471 | ..Default::default() 472 | }; 473 | 474 | let parser = CommandLineArgsParser::new( 475 | &command_line_args, 476 | &RegexProcessor::new(&command_line_args).unwrap(), 477 | ); 478 | 479 | let result = collect_into_vec(parser); 480 | 481 | assert_eq!( 482 | result, 483 | vec![ 484 | OwnedCommandAndArgs { 485 | command_path: PathBuf::from("echo"), 486 | args: ["got", "arg1=foo", "arg2=baz"] 487 | .into_iter() 488 | .map_into() 489 | .collect(), 490 | }, 491 | OwnedCommandAndArgs { 492 | command_path: PathBuf::from("echo"), 493 | args: ["got", "arg1=foo", "arg2=qux"] 494 | .into_iter() 495 | .map_into() 496 | .collect(), 497 | }, 498 | OwnedCommandAndArgs { 499 | command_path: PathBuf::from("echo"), 500 | args: ["got", "arg1=bar", "arg2=baz"] 501 | .into_iter() 502 | .map_into() 503 | .collect(), 504 | }, 505 | OwnedCommandAndArgs { 506 | command_path: PathBuf::from("echo"), 507 | args: ["got", "arg1=bar", "arg2=qux"] 508 | .into_iter() 509 | .map_into() 510 | .collect(), 511 | }, 512 | ] 513 | ); 514 | } 515 | } 516 | -------------------------------------------------------------------------------- /src/parser/regex.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Context; 2 | 3 | use itertools::Itertools; 4 | 5 | use tracing::warn; 6 | 7 | use std::{borrow::Cow, sync::Arc}; 8 | 9 | use crate::command_line_args::{COMMANDS_FROM_ARGS_SEPARATOR, CommandLineArgs}; 10 | 11 | #[derive(Debug, Eq, PartialEq)] 12 | pub struct ApplyRegexToArgumentsResult { 13 | pub arguments: Vec, 14 | pub modified_arguments: bool, 15 | } 16 | 17 | pub struct RegexProcessor { 18 | command_line_regex: Option, 19 | } 20 | 21 | impl RegexProcessor { 22 | pub fn new(command_line_args: &CommandLineArgs) -> anyhow::Result> { 23 | let auto_regex = AutoCommandLineArgsRegex::new(command_line_args); 24 | 25 | let command_line_regex = match (auto_regex, &command_line_args.regex) { 26 | (Some(auto_regex), _) => Some(CommandLineRegex::new(&auto_regex.0)?), 27 | (_, Some(cla_regex)) => Some(CommandLineRegex::new(cla_regex)?), 28 | _ => None, 29 | }; 30 | 31 | Ok(Arc::new(Self { command_line_regex })) 32 | } 33 | 34 | pub fn regex_mode(&self) -> bool { 35 | self.command_line_regex.is_some() 36 | } 37 | 38 | pub fn apply_regex_to_arguments( 39 | &self, 40 | arguments: &Vec, 41 | input_data: &str, 42 | ) -> Option { 43 | let command_line_regex = self.command_line_regex.as_ref()?; 44 | 45 | let mut results: Vec = Vec::with_capacity(arguments.len()); 46 | let mut found_input_data_match = false; 47 | let mut modified_arguments = false; 48 | 49 | for argument in arguments { 50 | match command_line_regex.expand(argument.into(), input_data) { 51 | Ok(result) => { 52 | results.push(result.argument.to_string()); 53 | found_input_data_match = true; 54 | modified_arguments = modified_arguments || result.modified_argument; 55 | } 56 | Err(ExpandError::RegexDoesNotMatchInputData) => { 57 | results.push(argument.clone()); 58 | } 59 | }; 60 | } 61 | 62 | if !found_input_data_match { 63 | warn!("regex did not match input data: {}", input_data); 64 | None 65 | } else { 66 | Some(ApplyRegexToArgumentsResult { 67 | arguments: results, 68 | modified_arguments, 69 | }) 70 | } 71 | } 72 | } 73 | 74 | #[derive(Debug)] 75 | struct ExpandResult<'a> { 76 | argument: Cow<'a, str>, 77 | modified_argument: bool, 78 | } 79 | 80 | #[derive(thiserror::Error, Debug)] 81 | enum ExpandError { 82 | #[error("regex does not match input data")] 83 | RegexDoesNotMatchInputData, 84 | } 85 | 86 | struct CommandLineRegex { 87 | regex: regex::Regex, 88 | numbered_group_match_keys: Vec, 89 | named_group_to_match_key: Vec<(String, String)>, 90 | } 91 | 92 | impl CommandLineRegex { 93 | fn new(command_line_args_regex: &str) -> anyhow::Result { 94 | let regex = regex::Regex::new(command_line_args_regex) 95 | .context("CommandLineRegex::new: error creating regex")?; 96 | 97 | let capture_names = regex.capture_names(); 98 | 99 | let mut numbered_group_match_keys = Vec::with_capacity(capture_names.len()); 100 | 101 | let mut named_group_to_match_key = Vec::with_capacity(capture_names.len()); 102 | 103 | for (i, capture_name_option) in capture_names.enumerate() { 104 | let match_key = format!("{{{}}}", i); 105 | numbered_group_match_keys.push(match_key); 106 | 107 | if let Some(capture_name) = capture_name_option { 108 | let match_key = format!("{{{}}}", capture_name); 109 | named_group_to_match_key.push((capture_name.to_owned(), match_key)); 110 | } 111 | } 112 | 113 | Ok(Self { 114 | regex, 115 | numbered_group_match_keys, 116 | named_group_to_match_key, 117 | }) 118 | } 119 | 120 | fn expand<'a>( 121 | &self, 122 | argument: Cow<'a, str>, 123 | input_data: &str, 124 | ) -> Result, ExpandError> { 125 | let captures = self 126 | .regex 127 | .captures(input_data) 128 | .ok_or(ExpandError::RegexDoesNotMatchInputData)?; 129 | 130 | let mut argument = argument; 131 | let mut modified_argument = false; 132 | 133 | let mut update_argument = |match_key, match_value| { 134 | if argument.contains(match_key) { 135 | argument = Cow::from(argument.replace(match_key, match_value)); 136 | modified_argument = true; 137 | } 138 | }; 139 | 140 | // numbered capture groups 141 | for (i, match_option) in captures.iter().enumerate() { 142 | if let (Some(match_value), Some(match_key)) = 143 | (match_option, self.numbered_group_match_keys.get(i)) 144 | { 145 | // make {} have the same behavior as {0} 146 | if i == 0 { 147 | update_argument("{}", match_value.as_str()); 148 | } 149 | update_argument(match_key, match_value.as_str()); 150 | } 151 | } 152 | 153 | // named capture groups 154 | for (group_name, match_key) in self.named_group_to_match_key.iter() { 155 | if let Some(match_value) = captures.name(group_name) { 156 | update_argument(match_key, match_value.as_str()); 157 | } 158 | } 159 | 160 | Ok(ExpandResult { 161 | argument, 162 | modified_argument, 163 | }) 164 | } 165 | } 166 | 167 | #[derive(Debug)] 168 | struct AutoCommandLineArgsRegex(String); 169 | 170 | impl AutoCommandLineArgsRegex { 171 | fn new(command_line_args: &CommandLineArgs) -> Option { 172 | if command_line_args.regex.is_none() && command_line_args.commands_from_args_mode() { 173 | Self::new_auto_interpolate_commands_from_args(command_line_args) 174 | } else { 175 | None 176 | } 177 | } 178 | 179 | fn new_auto_interpolate_commands_from_args( 180 | command_line_args: &CommandLineArgs, 181 | ) -> Option { 182 | let mut first = true; 183 | let mut argument_group_count = 0; 184 | 185 | for (separator, _group) in &command_line_args 186 | .command_and_initial_arguments 187 | .iter() 188 | .chunk_by(|arg| *arg == COMMANDS_FROM_ARGS_SEPARATOR) 189 | { 190 | if first { 191 | if separator { 192 | return None; 193 | } 194 | first = false; 195 | } else if !separator { 196 | argument_group_count += 1; 197 | } 198 | } 199 | 200 | let argument_group_count = argument_group_count; 201 | 202 | let mut generated_regex = String::with_capacity(argument_group_count * 5); 203 | 204 | for i in 0..argument_group_count { 205 | if i != 0 { 206 | generated_regex.push(' '); 207 | } 208 | generated_regex.push_str("(.*)"); 209 | } 210 | 211 | Some(Self(generated_regex)) 212 | } 213 | } 214 | 215 | #[cfg(test)] 216 | mod test { 217 | use super::*; 218 | 219 | #[test] 220 | fn test_regex_disabled() { 221 | let command_line_args = CommandLineArgs { 222 | regex: None, 223 | ..Default::default() 224 | }; 225 | 226 | let regex_processor = RegexProcessor::new(&command_line_args).unwrap(); 227 | 228 | assert_eq!(regex_processor.regex_mode(), false); 229 | 230 | let arguments = vec!["{0}".to_string()]; 231 | assert_eq!( 232 | regex_processor.apply_regex_to_arguments(&arguments, "input line"), 233 | None, 234 | ); 235 | } 236 | 237 | #[test] 238 | fn test_regex_numbered_groups() { 239 | let command_line_args = CommandLineArgs { 240 | regex: Some("(.*),(.*)".to_string()), 241 | ..Default::default() 242 | }; 243 | 244 | let regex_processor = RegexProcessor::new(&command_line_args).unwrap(); 245 | 246 | assert_eq!(regex_processor.regex_mode(), true); 247 | 248 | let arguments = vec!["{1} {2}".to_string()]; 249 | assert_eq!( 250 | regex_processor.apply_regex_to_arguments(&arguments, "hello,world"), 251 | Some(ApplyRegexToArgumentsResult { 252 | arguments: vec!["hello world".to_string()], 253 | modified_arguments: true, 254 | }) 255 | ); 256 | } 257 | 258 | #[test] 259 | fn test_regex_named_groups() { 260 | let command_line_args = CommandLineArgs { 261 | regex: Some("(?P.*),(?P.*)".to_string()), 262 | ..Default::default() 263 | }; 264 | 265 | let regex_processor = RegexProcessor::new(&command_line_args).unwrap(); 266 | 267 | assert_eq!(regex_processor.regex_mode(), true); 268 | 269 | let arguments = vec!["{arg1} {arg2}".to_string()]; 270 | assert_eq!( 271 | regex_processor.apply_regex_to_arguments(&arguments, "hello,world"), 272 | Some(ApplyRegexToArgumentsResult { 273 | arguments: vec!["hello world".to_string()], 274 | modified_arguments: true, 275 | }) 276 | ); 277 | } 278 | 279 | #[test] 280 | fn test_regex_numbered_groups_json() { 281 | let command_line_args = CommandLineArgs { 282 | regex: Some("(.*),(.*)".to_string()), 283 | ..Default::default() 284 | }; 285 | 286 | let regex_processor = RegexProcessor::new(&command_line_args).unwrap(); 287 | 288 | assert_eq!(regex_processor.regex_mode(), true); 289 | 290 | let arguments = 291 | vec![r#"{"id": 123, "$zero": "{0}", "one": "{1}", "two": "{2}"}"#.to_string()]; 292 | assert_eq!( 293 | regex_processor.apply_regex_to_arguments(&arguments, "hello,world",), 294 | Some(ApplyRegexToArgumentsResult { 295 | arguments: vec![ 296 | r#"{"id": 123, "$zero": "hello,world", "one": "hello", "two": "world"}"# 297 | .to_string(), 298 | ], 299 | modified_arguments: true, 300 | }) 301 | ); 302 | } 303 | 304 | #[test] 305 | fn test_regex_numbered_groups_json_empty_group() { 306 | let command_line_args = CommandLineArgs { 307 | regex: Some("(.*),(.*)".to_string()), 308 | ..Default::default() 309 | }; 310 | 311 | let regex_processor = RegexProcessor::new(&command_line_args).unwrap(); 312 | 313 | assert_eq!(regex_processor.regex_mode(), true); 314 | 315 | let arguments = 316 | vec![r#"{"id": 123, "$zero": "{}", "one": "{1}", "two": "{2}"}"#.to_string()]; 317 | assert_eq!( 318 | regex_processor.apply_regex_to_arguments(&arguments, "hello,world",), 319 | Some(ApplyRegexToArgumentsResult { 320 | arguments: vec![ 321 | r#"{"id": 123, "$zero": "hello,world", "one": "hello", "two": "world"}"# 322 | .to_string(), 323 | ], 324 | modified_arguments: true, 325 | }) 326 | ); 327 | } 328 | 329 | #[test] 330 | fn test_regex_named_groups_json() { 331 | let command_line_args = CommandLineArgs { 332 | regex: Some("(?P.*),(?P.*)".to_string()), 333 | ..Default::default() 334 | }; 335 | 336 | let regex_processor = RegexProcessor::new(&command_line_args).unwrap(); 337 | 338 | assert_eq!(regex_processor.regex_mode(), true); 339 | 340 | let arguments = 341 | vec![r#"{"id": 123, "$zero": "{0}", "one": "{arg1}", "two": "{arg2}"}"#.to_string()]; 342 | assert_eq!( 343 | regex_processor.apply_regex_to_arguments(&arguments, "hello,world",), 344 | Some(ApplyRegexToArgumentsResult { 345 | arguments: vec![ 346 | r#"{"id": 123, "$zero": "hello,world", "one": "hello", "two": "world"}"# 347 | .to_string() 348 | ], 349 | modified_arguments: true, 350 | }) 351 | ); 352 | } 353 | 354 | #[test] 355 | fn test_regex_named_groups_json_empty_group() { 356 | let command_line_args = CommandLineArgs { 357 | regex: Some("(?P.*),(?P.*)".to_string()), 358 | ..Default::default() 359 | }; 360 | 361 | let regex_processor = RegexProcessor::new(&command_line_args).unwrap(); 362 | 363 | assert_eq!(regex_processor.regex_mode(), true); 364 | 365 | let arguments = 366 | vec![r#"{"id": 123, "$zero": "{}", "one": "{arg1}", "two": "{arg2}"}"#.to_string()]; 367 | assert_eq!( 368 | regex_processor.apply_regex_to_arguments(&arguments, "hello,world",), 369 | Some(ApplyRegexToArgumentsResult { 370 | arguments: vec![ 371 | r#"{"id": 123, "$zero": "hello,world", "one": "hello", "two": "world"}"# 372 | .to_string() 373 | ], 374 | modified_arguments: true, 375 | }) 376 | ); 377 | } 378 | 379 | #[test] 380 | fn test_regex_string_containing_dollar_curly_brace_variable() { 381 | let command_line_args = CommandLineArgs { 382 | regex: Some("(?P.*),(?P.*)".to_string()), 383 | ..Default::default() 384 | }; 385 | 386 | let regex_processor = RegexProcessor::new(&command_line_args).unwrap(); 387 | 388 | assert_eq!(regex_processor.regex_mode(), true); 389 | 390 | let arguments = vec![r#"{arg2}${FOO}{arg1}$BAR${BAR}{arg2}"#.to_string()]; 391 | assert_eq!( 392 | regex_processor.apply_regex_to_arguments(&arguments, "hello,world"), 393 | Some(ApplyRegexToArgumentsResult { 394 | arguments: vec![r#"world${FOO}hello$BAR${BAR}world"#.to_string()], 395 | modified_arguments: true, 396 | }) 397 | ); 398 | } 399 | 400 | #[test] 401 | fn test_regex_not_matching_input_data() { 402 | let command_line_args = CommandLineArgs { 403 | regex: Some("(?P.*),(?P.*)".to_string()), 404 | ..Default::default() 405 | }; 406 | 407 | let regex_processor = RegexProcessor::new(&command_line_args).unwrap(); 408 | 409 | assert_eq!(regex_processor.regex_mode(), true); 410 | 411 | let arguments = vec!["{arg2},{arg1}".to_string()]; 412 | assert_eq!( 413 | regex_processor.apply_regex_to_arguments(&arguments, "hello,world"), 414 | Some(ApplyRegexToArgumentsResult { 415 | arguments: vec!["world,hello".to_string()], 416 | modified_arguments: true, 417 | }), 418 | ); 419 | 420 | assert_eq!( 421 | regex_processor.apply_regex_to_arguments(&arguments, "hello world"), 422 | None, 423 | ); 424 | } 425 | 426 | #[test] 427 | fn test_regex_invalid() { 428 | let command_line_args = CommandLineArgs { 429 | regex: Some("(?Parg1>.*),(?P.*)".to_string()), 430 | ..Default::default() 431 | }; 432 | 433 | let result = RegexProcessor::new(&command_line_args); 434 | 435 | assert!(result.is_err()); 436 | } 437 | 438 | #[test] 439 | fn test_auto_regex_command_line_regex() { 440 | let command_line_args = CommandLineArgs { 441 | regex: Some("(?Parg1>.*),(?P.*)".to_string()), 442 | ..Default::default() 443 | }; 444 | 445 | let auto_regex = AutoCommandLineArgsRegex::new(&command_line_args); 446 | 447 | assert!(auto_regex.is_none()); 448 | } 449 | 450 | #[test] 451 | fn test_auto_regex_not_command_line_args_mode() { 452 | let command_line_args = CommandLineArgs { 453 | regex: None, 454 | command_and_initial_arguments: ["echo"].into_iter().map_into().collect(), 455 | ..Default::default() 456 | }; 457 | 458 | let auto_regex = AutoCommandLineArgsRegex::new(&command_line_args); 459 | 460 | assert!(auto_regex.is_none()); 461 | } 462 | 463 | #[test] 464 | fn test_auto_regex() { 465 | let command_line_args = CommandLineArgs { 466 | regex: None, 467 | command_and_initial_arguments: ["echo", ":::", "A", "B", ":::", "C", "D"] 468 | .into_iter() 469 | .map_into() 470 | .collect(), 471 | ..Default::default() 472 | }; 473 | 474 | let auto_regex = AutoCommandLineArgsRegex::new(&command_line_args); 475 | 476 | assert!(auto_regex.is_some()); 477 | assert_eq!(auto_regex.unwrap().0, "(.*) (.*)"); 478 | } 479 | } 480 | -------------------------------------------------------------------------------- /src/process.rs: -------------------------------------------------------------------------------- 1 | use tokio::{ 2 | process::{Child, Command}, 3 | time::Duration, 4 | }; 5 | 6 | use std::{ 7 | ffi::OsStr, 8 | process::{Output, Stdio}, 9 | }; 10 | 11 | use crate::command_line_args::{CommandLineArgs, DiscardOutput}; 12 | 13 | #[derive(thiserror::Error, Debug)] 14 | pub enum ChildProcessExecutionError { 15 | #[error("timeout: {0}")] 16 | Timeout(#[from] tokio::time::error::Elapsed), 17 | 18 | #[error("i/o error: {0}")] 19 | IOError(#[from] std::io::Error), 20 | } 21 | 22 | #[derive(Debug)] 23 | pub struct ChildProcess { 24 | child: Child, 25 | discard_all_output: bool, 26 | timeout: Option, 27 | } 28 | 29 | impl ChildProcess { 30 | pub fn id(&self) -> Option { 31 | self.child.id() 32 | } 33 | 34 | async fn await_output(mut self) -> Result { 35 | let output = if self.discard_all_output { 36 | Output { 37 | status: self.child.wait().await?, 38 | stdout: vec![], 39 | stderr: vec![], 40 | } 41 | } else { 42 | self.child.wait_with_output().await? 43 | }; 44 | 45 | Ok(output) 46 | } 47 | 48 | pub async fn await_completion(self) -> Result { 49 | match self.timeout { 50 | None => self.await_output().await, 51 | Some(timeout) => { 52 | let result = tokio::time::timeout(timeout, self.await_output()).await?; 53 | 54 | let output = result?; 55 | 56 | Ok(output) 57 | } 58 | } 59 | } 60 | } 61 | 62 | #[derive(Debug)] 63 | pub struct ChildProcessFactory { 64 | discard_stdout: bool, 65 | discard_stderr: bool, 66 | timeout: Option, 67 | } 68 | 69 | impl ChildProcessFactory { 70 | pub fn new(command_line_args: &CommandLineArgs) -> Self { 71 | Self { 72 | discard_stdout: matches!( 73 | command_line_args.discard_output, 74 | Some(DiscardOutput::All) | Some(DiscardOutput::Stdout) 75 | ), 76 | discard_stderr: matches!( 77 | command_line_args.discard_output, 78 | Some(DiscardOutput::All) | Some(DiscardOutput::Stderr) 79 | ), 80 | timeout: command_line_args 81 | .timeout_seconds 82 | .map(Duration::from_secs_f64), 83 | } 84 | } 85 | 86 | fn stdout(&self) -> Stdio { 87 | if self.discard_stdout { 88 | Stdio::null() 89 | } else { 90 | Stdio::piped() 91 | } 92 | } 93 | 94 | fn stderr(&self) -> Stdio { 95 | if self.discard_stderr { 96 | Stdio::null() 97 | } else { 98 | Stdio::piped() 99 | } 100 | } 101 | 102 | fn discard_all_output(&self) -> bool { 103 | self.discard_stdout && self.discard_stderr 104 | } 105 | 106 | pub async fn spawn(&self, command: C, args: AI) -> std::io::Result 107 | where 108 | C: AsRef, 109 | AI: IntoIterator, 110 | A: AsRef, 111 | { 112 | let child = Command::new(command) 113 | .args(args) 114 | .stdin(Stdio::null()) 115 | .stdout(self.stdout()) 116 | .stderr(self.stderr()) 117 | .kill_on_drop(self.timeout.is_some()) 118 | .spawn()?; 119 | 120 | Ok(ChildProcess { 121 | child, 122 | discard_all_output: self.discard_all_output(), 123 | timeout: self.timeout, 124 | }) 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/progress.rs: -------------------------------------------------------------------------------- 1 | mod style; 2 | 3 | use indicatif::ProgressBar; 4 | 5 | use tokio::time::Duration; 6 | 7 | use std::sync::Arc; 8 | 9 | use crate::command_line_args::CommandLineArgs; 10 | 11 | pub struct Progress { 12 | progress_bar: Option, 13 | } 14 | 15 | impl Progress { 16 | pub fn new(command_line_args: &CommandLineArgs) -> anyhow::Result> { 17 | let progress_bar = if !command_line_args.progress_bar { 18 | None 19 | } else { 20 | let style_info = style::choose_progress_style(command_line_args)?; 21 | 22 | let progress_bar = ProgressBar::new(0); 23 | if style_info.enable_steady_tick { 24 | progress_bar.enable_steady_tick(Duration::from_millis(100)); 25 | } 26 | 27 | progress_bar.set_style(style_info.progress_style); 28 | 29 | Some(progress_bar) 30 | }; 31 | 32 | Ok(Arc::new(Self { progress_bar })) 33 | } 34 | 35 | pub fn increment_total_commands(&self, delta: usize) { 36 | if let Some(progress_bar) = &self.progress_bar { 37 | progress_bar.inc_length(delta.try_into().unwrap_or_default()); 38 | } 39 | } 40 | 41 | pub fn command_finished(&self) { 42 | if let Some(progress_bar) = &self.progress_bar { 43 | progress_bar.inc(1); 44 | } 45 | } 46 | 47 | pub fn finish(&self) { 48 | if let Some(progress_bar) = &self.progress_bar { 49 | progress_bar.finish(); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/progress/style.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Context; 2 | 3 | use indicatif::ProgressStyle; 4 | 5 | use crate::command_line_args::CommandLineArgs; 6 | 7 | const DEFAULT_PROGRESS_STYLE: &str = "default"; 8 | 9 | const SIMPLE_PROGRESS_STYLE: &str = "simple"; 10 | 11 | const SIMPLE_PROGRESS_STYLE_TEMPLATE: &str = 12 | "[{elapsed_precise}] Commands Done/Total: {pos:>2}/{len:2} {wide_bar} ETA {eta_precise}"; 13 | 14 | const LIGHT_BG_PROGRESS_STYLE: &str = "light_bg"; 15 | 16 | const LIGHT_BG_PROGRESS_STYLE_TEMPLATE: &str = "{spinner:.blue.bold} [{elapsed_precise}] Commands Done/Total: {pos:>2}/{len:2} [{wide_bar:.blue.bold/red}] ETA {eta_precise}"; 17 | 18 | const DARK_BG_PROGRESS_STYLE: &str = "dark_bg"; 19 | 20 | const DARK_BG_PROGRESS_STYLE_TEMPLATE: &str = "{spinner:.cyan.bold} [{elapsed_precise}] Commands Done/Total: {pos:>2}/{len:2} [{wide_bar:.cyan.bold/blue}] ETA {eta_precise}"; 21 | 22 | pub struct ProgressStyleInfo { 23 | _style_name: &'static str, 24 | pub progress_style: ProgressStyle, 25 | pub enable_steady_tick: bool, 26 | } 27 | 28 | pub fn choose_progress_style( 29 | command_line_args: &CommandLineArgs, 30 | ) -> anyhow::Result { 31 | let setting = match command_line_args.progress_bar_style { 32 | None => DEFAULT_PROGRESS_STYLE, 33 | Some(ref style) => style, 34 | }; 35 | 36 | match setting { 37 | SIMPLE_PROGRESS_STYLE => Ok(ProgressStyleInfo { 38 | _style_name: SIMPLE_PROGRESS_STYLE, 39 | progress_style: ProgressStyle::with_template(SIMPLE_PROGRESS_STYLE_TEMPLATE) 40 | .context("ProgressStyle::with_template error")?, 41 | enable_steady_tick: false, 42 | }), 43 | LIGHT_BG_PROGRESS_STYLE | DEFAULT_PROGRESS_STYLE => Ok(ProgressStyleInfo { 44 | _style_name: LIGHT_BG_PROGRESS_STYLE, 45 | progress_style: ProgressStyle::with_template(LIGHT_BG_PROGRESS_STYLE_TEMPLATE) 46 | .context("ProgressStyle::with_template error")? 47 | .progress_chars("#>-"), 48 | enable_steady_tick: true, 49 | }), 50 | DARK_BG_PROGRESS_STYLE => Ok(ProgressStyleInfo { 51 | _style_name: DARK_BG_PROGRESS_STYLE, 52 | progress_style: ProgressStyle::with_template(DARK_BG_PROGRESS_STYLE_TEMPLATE) 53 | .context("ProgressStyle::with_template error")? 54 | .progress_chars("#>-"), 55 | enable_steady_tick: true, 56 | }), 57 | _ => anyhow::bail!("unknown PROGRESS_STYLE: {}", setting), 58 | } 59 | } 60 | 61 | #[cfg(test)] 62 | mod test { 63 | use super::*; 64 | 65 | #[test] 66 | fn test_choose_progress_style_no_style_specified() { 67 | let command_line_args = CommandLineArgs { 68 | ..Default::default() 69 | }; 70 | 71 | let result = choose_progress_style(&command_line_args); 72 | assert_eq!(result.is_err(), false); 73 | let result = result.unwrap(); 74 | assert_eq!(result._style_name, LIGHT_BG_PROGRESS_STYLE); 75 | assert_eq!(result.enable_steady_tick, true); 76 | } 77 | 78 | #[test] 79 | fn test_choose_progress_style_default_style_specified() { 80 | let command_line_args = CommandLineArgs { 81 | progress_bar_style: Some(DEFAULT_PROGRESS_STYLE.to_string()), 82 | ..Default::default() 83 | }; 84 | 85 | let result = choose_progress_style(&command_line_args); 86 | assert_eq!(result.is_err(), false); 87 | let result = result.unwrap(); 88 | assert_eq!(result._style_name, LIGHT_BG_PROGRESS_STYLE); 89 | assert_eq!(result.enable_steady_tick, true); 90 | } 91 | 92 | #[test] 93 | fn test_choose_progress_style_light_bg_style_specified() { 94 | let command_line_args = CommandLineArgs { 95 | progress_bar_style: Some(LIGHT_BG_PROGRESS_STYLE.to_string()), 96 | ..Default::default() 97 | }; 98 | 99 | let result = choose_progress_style(&command_line_args); 100 | assert_eq!(result.is_err(), false); 101 | let result = result.unwrap(); 102 | assert_eq!(result._style_name, LIGHT_BG_PROGRESS_STYLE); 103 | assert_eq!(result.enable_steady_tick, true); 104 | } 105 | 106 | #[test] 107 | fn test_choose_progress_style_dark_bg_style_specified() { 108 | let command_line_args = CommandLineArgs { 109 | progress_bar_style: Some(DARK_BG_PROGRESS_STYLE.to_string()), 110 | ..Default::default() 111 | }; 112 | 113 | let result = choose_progress_style(&command_line_args); 114 | assert_eq!(result.is_err(), false); 115 | let result = result.unwrap(); 116 | assert_eq!(result._style_name, DARK_BG_PROGRESS_STYLE); 117 | assert_eq!(result.enable_steady_tick, true); 118 | } 119 | 120 | #[test] 121 | fn test_choose_progress_style_simple_style_specified() { 122 | let command_line_args = CommandLineArgs { 123 | progress_bar_style: Some(SIMPLE_PROGRESS_STYLE.to_string()), 124 | ..Default::default() 125 | }; 126 | 127 | let result = choose_progress_style(&command_line_args); 128 | assert_eq!(result.is_err(), false); 129 | let result = result.unwrap(); 130 | assert_eq!(result._style_name, SIMPLE_PROGRESS_STYLE); 131 | assert_eq!(result.enable_steady_tick, false); 132 | } 133 | 134 | #[test] 135 | fn test_choose_progress_style_unknown_style_specified() { 136 | let command_line_args = CommandLineArgs { 137 | progress_bar_style: Some("unknown".to_string()), 138 | ..Default::default() 139 | }; 140 | 141 | let result = choose_progress_style(&command_line_args); 142 | assert_eq!(result.is_err(), true); 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /tests/csv_file.txt: -------------------------------------------------------------------------------- 1 | 1,2,3 2 | foo,bar,baz -------------------------------------------------------------------------------- /tests/csv_file_badline.txt: -------------------------------------------------------------------------------- 1 | 1,2,3 2 | foo,bar,baz 3 | badline -------------------------------------------------------------------------------- /tests/dummy_shell.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "dummy_shell arg1=$1 arg2=$2" -------------------------------------------------------------------------------- /tests/file.txt: -------------------------------------------------------------------------------- 1 | hello 2 | from 3 | input 4 | file -------------------------------------------------------------------------------- /tests/integration_tests.rs: -------------------------------------------------------------------------------- 1 | use std::process::Command; 2 | 3 | use assert_cmd::cargo::CommandCargoExt; 4 | 5 | use predicates::prelude::*; 6 | 7 | fn rust_parallel_raw_command() -> Command { 8 | let mut cmd = Command::cargo_bin("rust-parallel").unwrap(); 9 | cmd.current_dir("tests/"); 10 | cmd 11 | } 12 | 13 | fn rust_parallel() -> assert_cmd::Command { 14 | assert_cmd::Command::from_std(rust_parallel_raw_command()) 15 | } 16 | 17 | #[test] 18 | fn runs_successfully() { 19 | rust_parallel() 20 | .assert() 21 | .success() 22 | .stdout(predicate::str::is_empty()) 23 | .stderr(predicate::str::is_empty()); 24 | } 25 | 26 | #[test] 27 | fn runs_echo_commands_from_args() { 28 | rust_parallel() 29 | .arg("echo") 30 | .arg(":::") 31 | .arg("A") 32 | .arg("B") 33 | .arg("C") 34 | .assert() 35 | .success() 36 | .stdout( 37 | (predicate::str::contains("\n").count(3)) 38 | .and(predicate::str::contains("A\n").count(1)) 39 | .and(predicate::str::contains("B\n").count(1)) 40 | .and(predicate::str::contains("C\n").count(1)), 41 | ) 42 | .stderr(predicate::str::is_empty()); 43 | } 44 | 45 | #[test] 46 | fn runs_echo_commands_from_args_j1() { 47 | rust_parallel() 48 | .arg("-j1") 49 | .arg("echo") 50 | .arg(":::") 51 | .arg("A") 52 | .arg("B") 53 | .arg("C") 54 | .assert() 55 | .success() 56 | .stdout(predicate::eq("A\nB\nC\n")) 57 | .stderr(predicate::str::is_empty()); 58 | } 59 | 60 | #[test] 61 | fn runs_echo_commands_dry_run() { 62 | rust_parallel() 63 | .arg("-s") 64 | .arg("--dry-run") 65 | .arg("echo") 66 | .arg(":::") 67 | .arg("A") 68 | .arg("B") 69 | .arg("C") 70 | .assert() 71 | .success() 72 | .stdout( 73 | (predicate::str::contains("\n").count(3)) 74 | .and( 75 | predicate::str::contains( 76 | r#"cmd="/bin/bash",args=["-c", "echo A"],line=command_line_args:1"#, 77 | ) 78 | .count(1), 79 | ) 80 | .and( 81 | predicate::str::contains( 82 | r#"cmd="/bin/bash",args=["-c", "echo B"],line=command_line_args:2"#, 83 | ) 84 | .count(1), 85 | ) 86 | .and( 87 | predicate::str::contains( 88 | r#"cmd="/bin/bash",args=["-c", "echo C"],line=command_line_args:3"#, 89 | ) 90 | .count(1), 91 | ), 92 | ) 93 | .stderr(predicate::str::is_empty()); 94 | } 95 | 96 | #[test] 97 | fn timeout_sleep_commands_from_args() { 98 | rust_parallel() 99 | .arg("-t1") 100 | .arg("sleep") 101 | .arg(":::") 102 | .arg("0") 103 | .arg("5") 104 | .assert() 105 | .failure() 106 | .code(1) 107 | .stdout( 108 | (predicate::str::contains("timeout: deadline has elapsed").count(1)) 109 | .and(predicate::str::contains("timeouts=1").count(1)), 110 | ) 111 | .stderr(predicate::str::is_empty()); 112 | } 113 | 114 | #[test] 115 | fn runs_echo_stdin() { 116 | let stdin = r#" 117 | echo A 118 | echo B 119 | echo C 120 | "#; 121 | rust_parallel() 122 | .write_stdin(stdin) 123 | .assert() 124 | .success() 125 | .stdout( 126 | (predicate::str::contains("\n").count(3)) 127 | .and(predicate::str::contains("A\n").count(1)) 128 | .and(predicate::str::contains("B\n").count(1)) 129 | .and(predicate::str::contains("C\n").count(1)), 130 | ) 131 | .stderr(predicate::str::is_empty()); 132 | } 133 | 134 | #[test] 135 | fn runs_echo_stdin_j1() { 136 | let stdin = r#" 137 | echo A 138 | echo B 139 | echo C 140 | "#; 141 | rust_parallel() 142 | .arg("-j1") 143 | .write_stdin(stdin) 144 | .assert() 145 | .success() 146 | .stdout(predicate::eq("A\nB\nC\n")) 147 | .stderr(predicate::str::is_empty()); 148 | } 149 | 150 | #[test] 151 | fn runs_file() { 152 | rust_parallel() 153 | .arg("-i") 154 | .arg("file.txt") 155 | .arg("echo") 156 | .assert() 157 | .success() 158 | .stdout( 159 | (predicate::str::contains("\n").count(4)) 160 | .and(predicate::str::contains("hello\n").count(1)) 161 | .and(predicate::str::contains("from\n").count(1)) 162 | .and(predicate::str::contains("input\n").count(1)) 163 | .and(predicate::str::contains("file\n").count(1)), 164 | ) 165 | .stderr(predicate::str::is_empty()); 166 | } 167 | 168 | #[test] 169 | fn runs_file_j1() { 170 | rust_parallel() 171 | .arg("-j1") 172 | .arg("-i") 173 | .arg("file.txt") 174 | .arg("echo") 175 | .assert() 176 | .success() 177 | .stdout(predicate::eq("hello\nfrom\ninput\nfile\n")) 178 | .stderr(predicate::str::is_empty()); 179 | } 180 | 181 | #[test] 182 | fn fails_j0() { 183 | rust_parallel() 184 | .arg("-j0") 185 | .assert() 186 | .failure() 187 | .stdout(predicate::str::is_empty()) 188 | .stderr(predicate::str::contains( 189 | "invalid value '0' for '--jobs '", 190 | )); 191 | } 192 | 193 | #[test] 194 | fn fails_t0() { 195 | rust_parallel() 196 | .arg("-t0") 197 | .assert() 198 | .failure() 199 | .stdout(predicate::str::is_empty()) 200 | .stderr(predicate::str::contains( 201 | "invalid value '0' for '--timeout-seconds '", 202 | )); 203 | } 204 | 205 | #[test] 206 | fn runs_shell_function_from_stdin_j1() { 207 | let stdin = r#"A 208 | B 209 | C"#; 210 | 211 | rust_parallel() 212 | .write_stdin(stdin) 213 | .arg("-j1") 214 | .arg("-s") 215 | .arg("--shell-path=./dummy_shell.sh") 216 | .arg("shell_function") 217 | .assert() 218 | .success() 219 | .stdout(predicate::eq( 220 | "dummy_shell arg1=-c arg2=shell_function A\ndummy_shell arg1=-c arg2=shell_function B\ndummy_shell arg1=-c arg2=shell_function C\n", 221 | )) 222 | .stderr(predicate::str::is_empty()); 223 | } 224 | 225 | #[test] 226 | fn runs_shell_function_from_file_j1() { 227 | rust_parallel() 228 | .arg("-j1") 229 | .arg("-i") 230 | .arg("file.txt") 231 | .arg("-s") 232 | .arg("--shell-path=./dummy_shell.sh") 233 | .arg("shell_function") 234 | .assert() 235 | .success() 236 | .stdout(predicate::eq( 237 | "dummy_shell arg1=-c arg2=shell_function hello\ndummy_shell arg1=-c arg2=shell_function from\ndummy_shell arg1=-c arg2=shell_function input\ndummy_shell arg1=-c arg2=shell_function file\n", 238 | )) 239 | .stderr(predicate::str::is_empty()); 240 | } 241 | 242 | #[test] 243 | fn runs_shell_function_from_args_j1() { 244 | rust_parallel() 245 | .arg("-j1") 246 | .arg("-s") 247 | .arg("--shell-path=./dummy_shell.sh") 248 | .arg("shell_function") 249 | .arg(":::") 250 | .arg("A") 251 | .arg("B") 252 | .arg("C") 253 | .assert() 254 | .success() 255 | .stdout(predicate::eq( 256 | "dummy_shell arg1=-c arg2=shell_function A\ndummy_shell arg1=-c arg2=shell_function B\ndummy_shell arg1=-c arg2=shell_function C\n", 257 | )) 258 | .stderr(predicate::str::is_empty()); 259 | } 260 | 261 | #[test] 262 | fn runs_regex_from_input_file_j1() { 263 | rust_parallel() 264 | .arg("-j1") 265 | .arg("-i") 266 | .arg("csv_file.txt") 267 | .arg("-r") 268 | .arg("(?P.*),(?P.*),(?P.*)") 269 | .arg("echo") 270 | .arg("arg1={arg1}") 271 | .arg("arg2={arg2}") 272 | .arg("arg3={arg3}") 273 | .arg("dollarzero={0}") 274 | .arg("emptygroup={}") 275 | .assert() 276 | .success() 277 | .stdout(predicate::eq( 278 | "arg1=1 arg2=2 arg3=3 dollarzero=1,2,3 emptygroup=1,2,3\narg1=foo arg2=bar arg3=baz dollarzero=foo,bar,baz emptygroup=foo,bar,baz\n", 279 | )) 280 | .stderr(predicate::str::is_empty()); 281 | } 282 | 283 | #[test] 284 | fn runs_regex_from_input_file_badline_j1() { 285 | rust_parallel() 286 | .arg("-j1") 287 | .arg("-i") 288 | .arg("csv_file_badline.txt") 289 | .arg("-r") 290 | .arg("(?P.*),(?P.*),(?P.*)") 291 | .arg("echo") 292 | .arg("arg1={arg1}") 293 | .arg("arg2={arg2}") 294 | .arg("arg3={arg3}") 295 | .arg("dollarzero={0}") 296 | .assert() 297 | .success() 298 | .stdout((predicate::str::contains("\n").count(3)).and(predicate::str::contains( 299 | "regex did not match input data: badline\n").and( 300 | predicate::str::contains( 301 | "arg1=1 arg2=2 arg3=3 dollarzero=1,2,3\narg1=foo arg2=bar arg3=baz dollarzero=foo,bar,baz\n", 302 | ) 303 | ) 304 | )) 305 | .stderr(predicate::str::is_empty()); 306 | } 307 | 308 | #[test] 309 | fn runs_regex_from_command_line_args_j1() { 310 | rust_parallel() 311 | .arg("-j1") 312 | .arg("-r") 313 | .arg("(.*),(.*),(.*)") 314 | .arg("echo") 315 | .arg("arg1={1}") 316 | .arg("arg2={2}") 317 | .arg("arg3={3}") 318 | .arg("dollarzero={0}") 319 | .arg("emptygroup={}") 320 | .arg(":::") 321 | .arg("a,b,c") 322 | .arg("d,e,f") 323 | .assert() 324 | .success() 325 | .stdout(predicate::eq( 326 | "arg1=a arg2=b arg3=c dollarzero=a,b,c emptygroup=a,b,c\narg1=d arg2=e arg3=f dollarzero=d,e,f emptygroup=d,e,f\n", 327 | )) 328 | .stderr(predicate::str::is_empty()); 329 | } 330 | 331 | #[test] 332 | fn runs_regex_from_command_line_args_nomatch_1() { 333 | rust_parallel() 334 | .arg("-j1") 335 | .arg("-r") 336 | .arg("(.*) (.*) (.*)") 337 | .arg("echo") 338 | .arg("arg1={1}") 339 | .arg("arg2={2}") 340 | .arg("arg3={3}") 341 | .arg("dollarzero={0}") 342 | .arg(":::") 343 | .arg("a,b,c") 344 | .arg("d,e,f") 345 | .assert() 346 | .success() 347 | .stdout((predicate::str::contains("\n").count(2)).and( 348 | predicate::str::contains("regex did not match input data: a,b,c\n").and( 349 | predicate::str::contains("regex did not match input data: d,e,f\n"), 350 | ), 351 | )) 352 | .stderr(predicate::str::is_empty()); 353 | } 354 | 355 | #[test] 356 | fn fails_invalid_regex() { 357 | rust_parallel() 358 | .arg("-r") 359 | .arg("((.*),(.*),(.*)") 360 | .arg("echo") 361 | .arg(":::") 362 | .arg("a,b,c") 363 | .arg("d,e,f") 364 | .assert() 365 | .failure() 366 | .stdout(predicate::str::contains( 367 | "CommandLineRegex::new: error creating regex:", 368 | )) 369 | .stderr(predicate::str::is_empty()); 370 | } 371 | 372 | #[test] 373 | fn runs_auto_regex_from_command_line_args_j1() { 374 | rust_parallel() 375 | .arg("-j1") 376 | .arg("echo") 377 | .arg("arg1={1}") 378 | .arg("arg2={2}") 379 | .arg("dollarzero={0}") 380 | .arg("emptygroup={}") 381 | .arg(":::") 382 | .arg("a") 383 | .arg("b") 384 | .arg(":::") 385 | .arg("c") 386 | .arg("d") 387 | .assert() 388 | .success() 389 | .stdout(predicate::eq( 390 | "arg1=a arg2=c dollarzero=a c emptygroup=a c\narg1=a arg2=d dollarzero=a d emptygroup=a d\narg1=b arg2=c dollarzero=b c emptygroup=b c\narg1=b arg2=d dollarzero=b d emptygroup=b d\n", 391 | )) 392 | .stderr(predicate::str::is_empty()); 393 | } 394 | 395 | #[test] 396 | fn runs_regex_from_input_file_produce_json_named_groups_j1() { 397 | let expected_stdout = r#"{"id": 123, "zero": "1,2,3", "empty": "1,2,3", "one": "1", "two": "2", "three": "3"} 398 | {"id": 123, "zero": "foo,bar,baz", "empty": "foo,bar,baz", "one": "foo", "two": "bar", "three": "baz"} 399 | "#; 400 | 401 | rust_parallel() 402 | .arg("-j1") 403 | .arg("-i") 404 | .arg("csv_file.txt") 405 | .arg("-r") 406 | .arg("(?P.*),(?P.*),(?P.*)") 407 | .arg("echo") 408 | .arg(r#"{"id": 123, "zero": "{0}", "empty": "{}", "one": "{arg1}", "two": "{arg2}", "three": "{arg3}"}"#) 409 | .assert() 410 | .success() 411 | .stdout(predicate::eq(expected_stdout)) 412 | .stderr(predicate::str::is_empty()); 413 | } 414 | 415 | #[test] 416 | fn runs_regex_from_input_file_produce_json_numbered_groups_j1() { 417 | let expected_stdout = r#"{"id": 123, "zero": "1,2,3", "empty": "1,2,3", "three": "3", "two": "2", "one": "1"} 418 | {"id": 123, "zero": "foo,bar,baz", "empty": "foo,bar,baz", "three": "baz", "two": "bar", "one": "foo"} 419 | "#; 420 | 421 | rust_parallel() 422 | .arg("-j1") 423 | .arg("-i") 424 | .arg("csv_file.txt") 425 | .arg("-r") 426 | .arg("(.*),(.*),(.*)") 427 | .arg("echo") 428 | .arg(r#"{"id": 123, "zero": "{0}", "empty": "{}", "three": "{3}", "two": "{2}", "one": "{1}"}"#) 429 | .assert() 430 | .success() 431 | .stdout(predicate::eq(expected_stdout)) 432 | .stderr(predicate::str::is_empty()); 433 | } 434 | 435 | #[test] 436 | fn runs_regex_command_with_dollar_signs() { 437 | let expected_stdout = "input 1$ input bar\n"; 438 | 439 | let stdin = "input"; 440 | 441 | rust_parallel() 442 | .write_stdin(stdin) 443 | .arg("-j1") 444 | .arg("-r") 445 | .arg(".*") 446 | .arg("-s") 447 | .arg(r#"foo={0}; echo $foo 1$ "$foo" "$(echo bar)""#) 448 | .assert() 449 | .success() 450 | .stdout(predicate::eq(expected_stdout)) 451 | .stderr(predicate::str::is_empty()); 452 | } 453 | 454 | #[test] 455 | fn runs_no_run_if_empty_echo_j1() { 456 | let stdin = r#" 457 | 458 | A 459 | 460 | B 461 | 462 | C 463 | 464 | "#; 465 | 466 | rust_parallel() 467 | .write_stdin(stdin) 468 | .arg("-j1") 469 | .arg("--no-run-if-empty") 470 | .arg("echo") 471 | .assert() 472 | .success() 473 | .stdout(predicate::eq("A\nB\nC\n")) 474 | .stderr(predicate::str::is_empty()); 475 | } 476 | 477 | #[test] 478 | fn runs_shell_function_from_stdin_no_run_if_empty_j1() { 479 | let stdin = r#" 480 | 481 | A 482 | 483 | B 484 | 485 | C 486 | 487 | "#; 488 | 489 | rust_parallel() 490 | .write_stdin(stdin) 491 | .arg("-j1") 492 | .arg("-s") 493 | .arg("--no-run-if-empty") 494 | .arg("--shell-path=./dummy_shell.sh") 495 | .arg("shell_function") 496 | .assert() 497 | .success() 498 | .stdout(predicate::eq( 499 | "dummy_shell arg1=-c arg2=shell_function A\ndummy_shell arg1=-c arg2=shell_function B\ndummy_shell arg1=-c arg2=shell_function C\n", 500 | )) 501 | .stderr(predicate::str::is_empty()); 502 | } 503 | 504 | #[test] 505 | fn test_exit_status_on_failing_commands() { 506 | rust_parallel() 507 | .arg("-j1") 508 | .arg("cat") 509 | .arg(":::") 510 | .arg("A") 511 | .arg("B") 512 | .arg("C") 513 | .assert() 514 | .failure() 515 | .code(1) 516 | .stdout( 517 | (predicate::str::contains("command failed").count(3)) 518 | .and(predicate::str::contains("command failures:")) 519 | .and(predicate::str::contains("exit_status_errors=3")), 520 | ) 521 | .stderr( 522 | (predicate::str::contains("cat: A: No such file or directory").count(1)) 523 | .and(predicate::str::contains("cat: B: No such file or directory").count(1)) 524 | .and(predicate::str::contains("cat: C: No such file or directory").count(1)), 525 | ); 526 | } 527 | 528 | #[test] 529 | fn test_exit_status_on_failing_commands_exit_on_error() { 530 | rust_parallel() 531 | .arg("-j1") 532 | .arg("--exit-on-error") 533 | .arg("cat") 534 | .arg(":::") 535 | .arg("A") 536 | .arg("B") 537 | .arg("C") 538 | .assert() 539 | .failure() 540 | .code(1) 541 | .stdout( 542 | (predicate::str::contains("command failed")) 543 | .and(predicate::str::contains("command failures:")) 544 | .and(predicate::str::contains("exit_status_errors=0").not()), 545 | ) 546 | .stderr(predicate::str::contains("cat: A: No such file or directory").count(1)); 547 | } 548 | --------------------------------------------------------------------------------