├── .gitignore
├── .jshintignore
├── .jshintrc
├── CHANGELOG.md
├── LICENSE.md
├── README.md
├── bin
    └── parallel.js
├── lib
    ├── cli.js
    ├── help.js
    ├── input.js
    ├── jobs.js
    ├── opts.js
    ├── placeholders.js
    └── util.js
└── package.json


/.gitignore:
--------------------------------------------------------------------------------
 1 | lib-cov
 2 | *.seed
 3 | *.log
 4 | *.csv
 5 | *.dat
 6 | *.out
 7 | *.pid
 8 | *.gz
 9 | 
10 | pids
11 | logs
12 | results
13 | 
14 | npm-debug.log
15 | node_modules
16 | tmp
17 | 


--------------------------------------------------------------------------------
/.jshintignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | test/


--------------------------------------------------------------------------------
/.jshintrc:
--------------------------------------------------------------------------------
 1 | {
 2 | 	"shadow": "inner",
 3 | 	"indent": 1,
 4 | 
 5 | 	"camelcase": false,
 6 | 	"eqeqeq": true,
 7 | 	"eqnull": true,
 8 | 	"freeze": true,
 9 | 	"funcscope": true,
10 | 	"newcap": true,
11 | 	"noarg": true,
12 | 	"noempty": true,
13 | 	"nonbsp": true,
14 | 	"unused": "vars",
15 | 	"undef": true,
16 | 	"scripturl": true,
17 | 	"strict": false,
18 | 	"loopfunc": true,
19 | 	"quotmark": "single",
20 | 
21 | 	"esnext": true,
22 | 	"globals": {"define": true, "jade":true},
23 | 	"browser": true,
24 | 	"devel": true,
25 | 	"mocha": true,
26 | 	"node": true,
27 | 	"jquery": true
28 | }
29 | 


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | # 1.3.0
 2 | - Add various new useful placeholders, that are not in the GNU version
 3 | 
 4 | # 1.2.0
 5 | - Added support for `--halt-on-error` option, if any job exits with non-zero code. Parallel will exit immediately
 6 | - Parallel exit code now reflects the amount of failed jobs up to 101
 7 | 
 8 | # 1.1.1
 9 | - Fixed {%} was NaN when `--jobs=0`
10 | - Changed the workaround to handle stdin not being piped in
11 | - Improved some cryptic variable names
12 | 
13 | # 1.0.10
14 | - Each argument after a `:::` is now a separate line
15 | - Including several `:::` permutates the arguments instead of just concatenating them
16 | - `--verbose` also logs complete command line contents for each job that starts
17 | 
18 | # 1.0.9
19 | - Workaround from 1.0.8 didn't work once process is run as bin, implemented another solution based on a timeout
20 | 
21 | # 1.0.8
22 | - `--jobs=0` is now supported for an unlimited amount of parallel jobs
23 | - Invalid options combinations are now validated and explicitely reported
24 | - Added support for input being passed on command-line arguments using the `:::` operator
25 | - Worked around the long standing issue where process won't close when stdin is not provided
26 | - Time measurement now uses `process.hrtime()` which is more accurate than `Date.now()`
27 | 
28 | # 1.0.7
29 | - Command-line options now support `--key=value` format in addition to `--key value`
30 | - Command-line options now support `-j2` format in addition to `-j 2`
31 | - Added `-D` as alias for `--dry-run`. GNU's parallel doesn't have any and I think it's a useful option and deserves one.
32 | 
33 | # 1.0.6
34 | - Fixed job.writable wasn't being restored when stdin is drained, would hang all until all input has been loaded
35 | - Added support for `--dry-run` option, resulting commands are printed to stdout within runnning. Incompatible with `--pipe`
36 | 
37 | # 1.0.5
38 | - Implemented positional placeholders to split input line in columns
39 | - Fixed {/.} would yield empty string when line had no dot after the last slash
40 | 
41 | # 1.0.4
42 | - Added support for `--timeout <sec>` option, kill jobs with SIGTERM if they take too long
43 | 
44 | # 1.0.3
45 | - Added support for `--delay <sec>` option, wait sec seconds before running a new job
46 | 
47 | # 1.0.2
48 | - Added support for `--bg` option, jobs are run detached and main process can exit
49 | 
50 | # 1.0.1
51 | - Reworded `replacement` as `placeholder` across the board, it's a more suitable name
52 | 
53 | # 1.0.0
54 | - First release
55 | 


--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2016, Ariel Flesler
 2 | All rights reserved.
 3 | 
 4 | Redistribution and use in source and binary forms, with or without modification,
 5 | are permitted provided that the following conditions are met:
 6 | 
 7 | - Redistributions of source code must retain the above copyright notice, this
 8 |   list of conditions and the following disclaimer.
 9 | 
10 | - Redistributions in binary form must reproduce the above copyright notice, this
11 |   list of conditions and the following disclaimer in the documentation and/or
12 |   other materials provided with the distribution.
13 | 
14 | - Neither the name of the organization nor the names of its
15 |   contributors may be used to endorse or promote products derived from
16 |   this software without specific prior written permission.
17 | 
18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | parallel
  2 | =======
  3 | 
  4 | CLI tool to execute shell commands in parallel.
  5 | 
  6 | Loosely based on [GNU parallel command](https://www.gnu.org/software/parallel/man.html).
  7 | 
  8 | # Installation
  9 | 
 10 | Using [npm](https://www.npmjs.com/package/parallel):
 11 | ```bash
 12 | $ npm install -g parallel
 13 | ```
 14 | 
 15 | # Usage
 16 | 
 17 | ```bash
 18 | # Pass input lines as command-line arguments
 19 | input | parallel [options] cmd [cmd-options] {} > output
 20 | 
 21 | # Pipe input lines through the jobs stdin
 22 | input | parallel [options] --pipe cmd [cmd-options] > output
 23 | ```
 24 | 
 25 | # Options
 26 | 
 27 | ```bash
 28 | -j, --jobs <n>          Max processes to run in parallel (0 for ∞) [default CPUs]
 29 | -n, --max-args <args>   Number of input lines per command line [default 1]
 30 | -d, --delimiter <delim> Input items are terminated by delim [default \n]
 31 | -0, --null              Use NUL as delimiter, alias for -d $'\\0'
 32 | -q, --quote             Quote each input line in case they contain special caracters
 33 | -t, --trim              Removes spaces, tabs and new lines around the input lines
 34 | -C, --colsep <regex>    Column separator for positional placeholders [default " "]
 35 | -a, --arg-file <file>   Use file as input source instead of stdin
 36 | -p, --pipe              Spread input lines to jobs via their stdin
 37 | -D, --dry-run           Print commands to run without running them
 38 | --bg                    Run commands in background and exit
 39 | --delay <secs>          Wait before starting new jobs, secs can be less than 1 [default 0]
 40 | --timeout <secs>        If the command runs longer than secs it gets killed with SIGTERM [default 0]
 41 | --halt-on-error         Kill all jobs and exit if any job exits with a code other than 0 [default false]
 42 | -v, --verbose           Output timing information to stderr
 43 | -s, --shell             Wrap command with shell (supports escaped pipes, redirection, etc.) [experimental]
 44 | --help                  Print this message and exit
 45 | --version               Print the comand version and exit
 46 | ```
 47 | 
 48 | # Arguments placeholders
 49 | 
 50 | Unless `--pipe` is used, the input lines will be sent to jobs as command-line arguments. You can include placeholders and they will be replaced with each input line.
 51 | If no placeholder is found, input lines will be appended to the end as last arguments.
 52 | Everything around each placeholder will be repeated for each input line. Use quotes to include spaces or escape them with backslashes.
 53 | 
 54 | ```
 55 | {}   input line
 56 | {.}  input line without extension
 57 | {/}  basename of the input line
 58 | {//} dirname of the input line
 59 | {/.} basename of the input line without extension
 60 | {n}  nth input column, followed by any operator above (f.e {2/.})
 61 | {#}  sequence number of the job to run [1, ∞]
 62 | {%}  job slot number [1, --jobs]
 63 | ```
 64 | 
 65 | These are not in the original GNU parallel, but were implemented here:
 66 | 
 67 | ```
 68 | {..} extension of the input line
 69 | {v} lower case the value
 70 | {^} upper case the value
 71 | {t} current time as a number
 72 | {T} current time in ISO as a string
 73 | {d} current date in ISO format
 74 | {r} random number between 100000 and 999999
 75 | {md5} MD5 hash of the input line
 76 | ```
 77 | 
 78 | # Input from command-line arguments
 79 | 
 80 | Input can be provided as command-line arguments preceeded by a `:::`.
 81 | Each argument will be considered a separate input line.
 82 | If you include several `:::`, parallel will use all the permutations between them as input lines.
 83 | While GNU´s version also permutates stdin and input files, this version won't.
 84 | Check examples (6) and (7) to see this in action.
 85 | 
 86 | # Examples
 87 | 
 88 | ```bash
 89 | # (1) Use all CPUs to grep a file
 90 | cat data.txt | parallel -p grep pattern > out.txt
 91 | ```
 92 | ```bash
 93 | # (2) Use all CPUs to gunzip and concat files to a single file, 10 per process at a time
 94 | find . -name '*.gz' | parallel -n10 gzip -dc {} > out.txt
 95 | ```
 96 | ```bash
 97 | # (3) Download files from a list, 10 at a time with all CPUs, use the URL basename as file name
 98 | cat urls.txt | parallel -j10 curl {} -o images/{/}
 99 | ```
100 | ```bash
101 | # (4) Generate 100 URLs and download them with `curl` (uses experimental --shell option)
102 | seq 100 | parallel -s curl http://xyz.com/image_{}.png \> image_{}.png
103 | ```
104 | ```bash
105 | # (5) Move each file to a subdir relative to their current dir
106 | find . -type f | parallel mkdir -p {//}/sub && mv {} {//}/sub/{/}
107 | ```
108 | ```bash
109 | # (6) Show how to provide input as command-line arguments and what the order is
110 | echo 4 | parallel -j1 echo ::: 1 2 3
111 | ```
112 | ```bash
113 | # (7) Rename extension from all txt to log
114 | parallel mv {} {.}.log ::: *.txt
115 | ```
116 | ```bash
117 | # (8) Showcase non-positional placeholders
118 | find . -type f | parallel echo "file={} noext={.} base={/} base_noext={/.} dir={//} jobid={#} jobslot={%} ext={..} lower={v} upper={^} time={t} timeiso={T} date={d} random={r} md5={md5}"
119 | ```
120 | ```bash
121 | # (9) Showcase positional placeholders
122 | echo A~B.ext~C~D | parallel -C '~' echo {4}+{3}+{2.}+{1}
123 | ```
124 | 
125 | # Command-line options
126 | Once a command-line parameter that is not an option is found, then the "command" starts.
127 | parallel supports command-line options in all these formats (all equivalent):
128 | - `--trim --jobs 2`
129 | - `--trim --jobs=2`
130 | - `-t -j 2`
131 | - `-tj 2`
132 | - `-tj2`
133 | 
134 | # Exit code
135 | Just like [GNU parallel](https://www.gnu.org/software/parallel/man.html#EXIT-STATUS) does, the exit code will be the amount of jobs that failed (up to 101). It means that if any job fails, "global" exit code will be non-zero as well. You can add `--halt-on-error` to abort as soon as one job fails.
136 | 
137 | # Differences with [GNU parallel](https://www.gnu.org/software/parallel/man.html)
138 | - Added aliases to some options: `-p` -> `--pipe`, `-D` -> `--dry-run`
139 | - `--round-robin` is implicit when `--pipe` is used
140 | - This module does support piped input and `:::` arguments together unlike GNU's
141 | - This module won't permutate input from `:::` and from stdin or `--arg-file`
142 | - GNU's `-m` can be achieved here with `--max-args=0` to distribute all input lines evenly among `--jobs`
143 | - `--shell` was added to allow pipes, redirection, etc
144 | - `--trim` doesn't support `<n|l|r|lr|rl>`, it trims all spaces, tabs and newlines from both sides
145 | - `--halt-on-error` doesn't support any option, it exits as soon as one job fails
146 | - A ton of missing options that I consider less useful
147 | - `--plus` placeholders are not supported
148 | - But this supports various placeholders that GNU's parallel doesn't (see above)
149 | - Many more
150 | 
151 | # ToDo
152 | - Implement backpressure to pause input if output is overwhelmed
153 | - Support `--header` for working with CSV-like files
154 | - Should it permutate lines from stdin and `--arg-file` ?
155 | - Could implement `--keep-order`
156 | - Use [node-shell-quote](https://github.com/substack/node-shell-quote) for `--dry-run` and `--shell`?
157 | - Clean up `jobs` module, maybe create a `job` module with some of its logic
158 | - Maybe avoid pre-spawning jobs when piping. Spawn on demand when overwhelmed, support `--delay` there too
159 | - Support multiple `-a`? can be achieved with `cat a b c` though, maybe it's pointless
160 | - Allow placeholders to be chained as such: `{..|v|md5}` (get the extension, then lowercase, then md5)
161 | 
162 | # License
163 | 
164 | Copyright (c) 2016, Ariel Flesler
165 | All rights reserved.
166 | 
167 | Redistribution and use in source and binary forms, with or without modification,
168 | are permitted provided that the following conditions are met:
169 | 
170 | * Redistributions of source code must retain the above copyright notice, this
171 |   list of conditions and the following disclaimer.
172 | 
173 | * Redistributions in binary form must reproduce the above copyright notice, this
174 |   list of conditions and the following disclaimer in the documentation and/or
175 |   other materials provided with the distribution.
176 | 
177 | * Neither the name of the organization nor the names of its
178 |   contributors may be used to endorse or promote products derived from
179 |   this software without specific prior written permission.
180 | 
181 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
182 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
183 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
184 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
185 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
186 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
187 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
188 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
189 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
190 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
191 | 


--------------------------------------------------------------------------------
/bin/parallel.js:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env node
 2 | var
 3 | 	opts = require('../lib/opts'),
 4 | 	jobs = require('../lib/jobs'),
 5 | 	util = require('../lib/util'),
 6 | 	input = require('../lib/input'),
 7 | 	cli = require('../lib/cli');
 8 | 
 9 | process.on('uncaughtException', util.fatal);
10 | process.setMaxListeners(Infinity);
11 | 
12 | cli.parse(process.argv.slice(2));
13 | 
14 | if (opts.pipeMode) {
15 | 	jobs.spawnPiped();
16 | }
17 | 
18 | input.open();
19 | 


--------------------------------------------------------------------------------
/lib/cli.js:
--------------------------------------------------------------------------------
  1 | var
  2 | 	fs = require('fs'),
  3 | 	util = require('./util'),
  4 | 	help = require('./help'),
  5 | 	input = require('./input'),
  6 | 	opts = require('./opts');
  7 | 
  8 | var options = {
  9 | 	'jobs': {
 10 | 		alias: 'j',
 11 | 		param: 'n',
 12 | 		default: 'CPUs('+opts.maxJobs+')',
 13 | 		desc: 'Max processes to run in parallel (0 for ∞)',
 14 | 		fn: function(value) { opts.maxJobs = parseInt(value, 10); }
 15 | 	},
 16 | 	'max-args': {
 17 | 		alias: 'n',
 18 | 		param: 'args',
 19 | 		default: opts.maxArgs,
 20 | 		desc: 'Number of input lines per jobs (0 to split evenly)',
 21 | 		fn: function(value) { opts.maxArgs = parseInt(value, 10); }
 22 | 	},
 23 | 	'delimiter': {
 24 | 		alias: 'd',
 25 | 		param: 'delim',
 26 | 		default: '\\n',// opts.lineSep,
 27 | 		desc: 'Input items are terminated by delim',
 28 | 		fn: function(value) { opts.lineSep = value; }
 29 | 	},
 30 | 	'null': {
 31 | 		alias: '0',
 32 | 		desc: 'Use NUL as delimiter, alias for -d $\'\\\\0\'',
 33 | 		fn: function() { opts.lineSep = '\\0'; }
 34 | 	},
 35 | 	'quote': {
 36 | 		alias: 'q',
 37 | 		desc: 'Quote each input line in case they contain special caracters',
 38 | 		fn: function() { opts.quote = true; }
 39 | 	},
 40 | 	'trim': {
 41 | 		alias: 't',
 42 | 		desc: 'Removes spaces, tabs and new lines around the input lines',
 43 | 		fn: function() { opts.trim = true; }
 44 | 	},
 45 | 	'colsep': {
 46 | 		alias: 'C',
 47 | 		desc: 'Column separator for positional placeholders',
 48 | 		param: 'regex',
 49 | 		default: '" "',
 50 | 		fn: function(value) { opts.colSep = value; }
 51 | 	},
 52 | 	'arg-file': {
 53 | 		alias: 'a',
 54 | 		param: 'file',
 55 | 		desc: 'Use file as input source instead of stdin',
 56 | 		fn: function(value) { opts.src = fs.createReadStream(value); }
 57 | 	},
 58 | 	'pipe': {
 59 | 		alias: 'p',
 60 | 		desc: 'Spread input lines to jobs via their stdin',
 61 | 		fn: function() { opts.pipeMode = true; }
 62 | 	},
 63 | 	'dry-run': {
 64 | 		alias: 'D',
 65 | 		desc: 'Print commands to run without running them',
 66 | 		fn: function(value) { opts.dryRun = true; }
 67 | 	},
 68 | 	'bg': {
 69 | 		desc: 'Run commands in background and exit',
 70 | 		fn: function() { opts.bgMode = true; }
 71 | 	},
 72 | 	'delay': {
 73 | 		desc: 'Wait before starting new jobs, secs can be less than 1',
 74 | 		param: 'secs',
 75 | 		default: opts.delay,
 76 | 		fn: function(value) { opts.delay = parseFloat(value); }
 77 | 	},
 78 | 	'timeout': {
 79 | 		desc: 'If the command runs for longer than secs it will get killed with SIGTERM',
 80 | 		param: 'secs',
 81 | 		default: opts.timeout,
 82 | 		fn: function(value) { opts.timeout = parseFloat(value); }
 83 | 	},
 84 | 	'halt-on-error': {
 85 | 		desc: 'Kill all jobs and exit if any job exits with a code other than 0',
 86 | 		default: opts.haltOnError,
 87 | 		fn: function() { opts.haltOnError = true; }
 88 | 	},
 89 | 	'verbose': {
 90 | 		alias: 'v',
 91 | 		desc: 'Output timing information to stderr',
 92 | 		fn: function() { opts.verbose = true; }
 93 | 	},
 94 | 	'shell': {
 95 | 		alias: 's',
 96 | 		desc: 'Wrap command with shell (supports escaped pipes, redirection, etc.) [experimental]',
 97 | 		fn: function() { opts.shell = true; }
 98 | 	},
 99 | 	'help': {
100 | 		desc: 'Print this message and exit',
101 | 		fn: function() { help.show(options); }
102 | 	},
103 | 	'version': {
104 | 		desc: 'Print the comand version and exit',
105 | 		fn: function() { help.version(); }
106 | 	}
107 | };
108 | 
109 | var aliases = {};
110 | for (var opt in options) {
111 | 	aliases[options[opt].alias] = opt;
112 | }
113 | 
114 | exports.parse = function(args) {
115 | 	opts._ = args;
116 | 	while (args.length) {
117 | 		var arg = args[0];
118 | 		// Rest belongs to the command
119 | 		if (arg[0] !== '-') break;
120 | 
121 | 		args.shift();
122 | 		// Long version --*
123 | 		if (arg[1] === '-') {
124 | 			if (~arg.indexOf('=')) {
125 | 				// Support --abc=123
126 | 				var p = arg.split('=');
127 | 				args.unshift(p[1]);
128 | 				arg = p[0];
129 | 			}
130 | 			processArg(arg.slice(2), arg);
131 | 		// Short-hand -*
132 | 		} else {
133 | 			for (var i = 1, l = arg.length; i < l;) {
134 | 				var chr = arg[i++];
135 | 				var opt = aliases[chr];
136 | 				// Support -j2 as an alternative to -j 2
137 | 				if (options[opt] && options[opt].param && i < l) {
138 | 					args.unshift(arg.slice(i));
139 | 					i = l;
140 | 				}
141 | 				processArg(opt, '-'+chr);
142 | 			}
143 | 		}
144 | 	}
145 | 
146 | 	// Check for invalid combinations
147 | 	validateOpts();
148 | 
149 | 	// Parse optional input provided with :::
150 | 	parseInlineInput();
151 | };
152 | 
153 | function processArg(opt, orig) {
154 | 	var option = options[opt];
155 | 	if (!option) util.fatal('Unknown option '+orig);
156 | 	if (option.param) {
157 | 		option.fn(opts._.shift());
158 | 	} else {
159 | 		option.fn();
160 | 	}
161 | }
162 | 
163 | function validateOpts() {
164 | 	if (!opts.maxJobs && opts.pipeMode) {
165 | 		util.fatal('--jobs=0 and --pipe cannot be used together');
166 | 	}
167 | 	if (!opts.maxJobs && !opts.maxArgs) {
168 | 		util.fatal('--jobs=0 and --max-args=0 cannot be used together');
169 | 	}
170 | 	if (opts.dryRun && opts.pipeMode) {
171 | 		util.fatal('--dry-run and --pipe are pointless together');
172 | 	}
173 | }
174 | 
175 | const INLINE_INPUT_SEP = ':::';
176 | 
177 | function parseInlineInput() {
178 | 	// TODO: If several ::: GNU yields the combination of all
179 | 	var args = opts._, inputs = [];
180 | 	while (true) {
181 | 		var index = args.lastIndexOf(INLINE_INPUT_SEP);
182 | 		if (index === -1) break;
183 | 		var cols = args.splice(index);
184 | 		inputs.unshift(cols.slice(1));
185 | 	}
186 | 
187 | 	if (inputs.length) {
188 | 		input.setInlineInput(inputs);
189 | 	}
190 | }
191 | 


--------------------------------------------------------------------------------
/lib/help.js:
--------------------------------------------------------------------------------
 1 | var	pkg = require('../package.json');
 2 | 
 3 | const MIN_LENGTH = 23;
 4 | 
 5 | exports.show = function(options) {
 6 | 	line(pkg.description);
 7 | 	line();
 8 | 	line('Usage:');
 9 | 	line('  # Pass input lines as command-line arguments');
10 | 	line('  input | parallel [options] cmd [cmd-options] {} > output');
11 | 	line('  # Pipe input lines through the jobs stdin');
12 | 	line('  input | parallel [options] --pipe cmd [cmd-options] > output');
13 | 	line();
14 | 	line('Options:');
15 | 	
16 | 	for (var opt in options) {
17 | 		var option = options[opt];
18 | 		var cols = ['--'+opt];
19 | 		if (option.alias) cols.unshift('-'+option.alias+',');
20 | 		if (option.param) cols.push('<'+option.param+'>');
21 | 		
22 | 		var str = cols.join(' ');
23 | 		// Align columns
24 | 		while (str.length < MIN_LENGTH) {
25 | 			str += ' ';
26 | 		}
27 | 		
28 | 		var desc = option.desc;
29 | 		if ('default' in option) {
30 | 			desc += ' [default '+option.default+']';
31 | 		}
32 | 		line('  ' + str + ' ' + desc);
33 | 	}
34 | 	line();
35 | 	line('Placeholders:');
36 | 	line('  {}   the input line');
37 | 	line('  {.}  the input line without extension');
38 | 	line('  {/}  the basename of the input line');
39 | 	line('  {//} the dirname of the input line');
40 | 	line('  {/.} the basename of the input line without extension');
41 | 	line('  {n}  nth input column, followed by any operator above (f.e {2/.})');
42 | 	line('  {#}  the sequence number of the job to run, [1,]');
43 | 	line('  {%}  the job slot number [1, --jobs]');
44 |   line('Non-GNU placeholders:')
45 |   line('  {..} the extension of the input line')
46 |   line('  {v} lower case the value')
47 |   line('  {^} upper case the value')
48 |   line('  {t} current time as a number')
49 |   line('  {T} current time in ISO as a string')
50 |   line('  {d} current date in ISO format')
51 |   line('  {r} random number between 100000 and 999999')
52 |   line('  {md5} MD5 hash of the input line');
53 | 	line();
54 | 	line('Visit '+pkg.homepage+'#examples to see examples');
55 | 
56 | 	process.exit();
57 | };
58 | 
59 | function line(text/*?*/) {
60 | 	if (text) {
61 | 		console.log('  ' + text);
62 | 	} else {
63 | 		console.log();
64 | 	}
65 | }
66 | 
67 | exports.version = function() {
68 | 	console.log(pkg.version);
69 | 	process.exit();
70 | };


--------------------------------------------------------------------------------
/lib/input.js:
--------------------------------------------------------------------------------
 1 | var
 2 | 	util = require('./util'),
 3 | 	opts = require('./opts'),
 4 | 	jobs = require('./jobs');
 5 | 
 6 | const NEW_LINE = '\n';
 7 | 
 8 | exports.open = function() {
 9 | 	// If new line, support with and without carriage return
10 | 	var sep = opts.lineSep === NEW_LINE ? /\r?\n/ : new RegExp(opts.lineSep);
11 | 
12 | 	var src = opts.src, buf = '';
13 | 	src.setEncoding('utf8');
14 | 	src.on('error', util.fatal);
15 | 	src.on('data', function(chunk) {
16 | 		if (sep.test(chunk)) {
17 | 			var buffer = (buf+chunk).split(sep);
18 | 			buf = buffer.pop();
19 | 			buffer.forEach(jobs.processLine);
20 | 		} else {
21 | 			buf += chunk;
22 | 		}
23 | 	});
24 | 	src.once('end', function() {
25 | 		if (buf) {
26 | 			jobs.processLine(buf);
27 | 			buf = '';
28 | 		}
29 | 		jobs.close();
30 | 	});
31 | 
32 | 	// When nothing is piped, stdin needs end() closed to be released
33 | 	if (src === process.stdin && process.stdin.isTTY) {
34 | 		src.end();
35 | 	}
36 | };
37 | 
38 | // Input provided through command-line with ::: is placed first
39 | exports.setInlineInput = function(inputs) {
40 | 	var perms = generatePermutations(inputs);
41 | 	perms.push('');
42 | 	// TODO: How do these interact with stdin/-a input(s)
43 | 	opts.src.unshift(perms.join(opts.lineSep));
44 | };
45 | 
46 | function generatePermutations(inputs) {
47 | 	var perms = [];
48 | 	var max = inputs.reduce(function(m, input) {
49 | 		return m * input.length;
50 | 	}, 1);
51 | 
52 | 	while (max--) {
53 | 		var rest = max;
54 | 		var cols = [];
55 | 		var i = inputs.length;
56 | 		while (i--) {
57 | 			var input = inputs[i];
58 | 			var l = input.length;
59 | 			cols.unshift(input[rest % l]);
60 | 			rest = Math.floor(rest / l);
61 | 		}
62 | 		perms.unshift(cols.join(opts.colSep));
63 | 	}
64 | 	return perms;
65 | }
66 | 


--------------------------------------------------------------------------------
/lib/jobs.js:
--------------------------------------------------------------------------------
  1 | var
  2 | 	cp = require('child_process'),
  3 | 	util = require('./util'),
  4 | 	opts = require('./opts'),
  5 | 	placeholders = require('./placeholders'),
  6 | 	procs = [],
  7 | 	buffer = [],
  8 | 	closing = false,
  9 | 	jobId = 0,
 10 | 	timer = util.timer();
 11 | 
 12 | function spawn(args) {
 13 | 	if (opts.shell) {
 14 | 		// This is still experimental
 15 | 		args = ['sh', '-c', args.join(' ')];
 16 | 	}
 17 | 
 18 | 	var proc = cp.spawn(args[0], args.slice(1), {
 19 | 		stdio:['pipe', process.stdout, process.stderr],
 20 | 		detached: opts.bgMode,
 21 | 		timeout: opts.timeout * 1e3
 22 | 	});
 23 | 	proc.id = jobId;
 24 | 	procs.push(proc);
 25 | 
 26 | 	if (opts.verbose) {
 27 | 		proc.timer = util.timer();
 28 | 		// TODO: Kind of point-less when --pipe, maybe don't print in that case
 29 | 		util.error('Job %d (%d) command: "%s"', proc.id, proc.pid, args.join(' '));
 30 | 		proc.on('exit', function(code) {
 31 | 			util.error('Job %d (%d) handled %d line(s) in %s and exited with code %d', proc.id, proc.pid, proc.handled, timer(), code);
 32 | 		});
 33 | 	}
 34 | 	proc.on('close', function(code) {
 35 | 		if (!code) return;
 36 | 		// Exit code will include the amount of failed jobs (up to 101) like GNU version does
 37 | 		process.exitCode = Math.min(100, process.exitCode || 0) + 1;
 38 | 		if (opts.haltOnError) {
 39 | 			process.exit();
 40 | 		}
 41 | 	});
 42 | 	return proc;
 43 | }
 44 | 
 45 | exports.spawnPiped = function() {
 46 | 	// Spawn them in advance
 47 | 	while (jobId++ < opts.maxJobs) {
 48 | 		var proc = spawn(opts._);
 49 | 		proc.handled = 0;
 50 | 		proc.stdin.on('drain', function() {
 51 | 			this.writable = true;
 52 | 			flush();
 53 | 		}.bind(proc));
 54 | 	}
 55 | };
 56 | 
 57 | function flushPiped() {
 58 | 	while (buffer.length) {
 59 | 		for (var i = 0; i < opts.maxJobs; i++) {
 60 | 			var proc = procs[i];
 61 | 			// Skip job if not writable unless all input was received
 62 | 			if (!closing && proc.writable === false) {
 63 | 				continue;
 64 | 			}
 65 | 			proc.writable = proc.stdin.write(buffer.shift() + opts.eol);
 66 | 			proc.handled++;
 67 | 			// Round-Robin the input between the jobs
 68 | 			procs.splice(i, 1);
 69 | 			procs.push(proc);
 70 | 			break;
 71 | 		}
 72 | 		// No job handled
 73 | 		// TODO: pause input()
 74 | 		if (i === opts.maxJobs) break;
 75 | 	}
 76 | }
 77 | 
 78 | function flushWithArgs() {
 79 | 	var max = opts.maxArgs;
 80 | 	if (!max) {
 81 | 		// Wait until all lines are buffered
 82 | 		if (!closing) return;
 83 | 		// Distribute lines evenly among expected # of jobs
 84 | 		max = Math.ceil(buffer.length / opts.maxJobs);
 85 | 	}
 86 | 	var min = closing ? 1 : max;
 87 | 	while (buffer.length >= min) {
 88 | 		// TODO: pause input()
 89 | 		if (opts.maxJobs && procs.length === opts.maxJobs) {
 90 | 			return;
 91 | 		}
 92 | 
 93 | 		var lines = buffer.splice(0, max);
 94 | 		// Send the jobId that would be assigned to this job
 95 | 		var args = placeholders.parse(++jobId, lines);
 96 | 		// Job doesn't really run
 97 | 		if (opts.dryRun) {
 98 | 			// It's not exactly the same as above, must be quoted
 99 | 			// TODO: Still, there's a lot of repetition between this and jobId, refactor
100 | 			if (opts.shell) args = ['sh', '-c', '"'+args.join(' ')+'"'];
101 | 			console.log(args.join(' '));
102 | 			continue;
103 | 		}
104 | 
105 | 		var proc = spawn(args);
106 | 		proc.handled = lines.length;
107 | 		proc.on('exit', function() {
108 | 			procs.splice(procs.indexOf(this), 1);
109 | 			// Wait `delay` seconds before starting a new job
110 | 			setTimeout(flush, opts.delay * 1e3);
111 | 		});
112 | 	}
113 | }
114 | 
115 | function flush() {
116 | 	if (opts.pipeMode) {
117 | 		flushPiped();
118 | 	} else  {
119 | 		flushWithArgs();
120 | 	}
121 | 	if (closing && opts.bgMode && !buffer.length) {
122 | 		// If --bg, don't wait for jobs to exit
123 | 		// TODO: Is this working for all cases? alternative is proc.unref()
124 | 		process.nextTick(process.exit);
125 | 	}
126 | }
127 | 
128 | exports.processLine = function(line) {
129 | 	if (opts.trim) line = line.trim();
130 | 	// Ignore empty lines
131 | 	if (!line) return;
132 | 	if (opts.quote) line = '"'+line+'"';
133 | 	buffer.push(line);
134 | 	flush();
135 | };
136 | 
137 | exports.close = function() {
138 | 	closing = true;
139 | 	flush();
140 | 
141 | 	procs.forEach(function(proc) {
142 | 		proc.stdin.end();
143 | 	});
144 | 
145 | 	if (opts.verbose) {
146 | 		process.on('exit', function() {
147 | 			util.error('Elapsed time:', timer());
148 | 		});
149 | 	}
150 | };
151 | 


--------------------------------------------------------------------------------
/lib/opts.js:
--------------------------------------------------------------------------------
 1 | var os = require('os');
 2 | 
 3 | //- Defaults
 4 | exports.maxJobs = os.cpus().length;
 5 | exports.maxArgs = 1;
 6 | exports.lineSep = '\n';
 7 | exports.quote = false;
 8 | exports.trim = false;
 9 | exports.colSep = ' ';
10 | exports.pipeMode = false;
11 | exports.bgMode = false;
12 | exports.delay = 0;
13 | exports.timeout = 0;
14 | exports.src = process.stdin;
15 | exports.eol = '\n';/*os.EOL*/
16 | exports.verbose = false;
17 | exports.shell = false;
18 | exports.haltOnError = false;
19 | // Remaining arguments (the command)
20 | exports._ = null;
21 | 


--------------------------------------------------------------------------------
/lib/placeholders.js:
--------------------------------------------------------------------------------
 1 | var opts = require('./opts');
 2 | var crypto = require('crypto');
 3 | 
 4 | // See https://www.gnu.org/software/parallel/man.html#OPTIONS
 5 | const OPS = {
 6 | 	// {} input line
 7 | 	'': function(val) { return val; },
 8 | 	// {.} input line without extension
 9 | 	'.': function(val) { return val.replace(/\.[^.]*$/,''); },
10 | 	// {/} basename of the input line
11 | 	'/': function(val) { return val.split(/[\/\\]/).pop(); },
12 | 	// {//} dirname of the input line
13 | 	'//': function(val) { return val.split(/([\/\\])/).slice(0,-2).join(''); },
14 | 	// {/.} basename of the input line without extension
15 | 	'/.': function(val) { return OPS['.'](OPS['/'](val)); },
16 | 	// {#} sequence number of the job to run, [1, ]
17 | 	'#': function(_, jobId) { return jobId; },
18 | 	// {%} job slot number [1, --jobs]
19 |   '%': function (_, jobId) { return opts.maxJobs ? ((jobId - 1) % opts.maxJobs) + 1 : jobId },
20 | 
21 |   // These are not from the original
22 | 
23 |   // {..} extension of the input line
24 |   '..': function (val) { return val.split('.').pop() },
25 |   // {v} lower case the value
26 |   'v': function (val) { return val.toLowerCase() },
27 |   // {^} upper case the value
28 |   '^': function (val) { return val.toUpperCase() },
29 |   // {t} current time as a number
30 |   't': function () { return Date.now() },
31 |   // {T} current time in ISO as a string
32 |   'T': function () { return new Date().toISOString().replace(/\D+/g, '_').replace(/_$/, '') },
33 |   // {d} current date in ISO format
34 |   'd': function () { return new Date().toISOString().split('T')[0].replace(/\D+/g, '_') },
35 |   // {r} random number between 100000 and 999999
36 |   'r': function () { return 1e5 + Math.floor(Math.random() * 9e5) },
37 |   // {md5} MD5 hash of the input line
38 |   'md5': function (val) { return crypto.createHash('md5').update(val).digest('hex') },
39 | };
40 | 
41 | // Matches any operator above, optionally preceeded by a number, enclosed in curly brackets
42 | const REGEX = new RegExp('\\{(-?\\d+)?('+Object.keys(OPS).map(escape).join('|')+')}', 'g');
43 | 
44 | exports.parse = function(jobId, lines) {
45 | 	var any = false;
46 | 	var out = [];
47 | 	opts._.forEach(function(arg) {
48 | 		// No placeholder in this one
49 | 		if (!REGEX.test(arg)) {
50 | 			return out.push(arg);
51 | 		}
52 | 		any = true;
53 | 		lines.forEach(function(line) {
54 | 			out.push(replace(arg, line, jobId));
55 | 		});
56 | 	});
57 | 	// No placeholder, append input to end
58 | 	if (!any) out.push.apply(out, lines);
59 | 	return out;
60 | };
61 | 
62 | function replace(arg, line, jobId) {
63 | 	return arg.replace(REGEX, function(_, num, op) {
64 | 		var val = extract(line, num);
65 | 		return OPS[op](val, jobId);
66 | 	});
67 | }
68 | 
69 | function escape(op) {
70 |   return op.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
71 | }
72 | 
73 | function extract(line, num) {
74 | 	num = parseInt(num, 10);
75 | 	if (!num) return line;
76 | 	var cols = split(line);
77 | 	if (num < 0) {
78 | 		num = cols.length + num;
79 | 	} else {
80 | 		// They are 1-based
81 | 		num--;
82 | 	}
83 | 	return cols[num] || '';
84 | }
85 | 
86 | var sep;
87 | function split(line) {
88 | 	sep = sep || new RegExp(opts.colSep);
89 | 	// TODO: Support quoted columns and that kind of thing
90 | 	return line.replace(/^"|"$/g, '').split(sep);
91 | }


--------------------------------------------------------------------------------
/lib/util.js:
--------------------------------------------------------------------------------
 1 | exports.error = function(/*...args*/) {
 2 | 	var a = arguments;
 3 | 	a[0] = 'parallel: ' + a[0];
 4 | 	console.error.apply(console, a);
 5 | };
 6 | 
 7 | exports.fatal = function(err) {
 8 | 	var msg = err.stack || err.message || err;
 9 | 	if (msg.indexOf('EPIPE') === -1) {
10 | 		exports.error(msg);
11 | 	}
12 | 	process.exit(process.exitCode || 1);
13 | };
14 | 
15 | exports.timer = function() {
16 | 	var start = process.hrtime();
17 | 	return function() {
18 | 		var elapsed = process.hrtime(start);
19 | 		return (elapsed[0] + elapsed[1] / 1e9).toFixed(2) + 's';
20 | 	};
21 | };
22 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "parallel",
 3 |   "author": "Ariel Flesler <aflesler@gmail.com>",
 4 |   "version": "1.3.1",
 5 |   "description": "CLI tool to execute shell commands in parallel, based on GNU parallel command",
 6 |   "keywords": ["parallel", "cli", "concurrent", "commands", "gnu", "execute", "standard", "input", "stdin"],
 7 |   "license": "MIT",
 8 |   "homepage": "https://github.com/flesler/parallel",
 9 |   "bugs": "https://github.com/flesler/parallel/issues",
10 |   "repository": "git://github.com/flesler/parallel",
11 |   "main": "./bin/parallel.js",
12 |   "preferGlobal": true,
13 |   "bin": {
14 |     "parallel": "./bin/parallel.js"
15 |   },
16 |   "scripts": {
17 |     "test": "echo TODO: Tests"
18 |   },
19 |   "engines": {
20 |     "node": "*"
21 |   },
22 |   "dependencies": {},
23 |   "devDependencies": {}
24 | }
25 | 


--------------------------------------------------------------------------------