├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── bin └── pjs ├── lib ├── pjs.js └── utils.js ├── package.json └── spec ├── helpers.js ├── pjsSpec.js └── utilsSpec.js /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | coverage 3 | *.seed 4 | *.log 5 | *.csv 6 | *.dat 7 | *.out 8 | *.pid 9 | *.gz 10 | 11 | pids 12 | logs 13 | results 14 | 15 | npm-debug.log 16 | node_modules 17 | 18 | .grunt 19 | .DS_Store 20 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.12" 4 | - "0.10" 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Daniel St. Jules 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![pjs](http://danielstjules.com/github/pjs-logo.png) 2 | 3 | Pipeable JavaScript - another utility like sed/awk/wc... but with JS! Quickly 4 | filter, map and reduce from the command line. Features a streaming API. 5 | Inspired by pipeable ruby. 6 | 7 | [![Build Status](https://api.travis-ci.org/danielstjules/pjs.svg?branch=master)](https://travis-ci.org/danielstjules/pjs) 8 | 9 | ## Overview 10 | 11 | pjs is a cli tool that can accept input on stdin, or read from a list of files. 12 | Its filter, map and reduce options take expressions to be run, in that order, 13 | and applies them to the supplied input. The expressions themselves can contain 14 | identifiers used by keys in String.prototype, which will automatically be bound 15 | to the given line. This lets you save a bit of typing with your one-liners, 16 | while still giving you access to all your JS string functions! Check out some 17 | of the examples below to see how they translate. 18 | 19 | ``` bash 20 | # Return all lines longer than 5 chars 21 | # => lines.filter(function(line) { return line.length > 5; }); 22 | ls -1 | pjs -f 'length > 5' 23 | 24 | # Count characters in each line 25 | # => lines.map(function(line) { return line.length; }); 26 | ls -1 | pjs -m 'length' 27 | 28 | # Uppercase and pad each line 29 | # => lines.map(function(line) { return ' ' + line.toUpperCase()"; }); 30 | ls -1 | pjs -m '" " + toUpperCase()' 31 | 32 | # Return lines longer than 5 chars, and remove any digits 33 | # => lines 34 | # .filter(function(line) { return line.length > 5; }) 35 | # .map(function(line) { return line.replace(/\d/g, ''); }); 36 | ls -1 | pjs -f 'length > 5' -m 'replace(/\d/g, "")' 37 | ``` 38 | 39 | The current line and value can also be accessed via the `$` variable, and the 40 | tool supports json output. 41 | 42 | ``` bash 43 | (echo 'foo' && echo 'foobar') | pjs -jm '{name: $, length: length}' 44 | [ 45 | {"name":"foo","length":3}, 46 | {"name":"foobar","length":6} 47 | ] 48 | ``` 49 | 50 | pjs also includes lodash functions, which can be accessed via the `_` object, 51 | and chained using $$ 52 | 53 | ``` bash 54 | echo 'hello' | pjs -m '_.upperFirst($)' 55 | # Hello 56 | 57 | echo 'please-titleize-this-sentence' | \ 58 | pjs -m '$$.lowerCase().split(" ").map(_.upperFirst).join(" ")' 59 | # Please Titleize This Sentence 60 | ``` 61 | 62 | as well as Ramda and point-free style 63 | 64 | ``` bash 65 | echo 'please-titleize-this-sentence' | \ 66 | pjs -m "R.compose(R.replace(/(^|\s)\w/g, R.toUpper), R.replace(/-/g, ' '))" 67 | # Please Titleize This Sentence 68 | ``` 69 | 70 | ## Installation 71 | 72 | It can be installed via `npm` using: 73 | 74 | ``` 75 | npm install -g pipeable-js 76 | ``` 77 | 78 | ## Usage 79 | 80 | ``` 81 | Usage: pjs [options] [files ...] 82 | 83 | Functions and expressions are invoked in the following order: 84 | filter, map, reduce 85 | 86 | All functions are passed the line ($) and index (i) 87 | Built-in reduce functions: length, min, max, sum, avg, concat 88 | Custom reduce expressions accept: prev, curr, i, array 89 | Includes lodash (_), and can be chained using $$ 90 | Supports Ramda (R) and point-free style 91 | 92 | Options: 93 | 94 | -h, --help output usage information 95 | -V, --version output the version number 96 | -i, --ignore ignore empty lines 97 | -j, --json output as json 98 | -f, --filter filter by a boolean expression 99 | -m, --map map values using the expression 100 | -r, --reduce reduce using a function or expression 101 | ``` 102 | 103 | ## Examples 104 | 105 | ### filter 106 | 107 | ``` bash 108 | # Print all odd lines 109 | # awk 'NR % 2 == 1' file 110 | pjs -f 'i % 2 == 0' file 111 | 112 | # Print all lines greater than 80 chars in length 113 | # awk 'length($0) > 80' file 114 | pjs -f 'length > 80' file 115 | ``` 116 | 117 | ### map 118 | 119 | ``` bash 120 | # Remove all digits 121 | # tr -d 0-9 < file 122 | pjs -m "replace(/\d/g, '')" file 123 | 124 | # Get second item of each line in csv 125 | # awk -F "," '{print $2}' file 126 | pjs -m 'split(",")[1]' file 127 | ``` 128 | 129 | ### reduce 130 | 131 | ``` bash 132 | # Count lines in file 133 | # wc -l file 134 | # awk 'END { print NR }' file 135 | pjs -r length file 136 | 137 | # Sum all decimal numbers in a file 138 | # awk '{ sum += $1 } END { print sum }' file 139 | # perl -nle '$sum += $_ } END { print $sum' file 140 | pjs -r 'Number(prev) + Number(curr)' file 141 | pjs -r '(+prev) + (+curr)' file 142 | pjs -r sum file 143 | 144 | # Concatenate all lines in multiple files 145 | # awk '{printf $0;}' file1 file2 146 | # cat file1 file2 | tr -d '\n' 147 | pjs -r concat file1 file2 148 | ``` 149 | 150 | ### mixed 151 | 152 | ``` bash 153 | # Print the length of the longest line 154 | # awk '{ if (length($0) > max) max = length($0) } END { print max }' file 155 | pjs -m 'length' -r max file 156 | ``` 157 | 158 | ## Comparison 159 | 160 | | Features | pjs | pythonpy | pru | 161 | |-----------------------|------------|------------------------|------------| 162 | | Language | JavaScript | Python | Ruby | 163 | | Streaming | Yes | Limited [1] | Yes | 164 | | Implementation | Streams | Iterables | Generators | 165 | | Easy JSON output | Yes | No | No | 166 | | WebscaleTM | YES | No | No | 167 | 168 | [1] Can't perform "tail -f logfile | py -x x" 169 | -------------------------------------------------------------------------------- /bin/pjs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var fs = require('fs'); 4 | var program = require('commander'); 5 | var split = require('split'); 6 | var async = require('async'); 7 | var pjs = require('../lib/pjs'); 8 | 9 | var summ = " Functions and expressions are invoked in the following order:\n" + 10 | " filter, map, reduce\n\n" + 11 | " All functions are passed the line ($) and index (i)\n" + 12 | " Built-in reduce functions: length, min, max, sum, avg, concat\n" + 13 | " Custom reduce expressions accept: prev, curr, i, array\n" + 14 | " Includes lodash (_), and can be chained using $$\n"+ 15 | " Supports Ramda (R) and point-free style" 16 | 17 | program 18 | .version(require('../package.json').version) 19 | .usage("[options] [files ...]\n\n" + summ) 20 | .option('-i, --ignore', 'ignore empty lines') 21 | .option('-j, --json', 'output as json') 22 | .option('-f, --filter ', 'filter by a boolean expression') 23 | .option('-m, --map ', 'map values using the expression') 24 | .option('-r, --reduce ', 25 | 'reduce using a function or expression') 26 | .parse(process.argv); 27 | 28 | process.on('uncaughtException', function(e) { 29 | // Handle errors in expression syntax 30 | if (e instanceof SyntaxError || e instanceof ReferenceError) { 31 | console.error('Invalid expression :', e.message); 32 | process.exit(3); 33 | } 34 | 35 | console.error(e.stack); 36 | process.exit(1); 37 | }); 38 | 39 | // Print help if no actions were provided 40 | if (!program.filter && !program.map && 41 | !program.reduce && !program.json) { 42 | program.help(); 43 | } 44 | 45 | // Assume all unconsumed arguments are paths 46 | var paths = program.args || []; 47 | 48 | // Pipe input and process 49 | var dstStream = getDestination(); 50 | if (paths.length) { 51 | pipeFiles(paths, dstStream); 52 | } else { 53 | process.stdin.pipe(dstStream); 54 | } 55 | 56 | /** 57 | * Returns the destination stream to be used by a readable stream, such as 58 | * stdin or a series of files. The destination stream is a split stream that 59 | * pipes to the various actions, and finally, stdout. 60 | * 61 | * @return {Stream} The destination stream 62 | */ 63 | function getDestination() { 64 | var prev, splitStream, ignoreStream, stream, outputString, opts; 65 | splitStream = split(); 66 | 67 | // Remove trailing line, and ignore empty lines if required 68 | prev = pjs.ignore(!!program.ignore); 69 | splitStream.pipe(prev); 70 | 71 | // Since we're using object mode, we have to track when to 72 | // output a string if writing to stdout 73 | outputString = { 74 | filter: !(program.map || program.reduce || program.json), 75 | map: !(program.reduce || program.json), 76 | reduce: !program.json 77 | }; 78 | 79 | // Stream pipe options 80 | opts = { 81 | reduce: {end: false} 82 | }; 83 | 84 | // Pipe to each action, when set 85 | ['filter', 'map', 'reduce'].forEach(function(action) { 86 | var userScript = program[action]; 87 | if (userScript) { 88 | userScript = supportRamdaPointFree(userScript); 89 | stream = pjs[action](userScript, outputString[action]); 90 | 91 | prev.pipe(stream, opts[action]); 92 | prev = stream; 93 | } 94 | }); 95 | 96 | // Add the json stream, if enabled 97 | if (program.json) { 98 | stream = pjs.json(!program.reduce); 99 | prev.pipe(stream); 100 | prev = stream; 101 | } 102 | 103 | prev.pipe(process.stdout); 104 | 105 | return splitStream; 106 | } 107 | 108 | /** 109 | * Iterates over the given paths, reading the files and piping their data to 110 | * the passed destination stream. 111 | * 112 | * @param {string[]} path The file paths 113 | * @param {Stream} dstStream The destination stream 114 | */ 115 | function pipeFiles(paths, dstStream) { 116 | // Keep the destination open between files 117 | var opts = {end: false}; 118 | 119 | async.eachSeries(paths, function(path, next) { 120 | fs.createReadStream(path) 121 | .on('end', next) 122 | .pipe(dstStream, opts); 123 | }, function () { 124 | dstStream.end(); 125 | }); 126 | } 127 | 128 | /** 129 | * Append the input token as a function call if the script expression is using 130 | * pointfree style 131 | * 132 | * @param {string} script The user-defined script 133 | * @return {string} The script with the added invocation or the original script 134 | */ 135 | function supportRamdaPointFree(script) { 136 | // Append the invocation if the script starts with a ramda function and it 137 | // doesn't include the call already 138 | return ~script.indexOf('R.') && !(/\$\);?$/.test(script)) 139 | ? script + '($)' 140 | : script; 141 | } 142 | -------------------------------------------------------------------------------- /lib/pjs.js: -------------------------------------------------------------------------------- 1 | /** 2 | * The pjs library module exports five functions: filter, map, reduce, 3 | * ignore and json. 4 | */ 5 | 6 | var stream = require('stream'); 7 | var Stringify = require('streaming-json-stringify'); 8 | var StringDecoder = require('string_decoder').StringDecoder; 9 | var inspect = require('util').inspect; 10 | var utils = require('./utils'); 11 | var _nullObject = utils._nullObject; 12 | var _ = require('lodash'); 13 | var R = require('ramda'); 14 | 15 | /** 16 | * Accepts an expression to evaluate, and two booleans. The first indicates 17 | * whether or not to cast output to a string, while the second requires that 18 | * "$" be used to refer to the current element. Given these arguments, a stream 19 | * is returned, behaving much like array.filter(). The expression is passed 20 | * two arguments: $, the line, and i, its index. It evaluates the expression 21 | * for each piped value, streaming those which resulted in a truthy expression. 22 | * 23 | * @param {string} expression The expression to evaluate 24 | * @param {bool} outputString Whether or not to output strings 25 | * returns {Transform} A transform stream in objectMode 26 | */ 27 | exports.filter = function(expression, outputString) { 28 | var i, include, filterStream; 29 | 30 | i = 0; 31 | include = function($, i) { 32 | with ($) { 33 | var $$ = _.chain($); 34 | var _result = eval(expression); 35 | _result = unpackIfLodash(_result); 36 | return _result; 37 | } 38 | }; 39 | 40 | filterStream = new stream.Transform({objectMode: true}); 41 | filterStream._transform = function(chunk, encoding, fn) { 42 | if (!include(chunk, i++)) return fn(); 43 | if (outputString) chunk = String(chunk) + '\n'; 44 | 45 | this.push(chunk); 46 | return fn(); 47 | }; 48 | 49 | return filterStream; 50 | }; 51 | 52 | /** 53 | * Accepts an expression to evaluate, and two booleans. The first indicates 54 | * whether or not to cast output to a string, while the second requires that 55 | * "$" be used to refer to the current element. Given these arguments, a stream 56 | * is returned, behaving much like array.map(). The expression is passed 57 | * two arguments: $, the line, and i, its index. It streams the modified values 58 | * as a result of the evaluated expression. 59 | * 60 | * @param {string} expression The expression to evaluate 61 | * @param {bool} outputString Whether or not to output strings 62 | * returns {Transform} A transform stream in objectMode 63 | */ 64 | exports.map = function(expression, outputString) { 65 | var i, update, mapStream; 66 | 67 | i = 0; 68 | update = function($, i) { 69 | with ($) { 70 | var $$ = _.chain($); 71 | var _result = eval('(' + expression + ')'); 72 | _result = unpackIfLodash(_result); 73 | return (_result === null) ? _nullObject : _result; 74 | } 75 | }; 76 | 77 | if (outputString) { 78 | update = function($, i) { 79 | with ($) { 80 | var $$ = _.chain($); 81 | var res = eval('(' + expression + ')'); 82 | res = unpackIfLodash(res); 83 | if (typeof res === 'string') return res + '\n'; 84 | return inspect(res) + '\n'; 85 | } 86 | }; 87 | } 88 | 89 | mapStream = new stream.Transform({objectMode: true}); 90 | mapStream._transform = function(chunk, encoding, fn) { 91 | this.push(update(chunk, i++)); 92 | return fn(); 93 | }; 94 | 95 | return mapStream; 96 | }; 97 | 98 | /** 99 | * Accepts an array and an expression to evaluate. It invokes a built-in 100 | * function if expression is one of: length, min, max, sum, avg or concat. 101 | * Otherwise, it evaluates the passed expression, passing it as the callback 102 | * to reduce. Must be piped to with end set to false. 103 | * 104 | * @param {*[]} array An array to reduce 105 | * @param {string} expression The expression to evaluate 106 | * returns {*} A reduced value 107 | */ 108 | exports.reduce = function(expression, outputString) { 109 | var accumulator, performReduce, collectStream; 110 | 111 | accumulator = []; 112 | performReduce = function() { 113 | var builtin = ['length', 'min', 'max', 'sum', 'avg', 'concat']; 114 | if (builtin.indexOf(expression) !== -1) { 115 | return utils[expression](accumulator); 116 | } 117 | 118 | return accumulator.reduce(function(prev, curr, i, array) { 119 | if (prev === _nullObject) { 120 | prev = null; 121 | } 122 | if (curr === _nullObject) { 123 | curr = null; 124 | } 125 | 126 | return eval(expression); 127 | }); 128 | }; 129 | 130 | collectStream = new stream.Transform({objectMode: true}); 131 | collectStream._transform = function(chunk, encoding, fn) { 132 | accumulator.push(chunk); 133 | return fn(); 134 | }; 135 | 136 | collectStream.on('pipe', function(src) { 137 | src.on('end', function() { 138 | var result = performReduce(); 139 | if (outputString) { 140 | result = String(result) + '\n'; 141 | } else if (result === null) { 142 | result = _nullObject; 143 | } 144 | 145 | collectStream.push(result); 146 | collectStream.end(); 147 | }); 148 | }); 149 | 150 | return collectStream; 151 | }; 152 | 153 | /** 154 | * Returns a Transform stream that ignores the last empty line in a file 155 | * created by splitting on newlines. If ignoreEmpty is true, all empty lines 156 | * are ignored. 157 | * 158 | * @param {bool} ignoreEmpty Whether or not to ignore all empty lines 159 | * @return {Transform} A transform stream in object mode 160 | */ 161 | exports.ignore = function(ignoreEmpty) { 162 | var emptyFlag, ignoreStream; 163 | 164 | ignoreStream = new stream.Transform({objectMode: true}); 165 | ignoreStream._transform = function(chunk, encoding, fn) { 166 | if (emptyFlag) { 167 | this.push(''); 168 | emptyFlag = false; 169 | } 170 | 171 | if (chunk !== '') { 172 | this.push(chunk); 173 | } else if (!ignoreEmpty) { 174 | emptyFlag = true; 175 | } 176 | 177 | return fn(); 178 | }; 179 | 180 | return ignoreStream; 181 | }; 182 | 183 | /** 184 | * Returns a Transform stream that can stringify any piped objects or arrays. 185 | * If an array is to be piped, an instance of json-array-stream is returned 186 | * which will stringify members individually. Otherwise, a transform stream is 187 | * returned that will buffer the full object before applying JSON.stringify. 188 | * 189 | * @param {bool} streamArray Whether or not an array will be streamed 190 | * @return {Transform} A transform stream in object mode 191 | */ 192 | exports.json = function(streamArray) { 193 | var stringify, jsonStream, i; 194 | 195 | // Stream array members 196 | if (streamArray) { 197 | return getStringifyStream(); 198 | } 199 | 200 | // Stringify entire object 201 | jsonStream = new stream.Transform({objectMode: true}); 202 | jsonStream._transform = function(chunk, encoding, fn) { 203 | if (chunk === _nullObject) { 204 | chunk = null; 205 | } 206 | 207 | this.push(JSON.stringify(chunk)); 208 | return fn(); 209 | }; 210 | 211 | return jsonStream; 212 | }; 213 | 214 | /** 215 | * Returns an instance of json-array-stream, or Stringify. The instance has 216 | * been updated to handle the ",\n" as a delimiter, as well as convert 217 | * _nullObject to null, prior to calling JSON.stringify. 218 | * 219 | * @returns {Stringify} The stringify stream 220 | */ 221 | function getStringifyStream() { 222 | stringify = new Stringify(); 223 | stringify.seperator = new Buffer(',\n', 'utf8'); 224 | 225 | // Overwrite default _transform to convert _nullObject 226 | stringify._transform = function(doc, enc, fn) { 227 | if (this.destroyed) return; 228 | 229 | if (this.started) { 230 | this.push(this.seperator); 231 | } else { 232 | this.push(this.open); 233 | this.started = true; 234 | } 235 | 236 | if (doc === _nullObject) { 237 | doc = null; 238 | } 239 | 240 | try { 241 | doc = JSON.stringify(doc, this.replacer, this.space); 242 | } catch (err) { 243 | return fn(err); 244 | } 245 | 246 | this.push(new Buffer(doc, 'utf8')); 247 | fn(); 248 | }; 249 | 250 | return stringify; 251 | } 252 | 253 | /** 254 | * If given a lodash object, the object's value is returned. Otherwise the 255 | * original argument is returned. 256 | * 257 | * @param {*} obj 258 | * @returns {*} 259 | */ 260 | function unpackIfLodash(obj) { 261 | return (obj && obj.constructor.name === 'LodashWrapper') ? obj.value() : obj; 262 | } 263 | -------------------------------------------------------------------------------- /lib/utils.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Exports array utility functions, as well as a null object. 3 | */ 4 | 5 | var utils = {}; 6 | module.exports = utils; 7 | 8 | /** 9 | * A null object representing a null value in a stream, so as not to terminate 10 | * an object stream. 11 | * 12 | * @private 13 | * 14 | * @var {object} 15 | */ 16 | var _nullObject = utils._nullObject = {value: null}; 17 | 18 | /** 19 | * Returns the length of the array. 20 | * 21 | * @param {*[]} array 22 | * @returns {Number} Length of the array 23 | */ 24 | utils.length = function(array) { 25 | return array.length; 26 | }; 27 | 28 | /** 29 | * Returns the minimum value in an array. Mimics Math.min by casting values 30 | * to a number, including the nullObject/null. 31 | * 32 | * @param {*[]} array The array for which to find its min 33 | * @returns {Number} Its minimum value 34 | */ 35 | utils.min = function(array) { 36 | var i, min, curr; 37 | min = (array[0] === _nullObject) ? 0 : Number(array[0]); 38 | 39 | for (i = 1; i < array.length; i++) { 40 | if (array[i] === _nullObject) { 41 | array[i] = 0; 42 | } 43 | 44 | curr = Number(array[i]); 45 | if (curr < min) { 46 | min = curr; 47 | } 48 | } 49 | 50 | return min; 51 | }; 52 | 53 | /** 54 | * Returns the maximum value in an array. Mimics Math.max by casting values 55 | * to a number, including the nullObject/null. 56 | * 57 | * @param {*[]} array The array for which to find its max 58 | * @returns {Number} Its maximum value 59 | */ 60 | utils.max = function(array) { 61 | var i, max, curr; 62 | max = (array[0] === _nullObject) ? 0 : Number(array[0]); 63 | 64 | for (i = 1; i < array.length; i++) { 65 | if (array[i] === _nullObject) { 66 | array[i] = 0; 67 | } 68 | 69 | curr = Number(array[i]); 70 | if (curr > max) { 71 | max = curr; 72 | } 73 | } 74 | 75 | return max; 76 | }; 77 | 78 | /** 79 | * Casts each element of the provided array to a Number and returns their sum. 80 | * 81 | * @param {*[]} array The array for which to calculate the sum 82 | * @returns {Number} The sum of its elements 83 | */ 84 | utils.sum = function(array) { 85 | return array.reduce(function(curr, prev) { 86 | if (curr === _nullObject) { 87 | curr = null; 88 | } 89 | if (prev === _nullObject) { 90 | prev = null; 91 | } 92 | 93 | return Number(curr) + Number(prev); 94 | }); 95 | }; 96 | 97 | /** 98 | * Casts each element of the array to a Number, and returns their mean value. 99 | * 100 | * @param {*[]} array The array for which to calculate the average 101 | * @returns {Number} The mean of its elements 102 | */ 103 | utils.avg = function(array) { 104 | return utils.sum(array) / array.length; 105 | }; 106 | 107 | /** 108 | * Given an array, casts each element to a String and returns their 109 | * concatenated value. 110 | * 111 | * @param {*[]} array The array for which to concatenate its values 112 | * @returns {String} The concatenated string 113 | */ 114 | utils.concat = function(array) { 115 | return array.reduce(function(curr, prev) { 116 | if (curr === _nullObject) { 117 | curr = null; 118 | } 119 | if (prev === _nullObject) { 120 | prev = null; 121 | } 122 | 123 | return String(curr) + String(prev); 124 | }); 125 | }; 126 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pipeable-js", 3 | "version": "0.10.0", 4 | "description": "Pipeable JavaScript", 5 | "keywords": [ 6 | "pipe", 7 | "pipeable", 8 | "stdin", 9 | "stdout", 10 | "map", 11 | "reduce" 12 | ], 13 | "author": "Daniel St. Jules ", 14 | "licenses": [ 15 | "MIT" 16 | ], 17 | "homepage": "https://github.com/danielstjules/pjs", 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/danielstjules/pjs.git" 21 | }, 22 | "dependencies": { 23 | "async": "*", 24 | "commander": "*", 25 | "lodash": "^4.3.0", 26 | "ramda": "^0.22.1", 27 | "split": "0.3.0", 28 | "streaming-json-stringify": "1.0.0" 29 | }, 30 | "devDependencies": { 31 | "mocha": "*", 32 | "expect.js": "*", 33 | "mock-fs": "*", 34 | "async": "*" 35 | }, 36 | "bin": { 37 | "pjs": "./bin/pjs" 38 | }, 39 | "scripts": { 40 | "test": "mocha --recursive -R spec spec" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /spec/helpers.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Helper functions for creating and testing pjs streams. 3 | */ 4 | 5 | var stream = require('stream'); 6 | 7 | var arrayStream, testStream; 8 | 9 | exports.arrayStream = arrayStream = function(array) { 10 | array = array.slice(0); 11 | var readable = new stream.Readable({objectMode: true}); 12 | 13 | readable._read = function() { 14 | this.push(array.shift()); 15 | if (!array.length) this.push(null); 16 | }; 17 | 18 | return readable; 19 | }; 20 | 21 | exports.testStream = testStream = function(array, transform, opts, fn) { 22 | array = array.slice(0); 23 | var readable = arrayStream(array); 24 | var result = []; 25 | 26 | var dst = new stream.Writable({objectMode: true}); 27 | dst._write = function(chunk, encoding, next) { 28 | result.push(chunk); 29 | next(); 30 | }; 31 | 32 | dst.on('error', fn); 33 | dst.on('finish', function() { 34 | fn(null, result); 35 | }); 36 | 37 | readable.pipe(transform, opts).pipe(dst); 38 | }; 39 | -------------------------------------------------------------------------------- /spec/pjsSpec.js: -------------------------------------------------------------------------------- 1 | var expect = require('expect.js'); 2 | var pjs = require('../lib/pjs.js'); 3 | var helpers = require('./helpers'); 4 | var testStream = helpers.testStream; 5 | 6 | describe('pjs', function() { 7 | var lines = ['a', 'b', 'foo', 'bar']; 8 | 9 | describe('filter', function() { 10 | it('does not modify the array if the exp is always true', function(done) { 11 | var filter = pjs.filter('true'); 12 | 13 | testStream(lines, filter, {}, function(err, res) { 14 | expect(err).to.be(null); 15 | expect(res).to.eql(lines); 16 | done(); 17 | }); 18 | }); 19 | 20 | it('binds any string prototype keys to the line in question', function(done) { 21 | var filter = pjs.filter('length === 3'); 22 | 23 | testStream(lines, filter, {}, function(err, res) { 24 | expect(err).to.be(null); 25 | expect(res).to.eql(['foo', 'bar']); 26 | done(); 27 | }); 28 | }); 29 | 30 | it("passes the line's index, i, to the function", function(done) { 31 | var filter = pjs.filter('i % 2 === 0'); 32 | 33 | testStream(lines, filter, {}, function(err, res) { 34 | expect(err).to.be(null); 35 | expect(res).to.eql(['a', 'foo']); 36 | done(); 37 | }); 38 | }); 39 | 40 | it('outputs newline delimited results if outputString is true', function(done) { 41 | var filter = pjs.filter('3 !== length', true); 42 | 43 | testStream(lines, filter, {}, function(err, res) { 44 | expect(err).to.be(null); 45 | expect(res).to.eql(["a\n", "b\n"]); 46 | done(); 47 | }); 48 | }); 49 | }); 50 | 51 | describe('map', function() { 52 | it('modifies the array with the given expression', function(done) { 53 | var map = pjs.map('"i"'); 54 | 55 | testStream(lines, map, {}, function(err, res) { 56 | expect(err).to.be(null); 57 | expect(res).to.eql(['i', 'i', 'i', 'i']); 58 | done(); 59 | }); 60 | }); 61 | 62 | it('binds any string prototype keys to the line in question', function(done) { 63 | var map = pjs.map('toUpperCase()'); 64 | 65 | testStream(lines, map, {}, function(err, res) { 66 | expect(err).to.be(null); 67 | expect(res).to.eql(['A', 'B', 'FOO', 'BAR']); 68 | done(); 69 | }); 70 | }); 71 | 72 | it("passes the line's index, i, to the function", function(done) { 73 | var map = pjs.map('i'); 74 | 75 | testStream(lines, map, {}, function(err, res) { 76 | expect(err).to.be(null); 77 | expect(res).to.eql(['0', '1', '2', '3']); 78 | done(); 79 | }); 80 | }); 81 | 82 | it('outputs newline delimited results if outputString is true', function(done) { 83 | var map = pjs.map('toLowerCase()', true); 84 | 85 | testStream(lines, map, {}, function(err, res) { 86 | expect(err).to.be(null); 87 | expect(res).to.eql(["a\n", "b\n", "foo\n", "bar\n"]); 88 | done(); 89 | }); 90 | }); 91 | 92 | it('can reference the line as "$"', function(done) { 93 | var map = pjs.map('$.charAt(0)', false); 94 | 95 | testStream(lines, map, {}, function(err, res) { 96 | expect(err).to.be(null); 97 | expect(res).to.eql(['a', 'b', 'f', 'b']); 98 | done(); 99 | }); 100 | }); 101 | 102 | it('encodes null values as an object to avoid ending the stream', function(done) { 103 | var map = pjs.map('null', false, true); 104 | 105 | testStream(lines, map, {}, function(err, res) { 106 | expect(err).to.be(null); 107 | expect(res).to.eql([ 108 | {value: null}, 109 | {value: null}, 110 | {value: null}, 111 | {value: null} 112 | ]); 113 | done(); 114 | }); 115 | }); 116 | 117 | it('treats input as expressions, accepting object literals', function(done) { 118 | var map = pjs.map('{length: length}'); 119 | 120 | testStream(lines, map, {}, function(err, res) { 121 | expect(err).to.be(null); 122 | expect(res).to.eql([ 123 | {length: 1}, 124 | {length: 1}, 125 | {length: 3}, 126 | {length: 3} 127 | ]); 128 | done(); 129 | }); 130 | }); 131 | 132 | it('it supports lodash functions', function(done) { 133 | var map = pjs.map('_.upperFirst($)'); 134 | 135 | testStream(lines, map, {}, function(err, res) { 136 | expect(err).to.be(null); 137 | expect(res).to.eql(['A', 'B', 'Foo', 'Bar']); 138 | done(); 139 | }); 140 | }); 141 | 142 | it('it creates a lodash chain when using $$', function(done) { 143 | var map = pjs.map('$$.lowerCase().split(" ").map(_.upperFirst).join(" ")'); 144 | 145 | testStream(['implementing_titleize', 'forWords'], map, {}, function(err, res) { 146 | expect(err).to.be(null); 147 | expect(res).to.eql(['Implementing Titleize', 'For Words']); 148 | done(); 149 | }); 150 | }); 151 | }); 152 | 153 | describe('reduce', function() { 154 | it('returns the length when passed as the expression', function(done) { 155 | var reduce = pjs.reduce('length'); 156 | 157 | testStream([1, 2, 3], reduce, {end: false}, function(err, res) { 158 | expect(err).to.be(null); 159 | expect(res[0]).to.eql(3); 160 | done(); 161 | }); 162 | }); 163 | 164 | it('returns the min when passed as the expression', function(done) { 165 | var reduce = pjs.reduce('min'); 166 | 167 | testStream([2, 4, 8], reduce, {end: false}, function(err, res) { 168 | expect(err).to.be(null); 169 | expect(res[0]).to.eql(2); 170 | done(); 171 | }); 172 | }); 173 | 174 | it('returns the max when passed as the expression', function(done) { 175 | var reduce = pjs.reduce('max'); 176 | 177 | testStream([2, 4, 8], reduce, {end: false}, function(err, res) { 178 | expect(err).to.be(null); 179 | expect(res[0]).to.eql(8); 180 | done(); 181 | }); 182 | }); 183 | 184 | it('returns the sum when passed as the expression', function(done) { 185 | var reduce = pjs.reduce('sum'); 186 | 187 | testStream([1, 2, 3], reduce, {end: false}, function(err, res) { 188 | expect(err).to.be(null); 189 | expect(res[0]).to.eql(6); 190 | done(); 191 | }); 192 | }); 193 | 194 | it('returns the avg when passed as the expression', function(done) { 195 | var reduce = pjs.reduce('avg'); 196 | 197 | testStream([1, 2, 3], reduce, {end: false}, function(err, res) { 198 | expect(err).to.be(null); 199 | expect(res[0]).to.eql(2); 200 | done(); 201 | }); 202 | }); 203 | 204 | it('returns the concatenated string when passed "concat"', function(done) { 205 | var reduce = pjs.reduce('concat'); 206 | 207 | testStream([1, 2, 3], reduce, {end: false}, function(err, res) { 208 | expect(err).to.be(null); 209 | expect(res[0]).to.eql('123'); 210 | done(); 211 | }); 212 | }); 213 | 214 | it('accepts a custom expression, passing prev and curr', function(done) { 215 | var reduce = pjs.reduce('prev + curr'); 216 | 217 | testStream([1, 2, 3], reduce, {end: false}, function(err, res) { 218 | expect(err).to.be(null); 219 | expect(res[0]).to.eql(6); 220 | done(); 221 | }); 222 | }); 223 | 224 | it('accepts a custom expression, also passing i and array', function(done) { 225 | var reduce = pjs.reduce('3 * array[i]'); 226 | 227 | testStream([1, 2, 3], reduce, {end: false}, function(err, res) { 228 | expect(err).to.be(null); 229 | expect(res[0]).to.eql(9); 230 | done(); 231 | }); 232 | }); 233 | 234 | it('encodes null values as an object to avoid ending the stream', function(done) { 235 | var reduce = pjs.reduce('null'); 236 | 237 | testStream([1, 2, 3], reduce, {end: false}, function(err, res) { 238 | expect(err).to.be(null); 239 | expect(res[0]).to.eql({value: null}); 240 | done(); 241 | }); 242 | }); 243 | }); 244 | 245 | describe('ignore', function() { 246 | it('ignores the last empty line resulting from split', function(done) { 247 | var ignore = pjs.ignore(); 248 | 249 | testStream(['a', '', 'c', ''], ignore, {}, function(err, res) { 250 | expect(err).to.be(null); 251 | expect(res).to.eql(['a', '', 'c']); 252 | done(); 253 | }); 254 | }); 255 | 256 | it('ignores all empty lines when ignoreEmpty is true', function(done) { 257 | var ignore = pjs.ignore(true); 258 | 259 | testStream(['a', '', 'c', ''], ignore, {}, function(err, res) { 260 | expect(err).to.be(null); 261 | expect(res).to.eql(['a', 'c']); 262 | done(); 263 | }); 264 | }); 265 | }); 266 | 267 | describe('json', function() { 268 | it('streams a single object when streamArray is false', function(done) { 269 | var json = pjs.json(); 270 | 271 | testStream([{test: 'value'}], json, {}, function(err, res) { 272 | expect(err).to.be(null); 273 | expect(res).to.eql(['{"test":"value"}']); 274 | done(); 275 | }); 276 | }); 277 | 278 | it('streams a string json array when streamArray is true', function(done) { 279 | var array = [{test: 'object1'}, {test: 'object2'}]; 280 | var json = pjs.json(true); 281 | 282 | testStream(array, json, {}, function(err, res) { 283 | res = res.map(function(buffer) { 284 | return buffer.toString(); 285 | }); 286 | 287 | expect(err).to.be(null); 288 | expect(res).to.eql([ 289 | '[\n', 290 | '{"test":"object1"}', 291 | ',\n', 292 | '{"test":"object2"}', 293 | '\n]\n' 294 | ]); 295 | done(); 296 | }); 297 | }); 298 | }); 299 | }); 300 | -------------------------------------------------------------------------------- /spec/utilsSpec.js: -------------------------------------------------------------------------------- 1 | var expect = require('expect.js'); 2 | var utils = require('../lib/utils.js'); 3 | var _nullObject = utils._nullObject; 4 | 5 | describe('utils', function() { 6 | describe('length', function() { 7 | it('returns the length of an array', function() { 8 | var result = utils.length([2, 4, 8]); 9 | expect(result).to.be(3); 10 | }); 11 | }); 12 | 13 | describe('min', function() { 14 | it('returns the min value in an array', function() { 15 | var result = utils.min([2, 4, 8]); 16 | expect(result).to.be(2); 17 | }); 18 | 19 | it('converts a null object to null', function() { 20 | var result = utils.min([_nullObject, 1, 2]); 21 | expect(result).to.be(0); 22 | }); 23 | }); 24 | 25 | describe('max', function() { 26 | it('returns the max value in an array', function() { 27 | var result = utils.max([2, 4, 8]); 28 | expect(result).to.be(8); 29 | }); 30 | 31 | it('converts a null object to null', function() { 32 | var result = utils.max([-2, _nullObject, -1]); 33 | expect(result).to.be(0); 34 | }); 35 | }); 36 | 37 | describe('sum', function() { 38 | it('calculates the sum of the elements in an array', function() { 39 | var result = utils.sum([1, 2, 3, 4]); 40 | expect(result).to.be(10); 41 | }); 42 | 43 | it('casts values to numbers', function() { 44 | var result = utils.sum([true, true, '2.5']); 45 | expect(result).to.be(4.5); 46 | }); 47 | 48 | it('converts a null object to null', function() { 49 | var result = utils.sum([2, _nullObject, -1]); 50 | expect(result).to.be(1); 51 | }); 52 | }); 53 | 54 | describe('avg', function() { 55 | it('calculates the avg of the elements in an array', function() { 56 | var result = utils.avg([1, 2, 3, 4]); 57 | expect(result).to.be(2.5); 58 | }); 59 | 60 | it('casts values to numbers', function() { 61 | var result = utils.avg([true, true, '1.5', '2.5']); 62 | expect(result).to.be(1.5); 63 | }); 64 | 65 | it('converts a null object to null', function() { 66 | var result = utils.avg([2, 4, _nullObject]); 67 | expect(result).to.be(2); 68 | }); 69 | }); 70 | 71 | describe('concat', function() { 72 | it('concatenates the elements in an array', function() { 73 | var result = utils.concat(['a', 'b', 'c']); 74 | expect(result).to.be('abc'); 75 | }); 76 | 77 | it('casts values to strings', function() { 78 | var result = utils.concat(['foo', true, 0]); 79 | expect(result).to.be('footrue0'); 80 | }); 81 | 82 | it('converts a null object to null', function() { 83 | var result = utils.concat(['foo', 'bar', _nullObject]); 84 | expect(result).to.be('foobarnull'); 85 | }); 86 | }); 87 | }); 88 | --------------------------------------------------------------------------------