├── .gitignore
├── gasket.png
├── test.js
├── help.txt
├── package.json
├── readme.md
├── bin.js
└── index.js


/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | node_modules
3 | .dat


--------------------------------------------------------------------------------
/gasket.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/dat-ecosystem-archive/gasket/HEAD/gasket.png


--------------------------------------------------------------------------------
/test.js:
--------------------------------------------------------------------------------
 1 | var gasket = require('./')
 2 | var through = require('through2')
 3 | console.log('running pipeline...')
 4 | 
 5 | var pipeline = {
 6 |   'example': [{
 7 |     'type': 'pipe',
 8 |     'command': 'echo hello world'
 9 |   }, {
10 |     'type': 'pipe',
11 |     'command': 'transform-uppercase'
12 |   }]
13 | }
14 | 
15 | var pipelines = gasket(pipeline)
16 | var stringifier = through.obj(function (buff, enc, next) {
17 |   next(null, buff.toString())
18 | })
19 | 
20 | pipelines.run('example').pipe(stringifier).pipe(process.stdout)
21 | 
22 | stringifier.on('finish', function () {
23 |   console.log('stringify finish')
24 | })
25 | 


--------------------------------------------------------------------------------
/help.txt:
--------------------------------------------------------------------------------
 1 | Usage: gasket [command]
 2 | 
 3 | Available commands are:
 4 |   run [pipe]            run a pipe (defaults to main)
 5 |   pipe [pipe]           pipe to a pipeline (keeps stdin open)
 6 |   exec [script]         execute a script in the local context
 7 |   add [pipe] [script]   add a script to a pipe
 8 |   rm [pipe]             remove a pipe
 9 |   ls                    list available pipes
10 |   completion            setup auto completion
11 |   help                  print this help
12 |   version               print the installed version
13 | 
14 | Per default ./package.json or ./gasket.json is used to find pipelines
15 | Use -c,--config to use another pipeline config


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "gasket",
 3 |   "version": "2.0.1",
 4 |   "description": "Preconfigured pipelines for node.js",
 5 |   "main": "index.js",
 6 |   "scripts": {
 7 |     "test": "standard && node test.js"
 8 |   },
 9 |   "repository": {
10 |     "type": "git",
11 |     "url": "https://github.com/datproject/gasket"
12 |   },
13 |   "bin": {
14 |     "gasket": "bin.js"
15 |   },
16 |   "license": "BSD-2-Clause",
17 |   "dependencies": {
18 |     "debug-stream": "2.0.2",
19 |     "duplexify": "3.2.0",
20 |     "multistream": "1.4.2",
21 |     "ndjson": "1.2.3",
22 |     "npm-execspawn": "1.0.6",
23 |     "pumpify": "1.3.3",
24 |     "resolve": "0.7.1",
25 |     "tabalot": "0.6.0",
26 |     "xtend": "3.0.0",
27 |     "parallel-multistream": "1.0.1"
28 |   },
29 |   "gasket": {
30 |     "test": [
31 |       "cat -"
32 |     ]
33 |   },
34 |   "bugs": {
35 |     "url": "https://github.com/datproject/gasket/issues"
36 |   },
37 |   "homepage": "https://github.com/datproject/gasket",
38 |   "devDependencies": {
39 |     "standard": "^6.0.7",
40 |     "through2": "^0.6.5",
41 |     "transform-uppercase": "^1.0.0"
42 |   },
43 |   "author": "Mathias Buus-Madsen (@mafintosh)"
44 | }
45 | 


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
  1 | [![deprecated](http://badges.github.io/stability-badges/dist/deprecated.svg)](https://dat-ecosystem.org/) 
  2 | 
  3 | More info on active projects and modules at [dat-ecosystem.org](https://dat-ecosystem.org/) <img src="https://i.imgur.com/qZWlO1y.jpg" width="30" height="30" /> 
  4 | 
  5 | ---
  6 | 
  7 | # gasket
  8 | 
  9 | Preconfigured pipelines for node.js
 10 | 
 11 | ![logo](https://raw.githubusercontent.com/datproject/gasket/master/gasket.png)
 12 | 
 13 | ```
 14 | $ npm install -g gasket
 15 | $ gasket # prints help
 16 | $ gasket completion --save # install tab completion
 17 | ```
 18 | 
 19 | ## Usage
 20 | 
 21 | To setup a pipeline add a `gasket` section to your package.json
 22 | 
 23 | ```json
 24 | {
 25 |   "name": "my-test-app",
 26 |   "dependencies" : {
 27 |     "transform-uppercase": "^1.0.0"
 28 |   },
 29 |   "gasket": {
 30 |     "example": [
 31 |       {
 32 |         "command": "echo hello world",
 33 |         "type": "pipe"
 34 |       },
 35 |       {
 36 |         "command": "transform-uppercase",
 37 |         "type": "pipe"
 38 |       }
 39 |     ]
 40 |   }
 41 | }
 42 | ```
 43 | 
 44 | To run the above `example` pipeline simply to the repo and run
 45 | 
 46 | ```
 47 | $ gasket run example # will print HELLO WORLD
 48 | ```
 49 | 
 50 | `gasket` will spawn each command in the pipeline (it supports modules/commands installed via npm)
 51 | and pipe them together (if the type is set to "pipe").
 52 | 
 53 | If you want to wait for the previous command to finish, set the type to "run" instead.
 54 | 
 55 | ```json
 56 | {
 57 |   "gasket": {
 58 |     "example": [
 59 |       {
 60 |         "command": "echo hello world",
 61 |         "type": "run"
 62 |       },
 63 |       {
 64 |         "command": "echo hello afterwards",
 65 |         "type": "run"
 66 |       }
 67 |     ]
 68 |   }
 69 | }
 70 | ```
 71 | 
 72 | Running the above will print
 73 | 
 74 | ```
 75 | hello world
 76 | hello afterwards
 77 | ```
 78 | 
 79 | ## Modules in pipelines
 80 | 
 81 | In addition to commands it supports node modules that return streams
 82 | 
 83 | ```json
 84 | {
 85 |   "gasket": [
 86 |     {
 87 |       "command": "echo hello world",
 88 |       "type": "pipe"
 89 |     }
 90 |     {
 91 |       "command": {"module":"./uppercase.js"},
 92 |       "type": "pipe"
 93 |     }
 94 |   ]
 95 | }
 96 | ```
 97 | 
 98 | Where `uppercase.js` is a file that looks like this
 99 | 
100 | ``` js
101 | var through = require('through2')
102 | module.exports = function() {
103 |   return through(function(data, enc, cb) {
104 |     cb(null, data.toString().toUpperCase())
105 |   })
106 | }
107 | ```
108 | 
109 | If your module reads/writes JSON object set `json:true` in the pipeline.
110 | That will make gasket parse newline separated JSON before parsing the objects to the stream
111 | and stringify the output.
112 | 
113 | Running `gasket run main` will produce `HELLO WORLD`
114 | 
115 | ## Using gasket.json
116 | 
117 | If you don't have a package.json file you can add the tasks to a `gasket.json` file instead
118 | 
119 | ```json
120 | {
121 |   "example": [
122 |     {
123 |       "command": "echo hello world",
124 |       "type": "pipe"
125 |     },
126 |     {
127 |       "command": "transform-uppercase",
128 |       "type": "pipe"
129 |     }
130 |   ]
131 | }
132 | ```
133 | 
134 | ## gasket as a module
135 | 
136 | You can use gasket as a module as well
137 | 
138 | ``` js
139 | var gasket = require('gasket')
140 | 
141 | var pipelines = gasket({
142 |   example: [
143 |     {
144 |       "command": "echo hello world",
145 |       "type": "pipe"
146 |     },
147 |     {
148 |       "command": "transform-uppercase",
149 |       "type": "pipe"
150 |     }
151 |   ]
152 | })
153 | 
154 | pipelines.run('example').pipe(process.stdout)
155 | ```
156 | 


--------------------------------------------------------------------------------
/bin.js:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env node
  2 | 
  3 | var tab = require('tabalot')
  4 | var fs = require('fs')
  5 | var path = require('path')
  6 | var gasket = require('./')
  7 | 
  8 | process.stdout.setMaxListeners(0)
  9 | process.stderr.setMaxListeners(0)
 10 | process.stdin.setMaxListeners(0)
 11 | 
 12 | process.stdout.on('error', function (err) {
 13 |   if (err.code !== 'EPIPE') throw err
 14 | })
 15 | 
 16 | var help = function (code) {
 17 |   console.log(fs.readFileSync(path.join(__dirname, 'help.txt'), 'utf-8'))
 18 |   process.exit(code)
 19 | }
 20 | 
 21 | var onerror = function (err) {
 22 |   console.error(err.message || err)
 23 |   process.exit(2)
 24 | }
 25 | 
 26 | var save = function (filename, data) {
 27 |   var write = function (data) {
 28 |     fs.writeFile(filename, JSON.stringify(data, null, 2), function (err) {
 29 |       if (err) return onerror(err)
 30 |       process.exit()
 31 |     })
 32 |   }
 33 | 
 34 |   if (path.basename(filename) === 'gasket.json') return write(data)
 35 | 
 36 |   fs.readFile(filename, 'utf-8', function (err, pkg) {
 37 |     if (err) return onerror(err)
 38 |     try {
 39 |       pkg = JSON.parse(pkg)
 40 |     } catch (err) {
 41 |       return onerror(err)
 42 |     }
 43 |     pkg.gasket = data
 44 |     write(pkg)
 45 |   })
 46 | }
 47 | 
 48 | var load = function (opts, cb) {
 49 |   gasket.load(opts.config, function (err, g) {
 50 |     if (err) return onerror(err)
 51 |     cb(g)
 52 |   })
 53 | }
 54 | 
 55 | // completions
 56 | 
 57 | var bin = function (cb) {
 58 |   fs.readdir('node_modules/.bin', cb)
 59 | }
 60 | 
 61 | var pipes = function (pipe, opts, cb) {
 62 |   load(opts, function (gasket) {
 63 |     cb(null, gasket.list().filter(function (pipe) {
 64 |       return opts._.indexOf(pipe, 1) === -1
 65 |     }))
 66 |   })
 67 | }
 68 | 
 69 | // commands
 70 | 
 71 | tab()('--config', '-c', '@file')
 72 | 
 73 | tab('ls')(function (opts) {
 74 |   load(opts, function (gasket) {
 75 |     console.log(gasket.list().join('\n'))
 76 |   })
 77 | })
 78 | 
 79 | tab('exec')('*', bin)(function (opts) {
 80 |   if (opts._.length < 2) return onerror('Usage: gasket exec [commands...]')
 81 |   process.stdin
 82 |     .pipe(gasket().exec(opts._.slice(1).join(' '), opts['--'] || [], {stderr: true})).on('end', process.exit)
 83 |     .pipe(process.stdout)
 84 | })
 85 | 
 86 | tab('version')(function () {
 87 |   console.log(require('./package.json').version)
 88 | })
 89 | 
 90 | tab('help')(function () {
 91 |   help(0)
 92 | })
 93 | 
 94 | tab('add')(pipes)('*', bin)(function (pipe, opts) {
 95 |   if (!pipe || opts._.length < 3) return onerror('Usage: gasket add [pipe] [command]')
 96 | 
 97 |   load(opts, function (gasket) {
 98 |     var data = gasket.toJSON()
 99 |     if (!data[pipe]) data[pipe] = []
100 |     data[pipe].push(opts._.slice(2).join(' '))
101 |     save(gasket.config, data)
102 |   })
103 | })
104 | 
105 | tab('show')(pipes)(function (pipe, opts) {
106 |   if (!pipe) pipe = 'main'
107 |   load(opts, function (gasket) {
108 |     pipe = (gasket.toJSON()[pipe] || [])
109 |       .map(function (line) {
110 |         return line ? (' | ' + line) : '\n'
111 |       })
112 |       .join('').split('\n')
113 |       .map(function (line) {
114 |         return line.replace(/^ \| /, '')
115 |       })
116 |       .join('\n').trim()
117 | 
118 |     console.log(pipe)
119 |   })
120 | })
121 | 
122 | tab('rm')(pipes)(function (pipe, opts) {
123 |   if (!pipe) return onerror('Usage: gasket rm [pipe]')
124 | 
125 |   load(opts, function (gasket) {
126 |     var data = gasket.toJSON()
127 |     delete data[pipe]
128 |     save(gasket.config, data)
129 |   })
130 | })
131 | 
132 | tab('run')('*', pipes)(function (opts) {
133 |   var names = opts._.slice(1)
134 |   if (!names.length) names = ['main']
135 | 
136 |   load(opts, function (gasket) {
137 |     // var first = true
138 |     var loop = function () {
139 |       var name = names.shift()
140 |       if (!name) return process.exit()
141 | 
142 |       if (!gasket.has(name)) {
143 |         if (name !== 'main') console.error(name + ' does not exist')
144 |         return loop()
145 |       }
146 | 
147 |       var t = gasket.run(name, opts['--'] || [], {stderr: true})
148 | 
149 |       t.pipe(process.stdout)
150 |       t.on('end', loop)
151 |     }
152 | 
153 |     loop()
154 |   })
155 | })
156 | 
157 | tab('pipe')('*', pipes)(function (opts) {
158 |   var names = opts._.slice(1)
159 |   if (!names.length) names = ['main']
160 | 
161 |   load(opts, function (gasket) {
162 |     var streams = names
163 |       .map(function (name) {
164 |         if (!gasket.has(name)) {
165 |           if (name !== 'main') console.error(name + ' does not exist')
166 |           return null
167 |         }
168 |         return gasket.pipe(name, opts['--'] || [], {stderr: true})
169 |       })
170 |       .filter(function (s) {
171 |         return s
172 |       })
173 | 
174 |     if (!streams.length) return
175 | 
176 |     var last = [process.stdin].concat(streams).concat(process.stdout).reduce(function (a, b) {
177 |       return a.pipe(b)
178 |     })
179 | 
180 |     last.on('end', function () {
181 |       process.exit(0)
182 |     })
183 |   })
184 | })
185 | 
186 | tab.parse({'--': true}) || help(1)
187 | 


--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
  1 | var path = require('path')
  2 | var execspawn = require('npm-execspawn')
  3 | var xtend = require('xtend')
  4 | var resolve = require('resolve')
  5 | var ndjson = require('ndjson')
  6 | var duplexify = require('duplexify')
  7 | var pumpify = require('pumpify')
  8 | var fs = require('fs')
  9 | var multistream = require('multistream')
 10 | var parallel = require('parallel-multistream')
 11 | var debug = require('debug-stream')('gasket')
 12 | 
 13 | /*  Create a stream with cmd and opts*/
 14 | var toStream = function (cmd, opts) {
 15 |   var child = execspawn(cmd.command, cmd.params, opts)
 16 |   child.on('exit', function (code) {
 17 |     if (code) result.destroy(new Error('Process exited with code: ' + code))
 18 |   })
 19 | 
 20 |   if (opts.stderr === true) opts.stderr = process.stderr
 21 | 
 22 |   if (opts.stderr) child.stderr.pipe(opts.stderr)
 23 |   else child.stderr.resume()
 24 | 
 25 |   var result = duplexify(child.stdin, child.stdout)
 26 |   return result
 27 | }
 28 | 
 29 | /*  Create a stream which runs runCommands in sequence */
 30 | var runStream = function (runCommands) {
 31 |   var commands = runCommands.map(function (cmd) {
 32 |     var fn = function () {
 33 |       if (cmd.end) cmd.end() // not writable
 34 |       return cmd
 35 |     }
 36 |     return fn()
 37 |   })
 38 |   return multistream(commands)
 39 | }
 40 | 
 41 | /* Run forkCommands in parallel */
 42 | var forkStream = function (forkCommands) {
 43 |   var commands = forkCommands.map(function (cmd) {
 44 |     // no lazyness here since we are forking
 45 |     if (cmd.end) cmd.end() // not writable
 46 |     return cmd
 47 |   })
 48 |   return parallel(commands)
 49 | }
 50 | 
 51 | /* Pipe pipeCommands together */
 52 | var pipeStream = function (pipeCommands) {
 53 |   var pipe = pumpify(pipeCommands)
 54 |   pipe.end() // first not writable
 55 |   return pipe
 56 | }
 57 | 
 58 | /* Map commands according to type */
 59 | var mapToStream = function (commands, type) {
 60 |   var func
 61 |   // Map first command in commands to the rest
 62 |   if (type === 'map') func = function (p) { pipes.push(pipeStream([first, p])) }
 63 |   // Map the rest of the commands to the first command
 64 |   if (type === 'reduce') func = function (p) { pipes.push(pipeStream([p, first])) }
 65 |   var pipes = []
 66 |   var first = commands.shift()
 67 |   commands.map(func)
 68 |   return forkStream(pipes)
 69 | }
 70 | 
 71 | var compileModule = function (p, opts) {
 72 |   if (!p.exports) p.exports = require(resolve.sync(p.module, {basedir: opts.cwd}))
 73 |   return p.json ? pumpify(ndjson.parse(), p.exports(p), ndjson.serialize()) : p.exports(p)
 74 | }
 75 | 
 76 | var compile = function (name, pipeline, opts) {
 77 |   var wrap = function (i, msg, stream) {
 78 |     if (!process.env.DEBUG) return stream
 79 |     return pumpify(debug('#' + i + ' stdin:  ' + msg), stream, debug('#' + i + ' stdout: ' + msg))
 80 |   }
 81 |   var type = pipeline[0].type
 82 |   var visit = function (p, i) {
 83 |     if (typeof p === 'object') p = {command: p.command}
 84 |     if (typeof p === 'function') p = {exports: p, module: true}
 85 |     if (!p.params) p.params = [].concat(name, opts.params || [])
 86 |     if (p.command) return wrap(i, '(' + p.command + ')', toStream(p, opts))
 87 |     if (p.module) return wrap(i, '(' + p.module + ')', compileModule(p, opts))
 88 |     throw new Error('Unsupported pipeline #' + i + ' in ' + name)
 89 |   }
 90 |   pipeline = pipeline.map(visit)
 91 |   return [type, pipeline]
 92 | }
 93 | 
 94 | var split = function (pipeline) {
 95 |   var list = []
 96 |   var current = []
 97 | 
 98 |   var prevType = null
 99 |   var visit = function (p) {
100 |     if (p.type === prevType) {
101 |       return current.push(p)
102 |     } else {
103 |       prevType = p.type
104 |       if (current.length) list.push(current)
105 |       current = []
106 |       current.push(p)
107 |       return
108 |     }
109 |   }
110 |   pipeline = [].concat(pipeline || [])
111 |   pipeline.map(visit)
112 | 
113 |   if (current.length) list.push(current)
114 |   return list
115 | }
116 | 
117 | var gasket = function (config, defaults) {
118 |   if (!defaults) defaults = {}
119 |   if (!config) config = {}
120 |   if (Array.isArray(config)) config = {main: config}
121 | 
122 |   var that = {}
123 | 
124 |   that.cwd = defaults.cwd = path.resolve(defaults.cwd || '.')
125 |   that.env = defaults.env = defaults.env || process.env
126 | 
127 |   var pipes = Object.keys(config).reduce(function (result, key) {
128 |     var list = split(config[key])
129 | 
130 |     result[key] = function (opts) {
131 |       if (Array.isArray(opts)) opts = {params: opts}
132 |       opts = xtend(defaults, opts)
133 | 
134 |       var mainPipeline = []
135 |       var bkgds = []
136 |       list.forEach(function (pipeline) {
137 |         var compiled = compile(key, pipeline, opts)
138 |         var type = compiled[0]
139 |         var p = compiled[1]
140 |         switch (type) {
141 |           case ('pipe'):
142 |             mainPipeline.push(pipeStream(p))
143 |             break
144 |           case ('run'):
145 |             mainPipeline.push(runStream(p))
146 |             break
147 |           case ('fork'):
148 |             mainPipeline.push(forkStream(p))
149 |             break
150 |           case ('background'):
151 |             bkgds = bkgds.concat(p)
152 |             break
153 |           case ('map'):
154 |             mainPipeline.push(mapToStream(p, 'map'))
155 |             break
156 |           case ('reduce'):
157 |             mainPipeline.push(mapToStream(p, 'reduce'))
158 |             break
159 |           default:
160 |             throw new Error('Unsupported Type: ' + type)
161 |         }
162 |       })
163 | 
164 |       mainPipeline = runStream(mainPipeline)
165 | 
166 |       // Handle background processes
167 |       if (bkgds.length) {
168 |         bkgds = forkStream(bkgds)
169 |         mainPipeline.on('end', function () {
170 |           bkgds.destroy()
171 |         })
172 |         return parallel([mainPipeline, bkgds])
173 |       }
174 | 
175 |       return mainPipeline
176 |     }
177 |     return result
178 |   }, {})
179 | 
180 |   that.list = function () {
181 |     return Object.keys(pipes)
182 |   }
183 | 
184 |   that.has = function (name) {
185 |     return !!pipes[name]
186 |   }
187 | 
188 |   that.pipe = function (name, opts, extra) {
189 |     if (Array.isArray(opts)) {
190 |       extra = extra || {}
191 |       extra.params = opts
192 |       opts = extra
193 |     }
194 |     return pipes[name] && pipes[name](opts)
195 |   }
196 | 
197 |   that.run = function (name, opts, extra) {
198 |     var stream = that.pipe(name, opts, extra)
199 |     if (stream.end) stream.end()
200 |     return stream
201 |   }
202 | 
203 |   that.exec = function (cmd, params, opts) {
204 |     if (!Array.isArray(params)) return that.exec(cmd, [], params)
205 |     return toStream({command: cmd, params: ['exec'].concat(params)}, opts || {})
206 |   }
207 | 
208 |   that.toJSON = function () {
209 |     return config
210 |   }
211 |   return that
212 | }
213 | 
214 | gasket.load = function (cwd, opts, cb) {
215 |   if (typeof opts === 'function') return gasket.load(cwd, null, opts)
216 |   if (!opts) opts = {}
217 | 
218 |   var ready = function (pipelines, filename) {
219 |     var name = path.basename(filename)
220 |     if (name !== 'gasket.json') pipelines = pipelines.gasket || {}
221 |     var g = gasket(pipelines, opts)
222 |     g.config = filename
223 |     cb(null, g)
224 |   }
225 | 
226 |   var read = function (file, cb) {
227 |     file = path.resolve(process.cwd(), path.join(cwd || '.', file))
228 |     fs.readFile(file, 'utf-8', function (err, data) {
229 |       if (err) return cb(err)
230 | 
231 |       try {
232 |         data = JSON.parse(data)
233 |       } catch (err) {
234 |         return cb(err)
235 |       }
236 | 
237 |       opts.cwd = path.dirname(file)
238 |       cb(null, data, file)
239 |     })
240 |   }
241 | 
242 |   read('.', function (err, data, filename) {
243 |     // If it found ./package.json file but couldn't be parsed
244 |     if (err && err.name === 'SyntaxError') return cb(err)
245 |     if (data) return ready(data, filename)
246 |     read('gasket.json', function (err, data, filename) {
247 |       // If it found gasket.json it but couldn't be parsed
248 |       if (err && err.name === 'SyntaxError') return cb(err)
249 |       if (data) return ready(data, filename)
250 |       read('package.json', function (err, data, filename) {
251 |         if (err) return cb(err)
252 |         ready(data, filename)
253 |       })
254 |     })
255 |   })
256 | }
257 | 
258 | module.exports = gasket
259 | 


--------------------------------------------------------------------------------