├── .gitignore ├── libs └── async-eval.js ├── resources ├── graph-image.png └── graph.html ├── package.json ├── bower.json ├── LICENSE ├── examples ├── multi-thread.js ├── basic-example.js ├── browser-example.html ├── multi-example.js └── news-parser.js ├── index.js ├── README.md ├── dist └── flowpipe.js └── v0.4.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | test 3 | graph 4 | .idea 5 | .git 6 | .DS_Store 7 | npm-debug.log -------------------------------------------------------------------------------- /libs/async-eval.js: -------------------------------------------------------------------------------- 1 | var code = process.argv.splice(2)[0]; 2 | code = decodeURI(code); 3 | eval(code); -------------------------------------------------------------------------------- /resources/graph-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/proin/flowpipe/HEAD/resources/graph-image.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flowpipe", 3 | "version": "0.5.2", 4 | "description": "pipeline async utilities for javascript", 5 | "main": "index.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "https://github.com/proin/pipeflow.git" 9 | }, 10 | "scripts": { 11 | "test": "echo \"Error: no test specified\" && exit 1" 12 | }, 13 | "keywords": [ 14 | "async", 15 | "flowpipe", 16 | "pipe", 17 | "pipe", 18 | "loop", 19 | "loopback" 20 | ], 21 | "author": "Yeonghun Chae (http://proinlab.com)", 22 | "license": "ISC" 23 | } 24 | -------------------------------------------------------------------------------- /bower.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flowpipe", 3 | "description": "pipeline async utilities for javascript", 4 | "version":"0.3.6", 5 | "main": "dist/flowpipe.js", 6 | "authors": [ 7 | "Yeonghun Chae (http://proinlab.com)" 8 | ], 9 | "license": "ISC", 10 | "keywords": [ 11 | "async", 12 | "flowpipe", 13 | "pipe", 14 | "pipe", 15 | "loop", 16 | "loopback" 17 | ], 18 | "homepage": "https://github.com/proin/pipeflow", 19 | "ignore": [ 20 | "**/.*", 21 | "node_modules", 22 | "bower_components", 23 | "test", 24 | "tests" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Yeonghun Chae 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | -------------------------------------------------------------------------------- /examples/multi-thread.js: -------------------------------------------------------------------------------- 1 | var flowpipe = require('../index'); 2 | flowpipe 3 | .init(function (next) { 4 | var size = 2000000000; 5 | next(null, size); 6 | }) 7 | .pipe('parallel single thread', function (next, size) { 8 | var item = [1, 2, 3, 4, 5, 6, 7, 8]; 9 | var st = new Date().getTime(); 10 | next(null, item, st, size); 11 | }) 12 | .parallel('single thread', function (next, item, st, size) { 13 | for (var i = 0; i < size; i++) 14 | ; 15 | next(); 16 | }) 17 | .pipe('parallel multi thread', function (next, parallelData, st, size) { 18 | console.log('single thread total(ms)', new Date().getTime() - st); 19 | var item = [1, 2, 3, 4, 5, 6, 7, 8]; 20 | st = new Date().getTime(); 21 | next(null, item, st, size); 22 | }) 23 | .parallel('multi thread', function (next, item, st, size) { 24 | for (var i = 0; i < size; i++) 25 | ; 26 | next(); 27 | }, {multiThread: true}) 28 | .pipe('finalize', function (next, data, st) { 29 | console.log('multi thread total(ms)', new Date().getTime() - st); 30 | next(); 31 | }) 32 | .end(function (err) { 33 | }) 34 | .graph('./graph/multi-thread.html'); -------------------------------------------------------------------------------- /resources/graph.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Flowpipe: Graph 5 | 6 | 7 | 8 | 9 | 10 | 11 | 28 | 29 | 30 |
31 |
32 |
33 |
34 |
35 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /examples/basic-example.js: -------------------------------------------------------------------------------- 1 | var flowpipe = require('../index'); 2 | 3 | var graph = flowpipe 4 | .init(function (next) { 5 | // init before start 6 | var page = 1; 7 | next(null, page); 8 | }) 9 | .pipe('start', function (next, page) { 10 | // page is that previous next function's variable 11 | setTimeout(function (err) { 12 | // next is function that proceed next pipe, parallel or loopback 13 | next(err, page); 14 | }, 1000); 15 | }) 16 | .pipe('list', function (next, page) { 17 | console.log('start page: ' + page); 18 | // preparing list for parallel 19 | var list = [{no: 1}, {no: 2}, {no: 3}]; 20 | for (var i = 0; i < list.length; i++) { 21 | list[i].page = page; 22 | list[i].delay = 3 - i; // delay time in parallel process 23 | } 24 | // next before parallel, pass only second parameter to parallel process. 25 | // second parameter must be array. 26 | // other parameters pass to next pipe or loopback. 27 | next(null, list, page); 28 | }) 29 | .parallel('parallel process', function (next, data) { 30 | // data is indicating each of list items. 31 | setTimeout(function () { 32 | data.title = 'title-' + data.no; 33 | next(null, data); 34 | }, data.delay * 1000); 35 | }) 36 | .parallel('parallel process 2', function (next, data) { 37 | // data is indicating each of list items. 38 | setTimeout(function () { 39 | data.title = 'title-' + data.no; 40 | next(null, data); 41 | }, data.delay * 1000); 42 | }) 43 | .pipe('pass-1', function (next, parallel, page) { 44 | console.log('pass-1'); 45 | next(null, parallel, page); 46 | }) 47 | .pipe('pass-else', function (next, parallel, page) { 48 | console.log('pass-else'); 49 | next(null, parallel, page); 50 | }) 51 | .loopback('pipe-list', function (loop, next, instance, parallel, page) { 52 | // loopback(process_type-process_name, fn) 53 | // - process_type: pipe, parallel 54 | // - process_name: must defined 55 | // - fn(loop, next, instance, others...) 56 | // - loop(err, variables...): passing variable to target process 57 | // - next(err, variables...): proceeding if loop ended 58 | // - instance: maintainable variable in loop, object type {} 59 | // - others: passed from before process 60 | if (!instance.data) instance.data = []; 61 | for (var i = 0; i < parallel.length; i++) 62 | instance.data.push(parallel[i]); 63 | if (page < 5) loop(null, page + 1); 64 | else next(null, instance.data); 65 | }) 66 | .pipe('finalize', function (next, data) { 67 | next(null, data); 68 | }) 69 | .end(function (err, test) { 70 | // end(callback) or end() 71 | // this must be declared, if not all function don't working. 72 | // proceed in the end or occured error in process 73 | }) 74 | .graph('./graph/basic-example-graph.html'); -------------------------------------------------------------------------------- /examples/browser-example.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Flowpipe: Graph 5 | 6 | 7 | 8 | 9 | 10 | 84 | 85 | 86 | 88 | 89 | 106 | 107 | 108 |
109 |
110 |
111 |
112 |
113 | 126 | 127 | 128 | -------------------------------------------------------------------------------- /examples/multi-example.js: -------------------------------------------------------------------------------- 1 | var flowpipe1 = require('../index'); 2 | 3 | flowpipe1 4 | .init(function (next) { 5 | // init before start 6 | var page = 1; 7 | next(null, page); 8 | }) 9 | .pipe('start', function (next, page) { 10 | // page is that previous next function's variable 11 | setTimeout(function (err) { 12 | // next is function that proceed next pipe, parallel or loopback 13 | next(err, page); 14 | }, 1000); 15 | }) 16 | .pipe('list', function (next, page) { 17 | console.log('start page: ' + page); 18 | // preparing list for parallel 19 | var list = [{no: 1}, {no: 2}, {no: 3}]; 20 | for (var i = 0; i < list.length; i++) { 21 | list[i].page = page; 22 | list[i].delay = 3 - i; // delay time in parallel process 23 | } 24 | // next before parallel, pass only second parameter to parallel process. 25 | // second parameter must be array. 26 | // other parameters pass to next pipe or loopback. 27 | next(null, list, page); 28 | }) 29 | .parallel('parallel process', function (next, data) { 30 | // data is indicating each of list items. 31 | setTimeout(function () { 32 | data.title = 'title-' + data.no; 33 | next(null, data); 34 | }, data.delay * 1000); 35 | }) 36 | .parallel('parallel process 2', function (next, data) { 37 | // data is indicating each of list items. 38 | setTimeout(function () { 39 | data.title = 'title-' + data.no; 40 | next(null, data); 41 | }, data.delay * 1000); 42 | }) 43 | .pipe('pass-1', function (next, parallel, page) { 44 | console.log('pass-1'); 45 | next(null, parallel, page); 46 | }) 47 | .pipe('pass-else', function (next, parallel, page) { 48 | console.log('pass-else'); 49 | next(null, parallel, page); 50 | }) 51 | .loopback('pipe-list', function (loop, next, instance, parallel, page) { 52 | // loopback(process_type-process_name, fn) 53 | // - process_type: pipe, parallel 54 | // - process_name: must defined 55 | // - fn(loop, next, instance, others...) 56 | // - loop(err, variables...): passing variable to target process 57 | // - next(err, variables...): proceeding if loop ended 58 | // - instance: maintainable variable in loop, object type {} 59 | // - others: passed from before process 60 | if (!instance.data) instance.data = []; 61 | for (var i = 0; i < parallel.length; i++) 62 | instance.data.push(parallel[i]); 63 | if (page < 5) loop(null, page + 1); 64 | else next(null, instance.data); 65 | }) 66 | .pipe('finalize', function (next, data) { 67 | next(null, data); 68 | }) 69 | .end(function (err, test) { 70 | // end(callback) or end() 71 | // this must be declared, if not all function don't working. 72 | // proceed in the end or occured error in process 73 | }) 74 | .graph('./graph/basic-example-graph-2.html'); 75 | 76 | var flowpipe2 = require('../index'); 77 | 78 | flowpipe2 79 | .init(function (next) { 80 | // init before start 81 | var page = 1; 82 | next(null, page); 83 | }) 84 | .pipe('start', function (next, page) { 85 | // page is that previous next function's variable 86 | setTimeout(function (err) { 87 | // next is function that proceed next pipe, parallel or loopback 88 | next(err, page); 89 | }, 1000); 90 | }) 91 | .pipe('list', function (next, page) { 92 | console.log('start page: ' + page); 93 | // preparing list for parallel 94 | var list = [{no: 1}, {no: 2}, {no: 3}]; 95 | for (var i = 0; i < list.length; i++) { 96 | list[i].page = page; 97 | list[i].delay = 3 - i; // delay time in parallel process 98 | } 99 | // next before parallel, pass only second parameter to parallel process. 100 | // second parameter must be array. 101 | // other parameters pass to next pipe or loopback. 102 | next(null, list, page); 103 | }) 104 | .parallel('parallel process', function (next, data) { 105 | // data is indicating each of list items. 106 | setTimeout(function () { 107 | data.title = 'title-' + data.no; 108 | next(null, data); 109 | }, data.delay * 1000); 110 | }) 111 | .parallel('parallel process 2', function (next, data) { 112 | // data is indicating each of list items. 113 | setTimeout(function () { 114 | data.title = 'title-' + data.no; 115 | next(null, data); 116 | }, data.delay * 1000); 117 | }) 118 | .pipe('pass-1', function (next, parallel, page) { 119 | console.log('pass-1'); 120 | next(null, parallel, page); 121 | }) 122 | .pipe('pass-else', function (next, parallel, page) { 123 | console.log('pass-else'); 124 | next(null, parallel, page); 125 | }) 126 | .loopback('pipe-list', function (loop, next, instance, parallel, page) { 127 | // loopback(process_type-process_name, fn) 128 | // - process_type: pipe, parallel 129 | // - process_name: must defined 130 | // - fn(loop, next, instance, others...) 131 | // - loop(err, variables...): passing variable to target process 132 | // - next(err, variables...): proceeding if loop ended 133 | // - instance: maintainable variable in loop, object type {} 134 | // - others: passed from before process 135 | if (!instance.data) instance.data = []; 136 | for (var i = 0; i < parallel.length; i++) 137 | instance.data.push(parallel[i]); 138 | if (page < 5) loop(null, page + 1); 139 | else next(null, instance.data); 140 | }) 141 | .pipe('finalize', function (next, data) { 142 | next(null, data); 143 | }) 144 | .end(function (err, test) { 145 | // end(callback) or end() 146 | // this must be declared, if not all function don't working. 147 | // proceed in the end or occured error in process 148 | }) 149 | .graph('./graph/basic-example-graph-1.html'); -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = (()=> { 4 | let app = {}; 5 | 6 | app.older = require('./v0.4'); 7 | 8 | let assert = (condition, message)=> { 9 | if (!condition) { 10 | message = message || "Assertion failed"; 11 | if (typeof Error !== "undefined") { 12 | throw new Error(message); 13 | } 14 | throw message; 15 | } 16 | }; 17 | 18 | let instance = function (instanceName) { 19 | if (!instanceName) instanceName = 'flowpipe-' + new Date().getTime(); 20 | 21 | let works = {}; 22 | let order = []; 23 | let args = {}; 24 | let MAX_THREAD = 10; 25 | let AUTO_INCREASED = 0; 26 | this.then = this.add = this.pipe = (name, work)=> { 27 | if (!work) { 28 | work = name; 29 | name = work.name; 30 | if (!name || name == '') { 31 | name = 'auto-' + AUTO_INCREASED; 32 | AUTO_INCREASED++; 33 | } 34 | } 35 | order.push({type: 'work', name: name}); 36 | works[name] = work; 37 | return this; 38 | }; 39 | 40 | this.for = this.loop = this.loopback = (name, condition)=> { 41 | order.push({type: 'loop', to: name, condition: condition}); 42 | return this; 43 | }; 44 | 45 | this.init = (configure)=> { 46 | order.push({type: 'init', configure: configure}); 47 | return this; 48 | }; 49 | 50 | this.timestamp = (display)=> { 51 | order.push({type: 'timestamp', display: display}); 52 | return this; 53 | }; 54 | 55 | this.log = (display)=> { 56 | order.push({type: 'log', display: display}); 57 | return this; 58 | } 59 | 60 | this.parallel = (setter, work)=> { 61 | order.push({type: 'parallel', set: setter, do: work}); 62 | return this; 63 | }; 64 | 65 | this.args = args; 66 | 67 | this.list = ()=> order; 68 | this.maxThread = (maxThread)=> { 69 | order.push({type: 'MAX_THREAD', MAX_THREAD: maxThread}); 70 | return this; 71 | }; 72 | 73 | this.run = this.start = this.end = this.finish = (callback)=> { 74 | let starttime = new Date().getTime(); 75 | let pretime = new Date().getTime(); 76 | 77 | let manager = (wi)=> { 78 | if (typeof wi == 'undefined') wi = 0; 79 | let indicate = order[wi]; 80 | if (!indicate) { 81 | if (callback) callback(args); 82 | return; 83 | } 84 | 85 | if (indicate.type !== 'timestamp') 86 | pretime = new Date().getTime(); 87 | 88 | if (indicate.type === 'work') { 89 | works[indicate.name](args).then((result)=> { 90 | if (typeof result === 'object') 91 | for (let key in result) 92 | args[key] = result[key]; 93 | manager(wi + 1); 94 | }).catch((e)=> { 95 | console.log(`[${instanceName}] ERROR IN "${indicate.name}"`); 96 | console.log(e); 97 | }); 98 | } else if (indicate.type === 'loop') { 99 | try { 100 | if (indicate.condition(args)) { 101 | for (let i = 0; i < order.length; i++) 102 | if (order[i].name === indicate.to) 103 | wi = i; 104 | } else { 105 | wi++; 106 | } 107 | manager(wi); 108 | } catch (e) { 109 | console.log(`[${instanceName}] ERROR IN loop "${indicate.condition.toString()}"`); 110 | console.log(e); 111 | } 112 | } else if (indicate.type === 'init') { 113 | try { 114 | indicate.configure(args); 115 | manager(wi + 1); 116 | } catch (e) { 117 | console.log(`[${instanceName}] ERROR IN init "${indicate.configure.toString()}"`); 118 | console.log(e); 119 | } 120 | } else if (indicate.type === 'parallel') { 121 | try { 122 | let pdata = indicate.set(args); 123 | let pCnt = 0; 124 | let pidx = 0; 125 | let proceed = 0; 126 | let parallelWork = ()=> { 127 | if (!pdata[pidx]) return; 128 | if (pCnt > MAX_THREAD) return; 129 | pCnt++; 130 | indicate.do(args, pdata[pidx], pidx).then(()=> { 131 | proceed++; 132 | pCnt--; 133 | 134 | if (proceed === pdata.length) { 135 | manager(wi + 1); 136 | return; 137 | } 138 | parallelWork(); 139 | }); 140 | 141 | pidx++; 142 | parallelWork(); 143 | }; 144 | 145 | parallelWork(); 146 | } catch (e) { 147 | console.log(`[${instanceName}] ERROR IN parallel "${indicate.set.toString()}"`); 148 | console.log(e); 149 | } 150 | } else if (indicate.type === 'MAX_THREAD') { 151 | MAX_THREAD = indicate.MAX_THREAD; 152 | manager(wi + 1); 153 | } else if (indicate.type === 'log') { 154 | try { 155 | console.log(`[${instanceName}] ${indicate.display(args)}`); 156 | } catch (e) { 157 | } 158 | manager(wi + 1); 159 | } else if (indicate.type === 'timestamp') { 160 | try { 161 | console.log(`[${instanceName}] ${indicate.display(new Date().getTime() - starttime, new Date().getTime() - pretime)}`); 162 | } catch (e) { 163 | } 164 | manager(wi + 1); 165 | } 166 | }; 167 | 168 | manager(); 169 | }; 170 | }; 171 | 172 | app.instance = (instanceName)=> { 173 | return new instance(instanceName); 174 | }; 175 | 176 | return app; 177 | })(); -------------------------------------------------------------------------------- /examples/news-parser.js: -------------------------------------------------------------------------------- 1 | var flowpipe = require('flowpipe'); 2 | var request = require('request'); 3 | var cheerio = require('cheerio'); 4 | var mysql = require('mysql'); 5 | 6 | var category_list = [ 7 | {title: '경제', id: 949986}, 8 | {title: '정치', id: 950203}, 9 | {title: '사회', id: 949987}, 10 | {title: '생활/문화', id: 949988}, 11 | {title: '세계', id: 949990}, 12 | {title: 'IT/과학', id: 949984} 13 | ]; 14 | 15 | flowpipe 16 | .init(function (next) { 17 | var query = {}; 18 | query.date = new Date('2015-01-01'); 19 | next(null, query); 20 | }) 21 | .pipe('dateloop', function (next, query) { 22 | query.category = 0; 23 | next(null, query); 24 | }) 25 | .pipe('categoryloop', function (next, query) { 26 | query.page = 1; 27 | next(null, query); 28 | }) 29 | .pipe('pageloop', function (next, query) { 30 | setTimeout(function () { 31 | var dateformat = query.date.format('yyyy-MM-dd 00:00:00'); 32 | var category = category_list[query.category]; 33 | var url = 'http://news.naver.com/main/mainNews.nhn?componentId=' + category.id + '&date=' + dateformat + '&page=' + query.page; 34 | next(null, query, url); 35 | }, 500) 36 | }) 37 | .pipe('getHtml', function (next, query, url) { 38 | request.get({ 39 | url: url, 40 | encoding: 'binary' 41 | }, function (err, res, body) { 42 | if (err) return next(err); 43 | 44 | try { 45 | var charSet = 'euc-kr'; 46 | body = new Buffer(body, 'binary'); 47 | var Iconv = require('iconv').Iconv; 48 | var ic = new Iconv(charSet, 'utf-8'); 49 | body = ic.convert(body).toString(); 50 | } catch (e) { 51 | } 52 | 53 | var data = JSON.parse(body).itemList; 54 | next(null, query, data); 55 | }); 56 | }) 57 | .pipe('parse', function (next, query, data) { 58 | var result = []; 59 | var category = category_list[query.category]; 60 | for (var i = 0; i < data.length; i++) 61 | result.push({ 62 | title: data[i].titleWithUnescapeHtml, 63 | category: category.title, 64 | articleDate: data[i].articleDate, 65 | articleId: data[i].articleId, 66 | officeId: data[i].officeId, 67 | href: 'http://news.naver.com/main/read.nhn?mode=LSD&mid=shm&sid1=105&oid=' + data[i].officeId + '&aid=' + data[i].articleId 68 | }); 69 | 70 | next(null, query, result); 71 | }) 72 | .loopback('pipe-pageloop', function (loop, next, instance, query, result) { 73 | if (!instance.result) instance.result = []; 74 | if (!instance.pre) instance.pre = []; 75 | for (var i = 0; i < result.length; i++) 76 | for (var j = 0; j < instance.pre.length; j++) 77 | if (result[i].articleId == instance.pre[j].articleId) 78 | return next(null, instance.result, query); 79 | for (var i = 0; i < result.length; i++) 80 | instance.result.push(result[i]); 81 | instance.pre = result; 82 | 83 | query.page += 1; 84 | loop(null, query); 85 | }) 86 | .parallel('getContent', function (next, data) { 87 | request.get({ 88 | url: data.href, 89 | encoding: 'binary' 90 | }, function (err, res, body) { 91 | if (err) return next(err); 92 | 93 | try { 94 | var charSet = 'euc-kr'; 95 | body = new Buffer(body, 'binary'); 96 | var Iconv = require('iconv').Iconv; 97 | var ic = new Iconv(charSet, 'utf-8'); 98 | body = ic.convert(body).toString(); 99 | } catch (e) { 100 | } 101 | var $ = cheerio.load(body); 102 | data.text = $('#articleBodyContents').text().trim(); 103 | next(null, data); 104 | }); 105 | }) 106 | .pipe('createDB', function (next, parsed, query) { 107 | var connection = mysql.createConnection({ 108 | "host": "localhost", 109 | "user": "root", 110 | "password": "", 111 | "database": "async" 112 | }); 113 | 114 | next(null, parsed, connection, query); 115 | }) 116 | .parallel('insertDB', function (next, data, connection) { 117 | connection.query('INSERT INTO news VALUES(?,?,?,?,?,?,?)', [data.articleId, data.category, data.title, data.articleDate, data.officeId, data.href, data.text], function (err) { 118 | next(null, err ? err : 'no-error'); 119 | }); 120 | }) 121 | .pipe('closeDB', function (next, errors, connection, query) { 122 | var errorCnt = 0; 123 | for (var i = 0; i < errors.length; i++) 124 | if (errors[i] != 'no-error') errorCnt++; 125 | 126 | console.log( 127 | '[' + query.date.format('yyyy-MM-dd') + ']', 128 | '[' + category_list[query.category].title + ']', 129 | errors.length, 'data parsed.', 130 | errorCnt, 'error at inserting.' 131 | ); 132 | 133 | connection.end(function () { 134 | next(null, query); 135 | }); 136 | }) 137 | .loopback('pipe-categoryloop', function (loop, next, instance, query) { 138 | query.category += 1; 139 | if (category_list[query.category]) 140 | loop(null, query); 141 | else { 142 | next(null, query); 143 | } 144 | }) 145 | .loopback('pipe-dateloop', function (loop, next, instance, query) { 146 | if (query.date.getTime() < new Date('2015-12-31').getTime()) { 147 | query.date = query.date.day(+1); 148 | loop(null, query); 149 | } else { 150 | next(null); 151 | } 152 | }) 153 | .end().graph('./graph/news-parser.html'); 154 | 155 | // Date Format Prototypes 156 | Date.prototype.day = function (val) { 157 | return new Date(this.getTime() + val * 1000 * 60 * 60 * 24); 158 | }; 159 | 160 | Date.prototype.format = function (f) { 161 | if (!this.valueOf()) return " "; 162 | 163 | var weekName = ["일요일", "월요일", "화요일", "수요일", "목요일", "금요일", "토요일"]; 164 | var d = this; 165 | 166 | return f.replace(/(yyyy|yy|MM|dd|E|hh|mm|ss|a\/p)/gi, function ($1) { 167 | switch ($1) { 168 | case "yyyy": 169 | return d.getFullYear(); 170 | case "yy": 171 | return (d.getFullYear() % 1000).zf(2); 172 | case "MM": 173 | return (d.getMonth() + 1).zf(2); 174 | case "dd": 175 | return d.getDate().zf(2); 176 | case "E": 177 | return weekName[d.getDay()]; 178 | case "HH": 179 | return d.getHours().zf(2); 180 | case "hh": 181 | return ((h = d.getHours() % 12) ? h : 12).zf(2); 182 | case "mm": 183 | return d.getMinutes().zf(2); 184 | case "ss": 185 | return d.getSeconds().zf(2); 186 | case "a/p": 187 | return d.getHours() < 12 ? "오전" : "오후"; 188 | default: 189 | return $1; 190 | } 191 | }); 192 | }; 193 | 194 | String.prototype.string = function (len) { 195 | var s = '', i = 0; 196 | while (i++ < len) { 197 | s += this; 198 | } 199 | return s; 200 | }; 201 | 202 | String.prototype.zf = function (len) { 203 | return "0".string(len - this.length) + this; 204 | }; 205 | 206 | Number.prototype.zf = function (len) { 207 | return this.toString().zf(len); 208 | }; -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | `flowpipe` is installable via: 2 | 3 | - npm: `npm install flowpipe --save` 4 | 5 | ## Promise Based Flowpipe: upper v0.5.0 6 | 7 | - v0.5.0 is based on `Promise` Object, so not support previous version. 8 | - for using older version, `let flowpipe = require('flowpipe').older` 9 | 10 | ### Quick Start 11 | 12 | ```javascript 13 | 'use strict'; 14 | 15 | const Flowpipe = require('flowpipe'); 16 | 17 | let myWork = (args)=> new Promise((resolve)=>{ 18 | args.data2.push(Math.random()); 19 | setTimeout(resolve, 5); 20 | }); 21 | 22 | let flowpipe = Flowpipe.instance('MyWork'); 23 | flowpipe 24 | // set arguments 25 | .init((args)=> args.index = 0) 26 | .init((args)=> args.data1 = []) 27 | .init((args)=> args.data2 = []) 28 | .init((args)=> args.data3 = []) 29 | // set work, name as work-1 30 | .then('work-1', (args)=> new Promise((resolve)=> { 31 | args.data1.push(Math.random()); 32 | setTimeout(resolve, 5); 33 | })) 34 | // set work, set name as myWork 35 | .then(myWork) 36 | // set work, set auto created name 37 | .then((args)=> new Promise((resolve)=> { 38 | args.data3.push(Math.random()); 39 | setTimeout(resolve, 5); 40 | })) 41 | // loop back to work-1 if args.index < 100 and increase args.index 42 | .loop('work-1', (args)=> ++args.index < 100) 43 | // print something 44 | .log((args)=> `data1 data2 data3 / ${args.data1.length} ${args.data2.length} ${args.data3.length}`) 45 | // set parallel max thread 46 | .maxThread(30) 47 | // set parallel 48 | .parallel((args)=> args.data1, (args, data, idx)=> new Promise((resolve)=> { 49 | setTimeout(()=> { 50 | resolve(); 51 | }, data * 1000); 52 | })) 53 | // print timestamp 54 | .timestamp((total, premodule)=> `total: ${total}ms, parallel ${premodule}ms`) 55 | // run 56 | .run(); 57 | ``` 58 | 59 | ### Document 60 | 61 | - Flowpipe.instance(name) 62 | - `return` Flowpipe instance 63 | - such as `let myjob = require('flowpipe').instance('myJob')` 64 | - instance.init(setterFn) 65 | - set instance's local variables 66 | - `setterFn(args)`: must sync function, not async. 67 | - such as `instance.init((args)=> args.data = [])` 68 | - instance.log(printFn) 69 | - print log 70 | - `printFn(args)`: must have return string 71 | - instance.timestamp(printFn) 72 | - print timestamp 73 | - `printFn(total, preModule)`: must have return string 74 | - instance.then(name, work) 75 | - add some work 76 | - also, `then, add, pipe` 77 | - `name(optional)`: work name for loop back 78 | - `work`: work function, 79 | - instance.loop(workname, condition) 80 | - also, `loop, for, loopback` 81 | - `workname`: target to loopback 82 | - `condition`: must have return boolean 83 | - instance.maxThread(number) 84 | - set parallel max thread 85 | - instance.parallel(which, work) 86 | - `which`: must have return array 87 | - `work`: work function (args, data, idx) 88 | 89 | --- 90 | 91 | ## Older Flowpipe: v0.4.0 92 | 93 | ### Quick Start: v0.4.0 94 | 95 | ![Image of Graph](resources/graph-image.png) 96 | 97 | ```javascript 98 | var flowpipe = require('flowpipe').older; 99 | 100 | flowpipe 101 | .init(function (next) { 102 | // init before start 103 | var page = 1; 104 | next(null, page); 105 | }) 106 | .pipe('start', function (next, page) { 107 | // page is that previous next function's variable 108 | setTimeout(function (err) { 109 | // next is function that proceed next pipe, parallel or loopback 110 | next(err, page); 111 | }, 1000); 112 | }) 113 | .pipe('list', function (next, page) { 114 | console.log('start page: ' + page); 115 | // preparing list for parallel 116 | var list = [{no: 1}, {no: 2}, {no: 3}]; 117 | for (var i = 0; i < list.length; i++) { 118 | list[i].page = page; 119 | list[i].delay = 3 - i; // delay time in parallel process 120 | } 121 | // next before parallel, pass only second parameter to parallel process. 122 | // second parameter must be array. 123 | // other parameters pass to next pipe or loopback. 124 | next(null, list, page); 125 | }) 126 | .parallel('parallel process', function (next, data) { 127 | // data is indicating each of list items. 128 | setTimeout(function () { 129 | data.title = 'title-' + data.no; 130 | next(null, data); 131 | }, data.delay * 1000); 132 | }) 133 | .parallel('parallel process 2', function (next, data) { 134 | // data is indicating each of list items. 135 | setTimeout(function () { 136 | data.title = 'title-' + data.no; 137 | next(null, data); 138 | }, data.delay * 1000); 139 | }) 140 | .parallel('multi-thread', function (next, data) { 141 | // this option {multiThread: true} makes your function async. 142 | // in node.js's single thread, navie logic is processing by single thread. 143 | // if you use this, all of logics are processing in multi thread. 144 | // ***WARNING: use only local variables or parameters from previous pipe. 145 | // do not use global, or functional objects from previous pipe. 146 | for(var i=0;i<100000000;i++) ; 147 | next(null, data); 148 | }, {multiThread: true}) 149 | .pipe('pass-1', function (next, parallel, page) { 150 | console.log('pass-1'); 151 | next(null, parallel, page); 152 | }) 153 | .pipe('pass-else', function (next, parallel, page) { 154 | console.log('pass-else'); 155 | next(null, parallel, page); 156 | }) 157 | .loopback('pipe-list', function (loop, next, instance, parallel, page) { 158 | // loopback(process_type-process_name, fn) 159 | // - process_type: pipe, parallel 160 | // - process_name: must defined 161 | // - fn(loop, next, instance, others...) 162 | // - loop(err, variables...): passing variable to target process 163 | // - next(err, variables...): proceeding if loop ended 164 | // - instance: maintainable variable in loop, object type {} 165 | // - others: passed from before process 166 | if (!instance.data) instance.data = []; 167 | for (var i = 0; i < parallel.length; i++) 168 | instance.data.push(parallel[i]); 169 | if (page < 5) loop(null, page + 1); 170 | else next(null, instance.data); 171 | }) 172 | .pipe('finalize', function (next, data) { 173 | next(null, data); 174 | }) 175 | .end(function (err, test) { 176 | // end(callback) or end() 177 | // this must be declared, if not all function don't working. 178 | // proceed in the end or occured error in process 179 | }) 180 | .graph('./basic-example-graph.html'); 181 | ``` 182 | 183 | ### Documents: v0.4.0 184 | 185 | - flowpipe.init(work) 186 | - initialize variables 187 | - params 188 | - `work`: function(next) 189 | - `next`: function(err, arg1, arg2 ...) 190 | - callback for next work. must be execute this function in callback. 191 | - flowpipe.pipe(name, work) 192 | - params 193 | - `name`: current work's name 194 | - `work`: function(next, arg1, arg2 ...) 195 | - `next`: function(err, arg1, arg2 ...) 196 | - `args`: previous work's results 197 | - flowpipe.parallel(name, work, opts) 198 | - processing work in parallel. 199 | - params 200 | - `name`: current work's name 201 | - `work`: function(next, list_item, args ...) 202 | - `next`: function(err, item) 203 | - `item`: sync items to list 204 | - `list_item`: parameter in previous work's first variable. must be array in previous. 205 | - `args`: previous work's results, not array 206 | - `opts`: 207 | - multiThread: default `false` 208 | - flowpipe.loopback(target, work) 209 | - params 210 | - `target`: jump to, `function`-`name` 211 | - `work`: function(loop, next, instance, args ...) 212 | - `loop`: function(err, args ...), when loop continue 213 | - `next`: function(err, args ...), when loop end 214 | - `instance`: maintenance variable in loop 215 | - flowpipe.end(work) 216 | - this function must be added in last. 217 | - params 218 | - work: function(err, args ...) 219 | - flowpipe.graph(savePath) 220 | - save graph 221 | - must be proceed after end -------------------------------------------------------------------------------- /dist/flowpipe.js: -------------------------------------------------------------------------------- 1 | (function (window) { 2 | var flowpipe = function () { 3 | function getParamNames(func, sidx) { 4 | if (!sidx) sidx = 1; 5 | 6 | try { 7 | var STRIP_COMMENTS = /((\/\/.*$)|(\/\*[\s\S]*?\*\/))/mg; 8 | var ARGUMENT_NAMES = /([^\s,]+)/g; 9 | var fnStr = func.toString().replace(STRIP_COMMENTS, ''); 10 | var result = fnStr.slice(fnStr.indexOf('(') + 1, fnStr.indexOf(')')).match(ARGUMENT_NAMES); 11 | 12 | var returnVal = ''; 13 | if (result != null) 14 | for (var i = sidx; i < result.length; i++) 15 | returnVal += result[i] + ', ' 16 | returnVal = returnVal.substring(0, returnVal.length - 2); 17 | return returnVal; 18 | } catch (e) { 19 | return ''; 20 | } 21 | } 22 | 23 | var obj = {}; 24 | var flow = {}; 25 | var flowParams = {}; 26 | var action = {}; 27 | var next_id = []; 28 | var proc = 0; 29 | var loopback_instance = {}; 30 | var opts = {}; 31 | 32 | var manager = function (args) { 33 | if (args[0]) { 34 | action.end(args[0]); 35 | return; 36 | } 37 | 38 | var next = next_id[proc]; 39 | proc++; 40 | 41 | if (!next) { 42 | var exec = 'action.end(null'; 43 | for (var i = 1; i < args.length; i++) 44 | exec += ',args[' + i + ']' 45 | exec += ')'; 46 | eval(exec); 47 | return; 48 | } 49 | 50 | var args_pipe = []; 51 | for (var i = 1; i < args.length; i++) 52 | args_pipe.push(args[i]); 53 | 54 | var type = next.split('-')[0]; 55 | action[type](next, args_pipe); 56 | }; 57 | 58 | obj.init = function (_opts, work) { 59 | if (work) { 60 | flowParams['init'] = getParamNames(work); 61 | action.init = work; 62 | opts = _opts; 63 | } else { 64 | action.init = _opts; 65 | flowParams['init'] = getParamNames(_opts); 66 | } 67 | 68 | return obj; 69 | }; 70 | 71 | obj.pipe = function (name, work) { 72 | next_id.push('pipe-' + name); 73 | flow['pipe-' + name] = work; 74 | flowParams['pipe-' + name] = getParamNames(work); 75 | return obj; 76 | }; 77 | 78 | obj.parallel = function (name, work) { 79 | next_id.push('parallel-' + name); 80 | flow['parallel-' + name] = work; 81 | flowParams['parallel-' + name] = getParamNames(work); 82 | return obj; 83 | }; 84 | 85 | obj.loopback = function (name, work) { 86 | next_id.push('loopback-' + name); 87 | flow['loopback-' + name] = work; 88 | flowParams['loopback-' + name] = getParamNames(work, 3); 89 | return obj; 90 | }; 91 | 92 | obj.jump = function (name, work) { 93 | next_id.push('jump-' + name); 94 | flow['jump-' + name] = work; 95 | flowParams['jump-' + name] = getParamNames(work); 96 | return obj; 97 | }; 98 | 99 | obj.end = function (work) { 100 | action.end = work ? work : function () { 101 | }; 102 | flowParams['end'] = getParamNames(work); 103 | action.init(function () { 104 | manager(arguments); 105 | }); 106 | return obj; 107 | }; 108 | 109 | obj.graph = function () { 110 | var idx = 0; 111 | var nodes = []; 112 | var edges = []; 113 | var typeColor = {parallel: '#0277bd', pipe: '#039be5', jump: '#00acc1', loopback: '#f57c00'}; 114 | var arrows = {to: {enabled: true, scaleFactor: 1}}; 115 | 116 | var nodeCreate = function (id, label, color) { 117 | var lb = label.length > 8 ? label.substring(0, 6) + '...' : label; 118 | return {id: id, label: lb, title: label, shape: 'box', font: {color: '#fff'}, color: color} 119 | }; 120 | 121 | nodes.push(nodeCreate('init', 'init', '#7cb342')); 122 | var nodeidx = 0; 123 | for (var i = 0; i < next_id.length; i++, nodeidx++) { 124 | var current = next_id[i]; 125 | var type = current.split('-')[0]; 126 | var name = current.substring(type.length + 1); 127 | var pre = nodes[nodeidx]; 128 | var currentParam = flowParams[current]; 129 | 130 | if (type == 'loopback') { 131 | nodes.push(nodeCreate(current, 'loopback', typeColor[type])); 132 | var jumpParam = flowParams[name]; 133 | edges.push({ 134 | from: current, 135 | to: name, 136 | arrows: arrows, 137 | label: jumpParam, 138 | color: '#039be5' 139 | }); 140 | 141 | edges.push({ 142 | from: pre.id, 143 | to: current, 144 | arrows: arrows, 145 | label: currentParam 146 | }); 147 | } else if (type == 'parallel') { 148 | if (pre.id.split('-')[0] == 'parallel') { 149 | pre = nodes.splice(nodeidx, 1)[0]; 150 | nodeidx--; 151 | } 152 | 153 | for (var j = 0; j < 2; j++) { 154 | var insertnode = nodeCreate(j + '-' + current, name, typeColor[type]); 155 | nodes.push(insertnode); 156 | var from = pre.id; 157 | if (pre.id.split('-')[0] == 'parallel') 158 | from = j + '-' + from; 159 | edges.push({ 160 | from: from, 161 | to: j + '-' + current, 162 | arrows: arrows, 163 | label: currentParam 164 | }); 165 | edges.push({from: j + '-' + current, to: current, arrows: arrows}); 166 | nodeidx++; 167 | } 168 | nodes.push(nodeCreate(current, 'Sync', typeColor[type])); 169 | } else { 170 | nodes.push(nodeCreate(current, name, typeColor[type])); 171 | edges.push({ 172 | from: pre.id, 173 | to: current, 174 | arrows: arrows, 175 | label: currentParam 176 | }); 177 | } 178 | } 179 | nodes.push(nodeCreate('end', 'end', '#f57c00')); 180 | edges.push({ 181 | from: next_id[next_id.length - 1], 182 | to: 'end', 183 | arrows: arrows, 184 | label: flowParams['end'] 185 | }); 186 | 187 | return {nodes: nodes, edges: edges}; 188 | }; 189 | 190 | action.pipe = function (next, args) { 191 | var work = flow[next]; 192 | 193 | var manage = function () { 194 | manager(arguments); 195 | }; 196 | 197 | var exec = 'work(manage'; 198 | for (var i = 0; i < args.length; i++) 199 | exec += ',args[' + i + ']' 200 | exec += ')'; 201 | eval(exec); 202 | }; 203 | 204 | action.parallel = function (next, args) { 205 | var work = flow[next]; 206 | 207 | if (typeof args[0] != 'object') { 208 | var err = new Error("Parameter Type Error!"); 209 | err.msg = 'parameter allow only array'; 210 | action.end(err); 211 | return; 212 | } 213 | 214 | var manage = function () { 215 | if (status.length == args[0].length) { 216 | for (var i = 0; i < status.length; i++) 217 | if (!status[i]) 218 | return; 219 | 220 | for (var i = 0; i < err.length; i++) 221 | if (err[i]) 222 | return action.end(err[i]); 223 | 224 | args[0] = result; 225 | args.unshift(null); 226 | manager(args); 227 | } 228 | }; 229 | 230 | var result = []; 231 | var status = []; 232 | var err = []; 233 | 234 | var parallel_fn = function (parallel_obj) { 235 | return function () { 236 | err[parallel_obj] = arguments[0]; 237 | result[parallel_obj] = arguments[1]; 238 | status[parallel_obj] = true; 239 | manage(); 240 | } 241 | }; 242 | 243 | if (!args[0] || args[0].length == 0) { 244 | args.unshift(null); 245 | return manager(args); 246 | } 247 | 248 | try { 249 | for (var i = 0; i < args[0].length; i++) { 250 | var exec = 'work(parallel_fn(' + i + '),args[0][i]'; 251 | for (var j = 1; j < args.length; j++) 252 | exec += ',args[' + j + ']' 253 | exec += ')'; 254 | eval(exec); 255 | } 256 | } catch (e) { 257 | } 258 | }; 259 | 260 | action.loopback = function (next, args) { 261 | var work = flow[next]; 262 | var loopback_target = next.substring(9); 263 | if (!loopback_instance[next]) loopback_instance[next] = {}; 264 | 265 | var loop = function () { 266 | for (var i = 0; i < next_id.length; i++) 267 | if (next_id[i] == loopback_target) 268 | proc = i; 269 | manager(arguments); 270 | }; 271 | 272 | var next_fn = function () { 273 | manager(arguments); 274 | delete loopback_instance[next]; 275 | }; 276 | 277 | var exec = 'work(loop,next_fn,loopback_instance[next]'; 278 | for (var i = 0; i < args.length; i++) 279 | exec += ',args[' + i + ']' 280 | exec += ')'; 281 | eval(exec); 282 | }; 283 | 284 | action.jump = function (next, args) { 285 | var work = flow[next]; 286 | 287 | var jump = function () { 288 | var jump_to = arguments[0]; 289 | arguments[0] = null; 290 | 291 | for (var i = 0; i < next_id.length; i++) 292 | if (next_id[i] == jump_to) 293 | proc = i; 294 | 295 | manager(arguments); 296 | }; 297 | 298 | var exec = 'work(jump'; 299 | for (var i = 0; i < args.length; i++) 300 | exec += ',args[' + i + ']' 301 | exec += ')'; 302 | eval(exec); 303 | }; 304 | 305 | return obj; 306 | }; 307 | 308 | var obj = {}; 309 | obj.init = function (next) { 310 | return flowpipe().init(next); 311 | }; 312 | 313 | window.flowpipe = obj; 314 | })(this); -------------------------------------------------------------------------------- /v0.4.js: -------------------------------------------------------------------------------- 1 | module.exports = (function () { 2 | var flowpipe = function () { 3 | function getParamNames(func, sidx) { 4 | if (!sidx) sidx = 1; 5 | 6 | try { 7 | var STRIP_COMMENTS = /((\/\/.*$)|(\/\*[\s\S]*?\*\/))/mg; 8 | var ARGUMENT_NAMES = /([^\s,]+)/g; 9 | var fnStr = func.toString().replace(STRIP_COMMENTS, ''); 10 | var result = fnStr.slice(fnStr.indexOf('(') + 1, fnStr.indexOf(')')).match(ARGUMENT_NAMES); 11 | 12 | var returnVal = ''; 13 | if (result != null) 14 | for (var i = sidx; i < result.length; i++) 15 | returnVal += result[i] + ', ' 16 | returnVal = returnVal.substring(0, returnVal.length - 2); 17 | return returnVal; 18 | } catch (e) { 19 | return ''; 20 | } 21 | } 22 | 23 | var obj = {}; 24 | var flow = {}; 25 | var flowParams = {}; 26 | var action = {}; 27 | var next_id = []; 28 | var proc = 0; 29 | var loopback_instance = {}; 30 | var opts = {}; 31 | 32 | var manager = function (args) { 33 | if (args[0]) { 34 | action.end(args[0]); 35 | return; 36 | } 37 | 38 | var next = next_id[proc]; 39 | proc++; 40 | 41 | if (!next) { 42 | var exec = 'action.end(null'; 43 | for (var i = 1; i < args.length; i++) 44 | exec += ',args[' + i + ']' 45 | exec += ')'; 46 | eval(exec); 47 | return; 48 | } 49 | 50 | var args_pipe = []; 51 | for (var i = 1; i < args.length; i++) 52 | args_pipe.push(args[i]); 53 | 54 | var type = next.split('-')[0]; 55 | action[type](next, args_pipe); 56 | }; 57 | 58 | obj.init = function (work) { 59 | flowParams['init'] = getParamNames(work); 60 | action.init = work; 61 | return obj; 62 | }; 63 | 64 | obj.pipe = function (name, work) { 65 | next_id.push('pipe-' + name); 66 | flow['pipe-' + name] = work; 67 | flowParams['pipe-' + name] = getParamNames(work); 68 | return obj; 69 | }; 70 | 71 | obj.parallel = function (name, work, _opts) { 72 | next_id.push('parallel-' + name); 73 | flow['parallel-' + name] = work; 74 | flowParams['parallel-' + name] = getParamNames(work); 75 | opts['parallel-' + name] = _opts; 76 | return obj; 77 | }; 78 | 79 | obj.loopback = function (name, work) { 80 | next_id.push('loopback-' + name); 81 | flow['loopback-' + name] = work; 82 | flowParams['loopback-' + name] = getParamNames(work, 3); 83 | return obj; 84 | }; 85 | 86 | obj.jump = function (name, work) { 87 | next_id.push('jump-' + name); 88 | flow['jump-' + name] = work; 89 | flowParams['jump-' + name] = getParamNames(work); 90 | return obj; 91 | }; 92 | 93 | obj.end = function (work) { 94 | action.end = work ? work : function () { 95 | }; 96 | flowParams['end'] = getParamNames(work); 97 | action.init(function () { 98 | manager(arguments); 99 | }); 100 | return obj; 101 | }; 102 | 103 | obj.graph = function (save_path) { 104 | var idx = 0; 105 | var nodes = []; 106 | var edges = []; 107 | var typeColor = {parallel: '#0277bd', pipe: '#039be5', jump: '#00acc1', loopback: '#f57c00'}; 108 | var arrows = {to: {enabled: true, scaleFactor: 1}}; 109 | 110 | var nodeCreate = function (id, label, color) { 111 | var lb = label.length > 8 ? label.substring(0, 6) + '...' : label; 112 | return {id: id, label: lb, title: label, shape: 'box', font: {color: '#fff'}, color: color} 113 | }; 114 | 115 | nodes.push(nodeCreate('init', 'init', '#7cb342')); 116 | var nodeidx = 0; 117 | for (var i = 0; i < next_id.length; i++, nodeidx++) { 118 | var current = next_id[i]; 119 | var type = current.split('-')[0]; 120 | var name = current.substring(type.length + 1); 121 | var pre = nodes[nodeidx]; 122 | var currentParam = flowParams[current]; 123 | 124 | if (type == 'loopback') { 125 | nodes.push(nodeCreate(current, 'loopback', typeColor[type])); 126 | var jumpParam = flowParams[name]; 127 | edges.push({ 128 | from: current, 129 | to: name, 130 | arrows: arrows, 131 | label: jumpParam, 132 | color: '#039be5' 133 | }); 134 | 135 | edges.push({ 136 | from: pre.id, 137 | to: current, 138 | arrows: arrows, 139 | label: currentParam 140 | }); 141 | } else if (type == 'parallel') { 142 | if (pre.id.split('-')[0] == 'parallel') { 143 | pre = nodes.splice(nodeidx, 1)[0]; 144 | nodeidx--; 145 | } 146 | 147 | for (var j = 0; j < 2; j++) { 148 | var insertnode = nodeCreate(j + '-' + current, name, typeColor[type]); 149 | nodes.push(insertnode); 150 | var from = pre.id; 151 | if (pre.id.split('-')[0] == 'parallel') 152 | from = j + '-' + from; 153 | edges.push({ 154 | from: from, 155 | to: j + '-' + current, 156 | arrows: arrows, 157 | label: currentParam 158 | }); 159 | edges.push({from: j + '-' + current, to: current, arrows: arrows}); 160 | nodeidx++; 161 | } 162 | nodes.push(nodeCreate(current, 'Sync', typeColor[type])); 163 | } else { 164 | nodes.push(nodeCreate(current, name, typeColor[type])); 165 | edges.push({ 166 | from: pre.id, 167 | to: current, 168 | arrows: arrows, 169 | label: currentParam 170 | }); 171 | } 172 | } 173 | nodes.push(nodeCreate('end', 'end', '#f57c00')); 174 | edges.push({ 175 | from: next_id[next_id.length - 1], 176 | to: 'end', 177 | arrows: arrows, 178 | label: flowParams['end'] 179 | }); 180 | 181 | if (save_path) { 182 | var fs = require('fs'); 183 | var html = fs.readFileSync(__dirname + '/resources/graph.html') + ''; 184 | html = html.replace('flowpipegraphdata', JSON.stringify({nodes: nodes, edges: edges})); 185 | fs.writeFileSync(save_path, html); 186 | } 187 | 188 | return {nodes: nodes, edges: edges}; 189 | }; 190 | 191 | action.pipe = function (next, args) { 192 | var work = flow[next]; 193 | 194 | var manage = function () { 195 | manager(arguments); 196 | }; 197 | 198 | var exec = 'work(manage'; 199 | for (var i = 0; i < args.length; i++) 200 | exec += ',args[' + i + ']' 201 | exec += ')'; 202 | eval(exec); 203 | }; 204 | 205 | action.parallel = function (next, args) { 206 | if (!opts[next]) opts[next] = {}; 207 | var work = flow[next]; 208 | 209 | if (typeof args[0] != 'object') { 210 | var err = new Error("Parameter Type Error!"); 211 | err.msg = 'parameter allow only array'; 212 | action.end(err); 213 | return; 214 | } 215 | 216 | var manage = function () { 217 | if (status.length == args[0].length) { 218 | for (var i = 0; i < status.length; i++) 219 | if (!status[i]) 220 | return; 221 | for (var i = 0; i < err.length; i++) 222 | if (err[i]) 223 | return action.end(err[i]); 224 | 225 | args[0] = result; 226 | args.unshift(null); 227 | manager(args); 228 | } 229 | }; 230 | 231 | var result = []; 232 | var status = []; 233 | var err = []; 234 | 235 | var parallel_fn = function (parallel_obj) { 236 | return function () { 237 | err[parallel_obj] = arguments[0]; 238 | result[parallel_obj] = arguments[1]; 239 | status[parallel_obj] = true; 240 | manage(); 241 | } 242 | }; 243 | 244 | if (!args[0] || args[0].length == 0) { 245 | args.unshift(null); 246 | return manager(args); 247 | } 248 | 249 | var execArray = []; 250 | for (var i = 0; i < args[0].length; i++) { 251 | var exec = 'work(parallel_fn(' + i + '),args[0][' + i + ']'; 252 | for (var j = 1; j < args.length; j++) 253 | exec += ',args[' + j + ']' 254 | exec += ')'; 255 | execArray.push(exec); 256 | } 257 | 258 | if (opts[next].multiThread) { 259 | var full_script = 'var args = ' + JSON.stringify(args) + ';\n\n'; 260 | var script_parallel_fn = 'var parallel_fn = ' + (function (parallel_obj) { 261 | return function () { 262 | var parallelResult = {}; 263 | parallelResult.id = parallel_obj; 264 | parallelResult.err = arguments[0]; 265 | parallelResult.result = arguments[1]; 266 | parallelResult.status = true; 267 | console.log('FLOWPIPE-PARALLEL-RESULT' + JSON.stringify(parallelResult)); 268 | } 269 | }).toString(); 270 | var script_work = 'var work = ' + work.toString(); 271 | full_script += script_parallel_fn + '\n\n' + script_work + '\n\n'; 272 | 273 | for (var i = 0; i < execArray.length; i++) { 274 | var script_fn = full_script + execArray[i]; 275 | script_fn = encodeURI(script_fn); 276 | 277 | function runScript(script, pid) { 278 | var spawn = require('child_process').spawn; 279 | var bat = spawn('node', [__dirname + '/libs/async-eval.js', script]); 280 | var flowpipe_parallel_result = null; 281 | var multiThreadError = null; 282 | bat.stdout.on('data', function (data) { 283 | data = data + ''; 284 | if (data.indexOf('FLOWPIPE-PARALLEL-RESULT') != -1) { 285 | flowpipe_parallel_result = JSON.parse(data.replace('FLOWPIPE-PARALLEL-RESULT', '')); 286 | return; 287 | } 288 | if ((data + '').length > 0) 289 | console.log((data + '').trim()); 290 | }); 291 | 292 | bat.stderr.on('data', function (data) { 293 | multiThreadError = data + ''; 294 | }); 295 | 296 | bat.on('exit', function () { 297 | try { 298 | err[flowpipe_parallel_result.id] = flowpipe_parallel_result.err; 299 | result[flowpipe_parallel_result.id] = flowpipe_parallel_result.result; 300 | status[flowpipe_parallel_result.id] = flowpipe_parallel_result.status; 301 | } catch (e) { 302 | } 303 | 304 | if (multiThreadError) { 305 | err[pid] = multiThreadError; 306 | status[pid] = true; 307 | result[pid] = null; 308 | } 309 | 310 | manage(); 311 | return; 312 | }); 313 | } 314 | 315 | runScript(script_fn, i); 316 | } 317 | } else { 318 | for (var i = 0; i < execArray.length; i++) { 319 | eval(execArray[i]); 320 | } 321 | } 322 | }; 323 | 324 | action.loopback = function (next, args) { 325 | var work = flow[next]; 326 | var loopback_target = next.substring(9); 327 | if (!loopback_instance[next]) loopback_instance[next] = {}; 328 | 329 | var loop = function () { 330 | for (var i = 0; i < next_id.length; i++) 331 | if (next_id[i] == loopback_target) 332 | proc = i; 333 | manager(arguments); 334 | }; 335 | 336 | var next_fn = function () { 337 | manager(arguments); 338 | delete loopback_instance[next]; 339 | }; 340 | 341 | var exec = 'work(loop,next_fn,loopback_instance[next]'; 342 | for (var i = 0; i < args.length; i++) 343 | exec += ',args[' + i + ']' 344 | exec += ')'; 345 | eval(exec); 346 | }; 347 | 348 | action.jump = function (next, args) { 349 | var work = flow[next]; 350 | 351 | var jump = function () { 352 | var jump_to = arguments[0]; 353 | arguments[0] = null; 354 | 355 | for (var i = 0; i < next_id.length; i++) 356 | if (next_id[i] == jump_to) 357 | proc = i; 358 | 359 | manager(arguments); 360 | }; 361 | 362 | var exec = 'work(jump'; 363 | for (var i = 0; i < args.length; i++) 364 | exec += ',args[' + i + ']' 365 | exec += ')'; 366 | eval(exec); 367 | }; 368 | 369 | return obj; 370 | }; 371 | 372 | var obj = {}; 373 | obj.init = function (next) { 374 | return flowpipe().init(next); 375 | }; 376 | return obj; 377 | })(); --------------------------------------------------------------------------------