├── .gitignore ├── .travis.yml ├── LICENSE.txt ├── README.md ├── cli └── pipes2js.js ├── index.js ├── lib ├── engine.js ├── import-pipe.js └── subkey.js ├── modules ├── count │ ├── count.js │ ├── package.json │ └── test.js ├── createrss │ ├── createrss.js │ └── package.json ├── datebuilder │ ├── datebuilder.js │ └── package.json ├── dateformat │ ├── dateformat.js │ └── package.json ├── dateinput │ ├── dateinput.js │ └── package.json ├── feedautodiscovery │ ├── feedautodiscovery.js │ └── package.json ├── fetch │ ├── fetch.js │ └── package.json ├── fetchsitefeed │ ├── fetchsitefeed.js │ └── package.json ├── filter │ ├── filter.js │ └── package.json ├── itembuilder │ ├── itembuilder.js │ ├── package.json │ └── test.js ├── locationbuilder │ ├── locationbuilder.js │ └── package.json ├── locationinput │ ├── locationinput.js │ ├── package.json │ └── test.js ├── loop │ ├── loop.js │ └── package.json ├── numberinput │ ├── numberinput.js │ └── package.json ├── output │ ├── output.js │ ├── package.json │ └── test.js ├── privateinput │ ├── package.json │ └── privateinput.js ├── privatestring │ ├── package.json │ └── privatestring.js ├── regex │ ├── package.json │ └── regex.js ├── rename │ ├── package.json │ └── rename.js ├── reverse │ ├── package.json │ ├── reverse.js │ └── test.js ├── rssitembuilder │ ├── package.json │ └── rssitembuilder.js ├── simplemath │ ├── package.json │ └── simplemath.js ├── sort │ ├── package.json │ └── sort.js ├── split │ ├── package.json │ └── split.js ├── strconcat │ ├── package.json │ └── strconcat.js ├── stringtokenizer │ ├── package.json │ └── stringtokenizer.js ├── strregex │ ├── package.json │ └── strregex.js ├── strreplace │ ├── package.json │ └── strreplace.js ├── subelement │ ├── package.json │ └── subelement.js ├── substr │ ├── package.json │ └── substr.js ├── tail │ ├── package.json │ └── tail.js ├── termextraction │ ├── package.json │ └── termextraction.js ├── textinput │ ├── package.json │ ├── test.js │ └── textinput.js ├── truncate │ ├── package.json │ └── truncate.js ├── union │ ├── package.json │ └── union.js ├── uniq │ ├── package.json │ ├── test.js │ └── uniq.js ├── urlbuilder │ ├── package.json │ └── urlbuilder.js ├── urlinput │ ├── package.json │ ├── test.js │ └── urlinput.js ├── webservice │ ├── package.json │ └── webservice.js ├── xpathfetchpage │ ├── package.json │ └── xpathfetchpage.js └── yql │ ├── package.json │ └── yql.js ├── package.json └── tests ├── test-import.js └── test-subkey.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 0.8 -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) <2012-2012> 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 5 | 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pipes2js 2 | 3 | Compile Yahoo! Pipes to Javascript (Node.js) 4 | 5 | 6 | ## Design 7 | 8 | This tool lets you import Yahoo! Pipes and run them on any machine with Node.js. 9 | 10 | The pipe is converted to a javascript file, called *decider.js*, which controls the execution logic of the workflow. 11 | 12 | Each Yahoo module is coded as a separate Javascript module. 13 | 14 | * The pipe can be executed localy through a simple run engine (called by the generated run.js) 15 | * or executed on Amazon SimpleWorkflow (SWF) 16 | * The rsulting modules can be used for other Amazon SWF projects through the [aws-swf library](https://github.com/neyric/aws-swf) 17 | 18 | ## Installation 19 | 20 | $ [sudo] npm install -g pipes2js 21 | 22 | ## Usage 23 | 24 | Import the pipe : 25 | 26 | $ pipes2js xOE_1Z8C3RGmkQrul7okhQ 27 | 28 | This will create a pipes/xOE_1Z8C3RGmkQrul7okhQ/ directory, which contains an npm package. 29 | The resulting package depends on the pipes2js package. Let's install it : 30 | 31 | $ cd pipes/xOE_1Z8C3RGmkQrul7okhQ/ 32 | $ npm install . 33 | 34 | You can then run it : 35 | 36 | $ node run.js 37 | 38 | ## More 39 | 40 | [Current module implementations](https://github.com/neyric/pipes2js/wiki/Yahoo-Pipes-modules) 41 | 42 | [EXPERIMENTAL: Running imported pipes on Amazon SimpleWorkflow (SWF)](https://github.com/neyric/pipes2js/wiki/Running-on-Amazon-SimpleWorkflow-SWF) 43 | 44 | ## Tests 45 | 46 | Run tests : 47 | 48 | 49 | npm test 50 | 51 | 52 | will perform : 53 | 54 | 55 | vows --spec modules/*/test.js tests/* 56 | 57 | 58 | [![build status](https://secure.travis-ci.org/neyric/pipes2js.png)](http://travis-ci.org/neyric/pipes2js) 59 | 60 | 61 | ## Credits 62 | 63 | Inspired by [pipe2py](https://github.com/ggaughan/pipe2py) 64 | 65 | -------------------------------------------------------------------------------- /cli/pipes2js.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var path = require('path'), 4 | pipe2js = require(path.join(__dirname, '..', 'index')); 5 | 6 | function usage() { 7 | console.log("Usage: pipes2js (pipeId)"); 8 | } 9 | 10 | if (process.argv.length < 3) { 11 | usage(); 12 | process.exit(0); 13 | } 14 | var pipeId = process.argv[2]; 15 | 16 | pipe2js.importPipe(pipeId); 17 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 2 | var engine = require('./lib/engine'), 3 | importPipe = require('./lib/import-pipe'); 4 | 5 | exports.run = engine.run; 6 | 7 | exports.importPipe = importPipe.importPipe; 8 | exports.fetchPipe = importPipe.fetchPipe; 9 | exports.pipe2decider = importPipe.pipe2decider; 10 | exports.conf2input = importPipe.conf2input; 11 | exports.pathsForTerminals = importPipe.pathsForTerminals; 12 | -------------------------------------------------------------------------------- /lib/engine.js: -------------------------------------------------------------------------------- 1 | var path = require('path'), 2 | vm = require('vm'); 3 | 4 | var decide = function (deciderCode, state) { 5 | 6 | var decisions = null; 7 | 8 | var sandbox = { 9 | 10 | just_started: (Object.keys(state).length === 0), 11 | 12 | schedule: function (id, params) { 13 | if (!decisions) { decisions = []; } 14 | decisions.push({ 15 | id: id, 16 | params: params 17 | }); 18 | }, 19 | scheduled: function (id) { 20 | return !!state[id]; 21 | }, 22 | waiting_for: function () { 23 | if (!decisions) { decisions = []; } 24 | }, 25 | completed: function (id) { 26 | return !!state[id] && state[id].state === 'completed'; 27 | }, 28 | stop: function () { 29 | }, 30 | results: function (id) { 31 | return state[id].results; 32 | }, 33 | workflow_input: function () { 34 | // TODO ! 35 | } 36 | }; 37 | 38 | vm.runInNewContext(deciderCode, sandbox, 'pipe.vm'); 39 | return decisions; 40 | }; 41 | 42 | 43 | 44 | var Task = function (id, cb) { 45 | this.id = id; 46 | this.cb = cb; 47 | }; 48 | Task.prototype = { 49 | respondCompleted: function (result) { 50 | this.cb(this.id, result); 51 | } 52 | }; 53 | 54 | 55 | var run = function (state, deciderCode, cb) { 56 | 57 | var decisions = decide(deciderCode, state); 58 | 59 | if (!decisions) { return; } 60 | 61 | var respondCompleted = function (id, result) { 62 | state[id].state = 'completed'; 63 | state[id].results = result; 64 | run(state, deciderCode, cb); 65 | }; 66 | var i; 67 | for (i = 0; i < decisions.length; i += 1) { 68 | var decision = decisions[i]; 69 | 70 | state[decision.id] = { 71 | state: 'started' 72 | }; 73 | 74 | var activityType = decision.params.activityType; 75 | var input = JSON.stringify(decision.params.input); 76 | 77 | var task = new Task(decision.id, respondCompleted); 78 | task.config = { 79 | input: input 80 | }; 81 | 82 | var worker = require(path.join(__dirname, '..', 'modules', activityType)).worker; 83 | console.log("Running " + decision.id + " (" + activityType + ")"); 84 | worker(task); 85 | 86 | if (activityType === 'output') { 87 | cb(null, state[decision.id].results._OUTPUT, state); 88 | } 89 | } 90 | 91 | }; 92 | 93 | 94 | exports.run = run; 95 | -------------------------------------------------------------------------------- /lib/import-pipe.js: -------------------------------------------------------------------------------- 1 | 2 | var fs = require('fs'), 3 | request = require('request'); 4 | 5 | 6 | var pipe2js = { 7 | 8 | importPipe: function (pipeId) { 9 | 10 | pipe2js.fetchPipe(pipeId, function (pipe) { 11 | 12 | var deciderCode = pipe2js.pipe2decider(pipe); 13 | 14 | try { 15 | fs.mkdirSync('pipes'); 16 | } catch (ex) {} 17 | 18 | try { 19 | console.log("Writing module in pipes/" + pipeId); 20 | fs.mkdirSync('pipes/' + pipeId); 21 | } catch (ex) {} 22 | 23 | fs.writeFileSync('pipes/' + pipeId + '/package.json', JSON.stringify({ 24 | "name" : pipeId, 25 | "main" : "./run.js", 26 | "version": "0.0.1", 27 | "dependencies": { 28 | "pipes2js": "0.0.1" 29 | } 30 | }, null, 3)); 31 | 32 | fs.writeFileSync('pipes/' + pipeId + '/decider.js', deciderCode); 33 | 34 | 35 | // run.js 36 | var runCode = []; 37 | runCode.push("var fs = require('fs'), path = require('path');"); 38 | runCode.push(""); 39 | runCode.push("var deciderCode = fs.readFileSync(path.join(__dirname, 'decider.js'), 'utf8');"); 40 | runCode.push(""); 41 | runCode.push("var run = require('pipes2js').run;"); 42 | runCode.push(""); 43 | runCode.push("run({}, deciderCode, function (err, results, state) {"); 44 | runCode.push(" console.log(JSON.stringify(results, null, 3));"); 45 | runCode.push("});"); 46 | 47 | fs.writeFileSync('pipes/' + pipeId + '/run.js', runCode.join('\n')); 48 | 49 | }); 50 | 51 | }, 52 | 53 | fetchPipe: function (pipeId, cb) { 54 | request('http://pipes.yahoo.com/pipes/pipe.info?_id=' + pipeId + '&_out=json&format=json', function (error, response, body) { 55 | if (!error && response.statusCode === 200) { 56 | var pipe = JSON.parse(JSON.parse(body).PIPE.working); 57 | cb(pipe); 58 | } 59 | }); 60 | }, 61 | 62 | conf2input: function (conf) { 63 | var i, k; 64 | if (Array.isArray(conf)) { 65 | for (i = 0; i < conf.length; i += 1) { 66 | if (conf[i].hasOwnProperty("terminal")) { // we remove config with "terminal" 67 | conf[i] = null; 68 | } else { 69 | conf[i] = pipe2js.conf2input(conf[i]); 70 | } 71 | } 72 | } else if (conf.hasOwnProperty('type') && conf.hasOwnProperty("value")) { 73 | return conf.value; 74 | } else if (typeof conf === 'object') { 75 | for (k in conf) { 76 | if (conf.hasOwnProperty(k)) { 77 | 78 | if (conf[k].hasOwnProperty("terminal")) { // we remove config with "terminal" 79 | conf[k] = undefined; 80 | } else { 81 | if (k === 'embed') { 82 | conf[k] = { 83 | type: conf[k].value.type, 84 | id: conf[k].value.id, 85 | conf: pipe2js.conf2input(conf[k].value.conf), 86 | parentModuleId: conf[k].value.parentModuleId 87 | }; 88 | } else { 89 | conf[k] = pipe2js.conf2input(conf[k]); 90 | } 91 | } 92 | } 93 | } 94 | } 95 | 96 | return conf; 97 | }, 98 | 99 | /** 100 | * Returns the following structure for each module config : 101 | * 102 | * { 103 | * '1_part': { 104 | * path: ['embed', 0, 'value'], 105 | * subkey: 'description' 106 | * } 107 | * } 108 | * 109 | */ 110 | pathsForTerminals: function (conf, pathItems, pathsForTerminals) { 111 | 112 | if (!pathItems) { pathItems = []; } 113 | if (!pathsForTerminals) { pathsForTerminals = {}; } 114 | 115 | var i, k; 116 | if (Array.isArray(conf)) { 117 | for (i = 0; i < conf.length; i += 1) { 118 | if (conf[i].hasOwnProperty("terminal")) { 119 | pathsForTerminals[conf[i].terminal] = { path: pathItems.concat(i), subkey: conf[i].subkey }; 120 | } else { 121 | pipe2js.pathsForTerminals(conf[i], pathItems.concat(i), pathsForTerminals); 122 | } 123 | } 124 | } else if (conf.hasOwnProperty('type') && conf.hasOwnProperty('terminal')) { 125 | pathsForTerminals[conf.terminal] = {path: pathItems.concat([]), subkey: conf.subkey }; 126 | } else if (typeof conf === 'object') { 127 | for (k in conf) { 128 | if (conf.hasOwnProperty(k)) { 129 | if (conf[k].hasOwnProperty("terminal")) { 130 | pathsForTerminals[conf[k].terminal] = { path: pathItems.concat(k), subkey: conf[k].subkey }; 131 | } else { 132 | pipe2js.pathsForTerminals(conf[k], pathItems.concat(k), pathsForTerminals); 133 | } 134 | } 135 | } 136 | } 137 | 138 | return pathsForTerminals; 139 | }, 140 | 141 | pipe2decider: function (pipe) { 142 | 143 | /** 144 | * Build a structure representing the modules (activities) and input and output wires 145 | */ 146 | var modules = pipe.modules; 147 | 148 | var modulesById = {}, 149 | embeddedModulesById = {}; 150 | 151 | modules.forEach(function (m) { 152 | // Index modules by Id 153 | modulesById[m.id] = m; 154 | 155 | // initialize wires list 156 | m.inputWires = []; 157 | m.outputWires = []; 158 | 159 | if (m.type === "loop") { 160 | var submodule = m.conf.embed.value; 161 | embeddedModulesById[submodule.id] = submodule; 162 | submodule.parentModuleId = m.id; 163 | } 164 | 165 | }); 166 | 167 | var wires = pipe.wires; 168 | wires.forEach(function (w) { 169 | modulesById[w.src.moduleid].outputWires.push(w); 170 | 171 | if (modulesById[w.tgt.moduleid]) { 172 | modulesById[w.tgt.moduleid].inputWires.push(w); 173 | } else if (embeddedModulesById[w.tgt.moduleid]) { 174 | modulesById[embeddedModulesById[w.tgt.moduleid].parentModuleId].inputWires.push(w); 175 | } 176 | }); 177 | 178 | 179 | /** 180 | * Generate decider code 181 | */ 182 | 183 | var deciderCode = []; 184 | 185 | deciderCode.push("/*globals just_started,schedule,scheduled,completed,workflow_input,stop,results,waiting_for */"); 186 | 187 | // Modules to run whren just_started = modules with no input wire 188 | deciderCode.push("if (just_started) {"); 189 | modules.filter(function (m) { 190 | if (m.inputWires.length === 0) { 191 | 192 | var params = { 193 | activityType: m.type, 194 | input: pipe2js.conf2input(m.conf) 195 | }; 196 | 197 | deciderCode.push(" schedule(" + JSON.stringify(m.id) + ", " + JSON.stringify(params, null, 4).replace(/\n/g, '\n ') + ");"); 198 | } 199 | }); 200 | deciderCode.push("}"); 201 | 202 | 203 | 204 | // For Each module with inputs 205 | modules.filter(function (m) { 206 | if (m.inputWires.length > 0) { 207 | 208 | var conditionLine = "if (!scheduled(" + JSON.stringify(m.id) + ")"; 209 | 210 | // For each incoming module : 211 | var incomingValues = {}; 212 | m.inputWires.map(function (w) { 213 | var m = modulesById[w.src.moduleid]; 214 | conditionLine += " && completed(" + JSON.stringify(m.id) + ")"; 215 | 216 | // prepare the incomingValues structure 217 | incomingValues[w.tgt.id] = w.src; 218 | }); 219 | 220 | conditionLine += ") {"; 221 | deciderCode.push(conditionLine); 222 | 223 | // walk the m.conf to find "type": "terminal" and remember the path 224 | // Important: do this before calling conf2input(m.conf) which change the config structure 225 | var pathsForTerminals = pipe2js.pathsForTerminals(m.conf); 226 | 227 | var params = { 228 | activityType: m.type, 229 | input: pipe2js.conf2input(m.conf) 230 | }; 231 | 232 | deciderCode.push(" var params = " + JSON.stringify(params, null, 4).replace(/\n/g, '\n ') + ";"); 233 | 234 | // Add the incomingValues 235 | var k; 236 | var pathify = function (i) { return '[' + JSON.stringify(i) + ']'; }; 237 | for (k in incomingValues) { 238 | if (incomingValues.hasOwnProperty(k)) { 239 | var src = incomingValues[k]; 240 | var pathToItem; 241 | // generate path to item 242 | if (!pathsForTerminals[k]) { 243 | pathToItem = '[' + JSON.stringify(k) + ']'; 244 | } else { 245 | if (pathsForTerminals[k].path.length > 0 && pathsForTerminals[k].path[0] === 'embed') { 246 | pathsForTerminals[k].path = ['embed'].concat(pathsForTerminals[k].path.slice(2)); 247 | } 248 | pathToItem = pathsForTerminals[k].path.map(pathify).join(''); 249 | } 250 | 251 | var srcPathToItem = [src.id]; 252 | 253 | // if subkey, add it to the path 254 | if (!!pathsForTerminals[k] && pathsForTerminals[k].subkey) { 255 | srcPathToItem.push(pathsForTerminals[k].subkey); 256 | } 257 | 258 | var srcPathStr = srcPathToItem.map(pathify).join(''); 259 | 260 | deciderCode.push(" params.input" + pathToItem + " = results(" + JSON.stringify(src.moduleid) + ")" + srcPathStr + ";"); 261 | } 262 | } 263 | 264 | deciderCode.push(" schedule(" + JSON.stringify(m.id) + ", params);"); 265 | 266 | deciderCode.push("}"); 267 | 268 | 269 | // TODO 270 | /*if (m.inputWires.length > 1) { 271 | deciderCode.push("if ( (scheduled('sw-61') && !completed('sw-61')) || (scheduled('sw-228') && !completed('sw-228')) || (scheduled('sw-232') && !completed('sw-232')) ) {"); 272 | deciderCode.push(" waiting_for('sw-61', 'sw-228', 'sw-232');"); // TODO: fix in aws-swf, if decisions, dont override this.decisions = []; 273 | deciderCode.push("}"); 274 | }*/ 275 | 276 | } 277 | }); 278 | 279 | // Stop = module _OUTPUT completed 280 | // TODO: find the id of the activityType === 'output' 281 | deciderCode.push("if (completed('_OUTPUT')) {"); 282 | deciderCode.push(" stop('finished !');"); 283 | deciderCode.push("}"); 284 | 285 | return deciderCode.join('\n'); 286 | } 287 | 288 | 289 | }; 290 | 291 | 292 | exports.importPipe = pipe2js.importPipe; 293 | exports.fetchPipe = pipe2js.fetchPipe; 294 | exports.pipe2decider = pipe2js.pipe2decider; 295 | exports.conf2input = pipe2js.conf2input; 296 | exports.pathsForTerminals = pipe2js.pathsForTerminals; 297 | -------------------------------------------------------------------------------- /lib/subkey.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | exports.subkey = function (item, subkey) { 4 | var src = item; 5 | 6 | var pathItems = subkey.split('.'), i; 7 | 8 | if (pathItems.length > 1) { 9 | for (i = 0; i < pathItems.length - 1; i += 1) { 10 | src = src[pathItems[i]]; 11 | } 12 | } 13 | 14 | return src[pathItems[pathItems.length - 1]]; 15 | }; 16 | 17 | -------------------------------------------------------------------------------- /modules/count/count.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | task.respondCompleted({ 7 | _OUTPUT: input._INPUT.length 8 | }); 9 | 10 | }; 11 | 12 | -------------------------------------------------------------------------------- /modules/count/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "count", 3 | "main" : "./count.js", 4 | "version": "0.0.1", 5 | "description": "count module" 6 | } -------------------------------------------------------------------------------- /modules/count/test.js: -------------------------------------------------------------------------------- 1 | var vows = require('vows'), 2 | assert = require('assert'); 3 | 4 | var worker = require('./count').worker; 5 | 6 | vows.describe('Test count module').addBatch({ 7 | 8 | 'should return 3': { 9 | topic: function () { 10 | 11 | var that = this; 12 | worker({ 13 | config: { 14 | input: JSON.stringify({ 15 | _INPUT: [1, 2, 3] 16 | }) 17 | }, 18 | respondCompleted: function (results) { 19 | that.callback(null, results); 20 | } 21 | }); 22 | 23 | }, 24 | 25 | 'count results': function (err, result) { 26 | assert.equal(result._OUTPUT, 3); 27 | } 28 | } 29 | 30 | }).export(module); -------------------------------------------------------------------------------- /modules/createrss/createrss.js: -------------------------------------------------------------------------------- 1 | 2 | var subkey = require('../../lib/subkey').subkey; 3 | 4 | exports.worker = function (task, config) { 5 | 6 | var input = JSON.parse(task.config.input); 7 | 8 | var items = input._INPUT; 9 | 10 | items.forEach(function (item) { 11 | 12 | if (input.title) { 13 | item.title = subkey(item, input.title); 14 | item["y:title"] = item.title; 15 | } else if (item.title && !item["y:title"]) { 16 | item["y:title"] = item.title; 17 | } 18 | if (input.description) { item.description = subkey(item, input.description); } 19 | if (input.link) { item.link = subkey(item, input.link); } 20 | 21 | if (input.pubdate) { item.pubdate = subkey(item, input.pubdate); } 22 | if (input.author) { item.author = subkey(item, input.author); } 23 | if (input.guid) { item.guid = subkey(item, input.guid); } 24 | 25 | if (input.mediaContentURL) { 26 | if (!item["media:content"]) { 27 | item["media:content"] = {}; 28 | } 29 | item["media:content"].url = subkey(item, input.mediaContentURL); 30 | 31 | if (input.mediaContentType) { item["media:content"].type = subkey(item, input.mediaContentType); } 32 | if (input.mediaContentWidth) { item["media:content"].width = subkey(item, input.mediaContentWidth); } 33 | if (input.mediaContentHeight) { item["media:content"].height = subkey(item, input.mediaContentHeight); } 34 | } 35 | 36 | if (input.mediaThumbURL) { 37 | if (!item["media:thumbnail"]) { 38 | item["media:thumbnail"] = {}; 39 | } 40 | item["media:thumbnail"].url = subkey(item, input.mediaThumbURL); 41 | 42 | if (input.mediaThumbWidth) { item["media:thumbnail"].width = subkey(item, input.mediaThumbWidth); } 43 | if (input.mediaThumbHeight) { item["media:thumbnail"].height = subkey(item, input.mediaThumbHeight); } 44 | } 45 | 46 | }); 47 | 48 | task.respondCompleted({ 49 | _OUTPUT: items 50 | }); 51 | 52 | }; 53 | 54 | -------------------------------------------------------------------------------- /modules/createrss/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "createrss", 3 | "main" : "./createrss.js", 4 | "version": "0.0.1", 5 | "description": "createrss module" 6 | } -------------------------------------------------------------------------------- /modules/datebuilder/datebuilder.js: -------------------------------------------------------------------------------- 1 | var strftime = require('strftime'); 2 | 3 | exports.worker = function (task, config) { 4 | 5 | var input = JSON.parse(task.config.input); 6 | 7 | var utime = Date.parse(input.DATE); 8 | var d = new Date(utime); 9 | 10 | var n = d.getDate(); 11 | var suff = ["th", "st", "nd", "rd", "th"]; // suff for suffix 12 | var ord = n < 21 ? (n < 4 ? suff[n] : suff[0]) : (n % 10 > 4 ? suff[0] : suff[n % 10]); 13 | 14 | 15 | var response = { 16 | "hour": d.getHours(), 17 | "timezone": strftime("%Z", d), 18 | "second": d.getSeconds(), 19 | "month": d.getMonth() + 1, 20 | "month_name": strftime("%B", d), 21 | "minute": d.getMinutes(), 22 | "utime": utime / 1000, 23 | "day": n, 24 | "day_ordinal_suffix": ord, 25 | "day_of_week": d.getDay(), 26 | "day_name": strftime("%A", d), 27 | "year": d.getFullYear() 28 | }; 29 | 30 | task.respondCompleted({ 31 | _OUTPUT: response 32 | }); 33 | 34 | }; 35 | 36 | -------------------------------------------------------------------------------- /modules/datebuilder/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "datebuilder", 3 | "main" : "./datebuilder.js", 4 | "version": "0.0.1", 5 | "description": "datebuilder module", 6 | "dependencies": { 7 | "strftime": "0.4.7" 8 | } 9 | } -------------------------------------------------------------------------------- /modules/dateformat/dateformat.js: -------------------------------------------------------------------------------- 1 | var strftime = require('strftime'); 2 | 3 | exports.worker = function (task, config) { 4 | 5 | var input = JSON.parse(task.config.input); 6 | 7 | var d; 8 | if (typeof input._INPUT === 'object') { 9 | var pipesDate = input._INPUT; 10 | d = new Date(pipesDate.year, pipesDate.month - 1, pipesDate.day, pipesDate.hour, pipesDate.minute, pipesDate.second); 11 | } else { 12 | var utime = Date.parse(input._INPUT); 13 | d = new Date(utime); 14 | } 15 | 16 | var str = strftime(input.format, d); 17 | 18 | task.respondCompleted({ 19 | _OUTPUT: str 20 | }); 21 | 22 | }; 23 | 24 | -------------------------------------------------------------------------------- /modules/dateformat/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "dateformat", 3 | "main" : "./dateformat.js", 4 | "version": "0.0.1", 5 | "description": "dateformat module", 6 | "dependencies": { 7 | "strftime": "0.4.7" 8 | } 9 | } -------------------------------------------------------------------------------- /modules/dateinput/dateinput.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | task.respondCompleted({ 7 | _OUTPUT: Date.parse(input.default || input.debug)/1000 8 | }); 9 | 10 | }; 11 | 12 | -------------------------------------------------------------------------------- /modules/dateinput/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "dateinput", 3 | "main" : "./dateinput.js", 4 | "version": "0.0.1", 5 | "description": "dateinput module" 6 | } -------------------------------------------------------------------------------- /modules/feedautodiscovery/feedautodiscovery.js: -------------------------------------------------------------------------------- 1 | 2 | var request = require('request'), 3 | htmlparser = require("htmlparser"), 4 | async = require('async'), 5 | xml2js = require('xml2js-expat'); 6 | 7 | var feed_auto_discovery = function (url, cb) { 8 | request(url, function (error, response, body) { 9 | var handler = new htmlparser.DefaultHandler(function (err, dom) { 10 | if (err) { 11 | cb(err, []); 12 | } else { 13 | var links = htmlparser.DomUtils.getElements({tag_name: 'link', rel: "alternate", type: 'application/rss+xml' }, dom); 14 | 15 | cb(null, links); 16 | } 17 | }, { verbose: false, ignoreWhitespace: true }); 18 | var parser = new htmlparser.Parser(handler); 19 | parser.parseComplete(body); 20 | }); 21 | }; 22 | 23 | exports.worker = function (task, config) { 24 | 25 | var input = JSON.parse(task.config.input); 26 | 27 | // fetch must be able to get multiple URLs 28 | var urls = []; 29 | if (Array.isArray(input.URL)) { 30 | input.URL.forEach(function (u) { 31 | urls.push(u); 32 | }); 33 | } else { 34 | urls.push(input.URL); 35 | } 36 | 37 | var feeds = []; 38 | async.forEachSeries(urls, function (url, cb) { 39 | 40 | feed_auto_discovery(url, function (err, results) { 41 | feeds = feeds.concat(results); 42 | cb(err, results); 43 | }); 44 | 45 | }, function () { 46 | 47 | var results = feeds.map(function (f) { 48 | var o = f.attribs; 49 | o["y:title"] = o.title; 50 | return o; 51 | }); 52 | 53 | task.respondCompleted({ 54 | _OUTPUT: results 55 | }); 56 | 57 | }); 58 | 59 | }; 60 | 61 | -------------------------------------------------------------------------------- /modules/feedautodiscovery/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "feedautodiscovery", 3 | "main" : "./feedautodiscovery.js", 4 | "version": "0.0.1", 5 | "description": "feedautodiscovery module", 6 | "dependencies": { 7 | "xml2js-expat": "0.2.2", 8 | "request": "2.11.4", 9 | "htmlparser": "1.7.6" 10 | } 11 | } -------------------------------------------------------------------------------- /modules/fetch/fetch.js: -------------------------------------------------------------------------------- 1 | 2 | var request = require('request'), 3 | xml2js = require('xml2js-expat'), 4 | async = require('async'); 5 | 6 | var fetch_feed = function (url, cb) { 7 | 8 | request(url, function (error, response, body) { 9 | 10 | console.log("Got response. Code = " + response.statusCode); 11 | 12 | if (!error && response.statusCode === 200) { 13 | var r = body; 14 | var contentType = response.headers["content-type"].split(';')[0]; 15 | if (contentType) { 16 | 17 | console.log("contentType = " + contentType); 18 | 19 | // Parsing XML 20 | var feed_content_types = ["text/xml", "application/rss+xml", "application/xml"]; 21 | if (feed_content_types.indexOf(contentType) !== -1) { 22 | console.log("parsing..."); 23 | var parser = new xml2js.Parser(); 24 | parser.addListener('end', function (r) { 25 | //console.log(r); 26 | var result = r.channel ? r.channel.item : r.entry; 27 | cb(null, result); 28 | }); 29 | parser.addListener('error', function (r) { 30 | console.log("Error parsing feed..."); 31 | }); 32 | parser.parseString(body); 33 | return; 34 | } 35 | } 36 | 37 | } else { 38 | // TODO: we should indicate errors in the response 39 | cb(null, []); 40 | } 41 | }); 42 | }; 43 | 44 | var fetch_feeds = function (urls, cb) { 45 | 46 | var items = []; 47 | async.forEachSeries(urls, function (url, cb) { 48 | 49 | fetch_feed(url, function (err, result) { 50 | items = items.concat(result); 51 | cb(err, result); 52 | }); 53 | 54 | }, function () { 55 | cb(null, items); 56 | }); 57 | 58 | }; 59 | 60 | exports.fetch_feeds = fetch_feeds; 61 | 62 | exports.worker = function (task, config) { 63 | 64 | var input = JSON.parse(task.config.input); 65 | 66 | // fetch must be able to get multiple URLs 67 | var urls = []; 68 | if (Array.isArray(input.URL)) { 69 | input.URL.forEach(function (u) { 70 | urls.push(u); 71 | }); 72 | } else { 73 | urls.push(input.URL); 74 | } 75 | 76 | console.log(urls); 77 | 78 | fetch_feeds(urls, function (err, items) { 79 | task.respondCompleted({ 80 | _OUTPUT: items 81 | }); 82 | }); 83 | 84 | }; 85 | 86 | -------------------------------------------------------------------------------- /modules/fetch/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "fetch", 3 | "main" : "./fetch.js", 4 | "version": "0.0.1", 5 | "description": "Fetch module", 6 | "dependencies": { 7 | "xml2js-expat": "0.2.2", 8 | "request": "2.11.4" 9 | } 10 | } -------------------------------------------------------------------------------- /modules/fetchsitefeed/fetchsitefeed.js: -------------------------------------------------------------------------------- 1 | 2 | var request = require('request'), 3 | htmlparser = require("htmlparser"), 4 | async = require('async'), 5 | xml2js = require('xml2js-expat'); 6 | 7 | var feed_auto_discovery = function (url, cb) { 8 | request(url, function (error, response, body) { 9 | var handler = new htmlparser.DefaultHandler(function (err, dom) { 10 | if (err) { 11 | cb(err, []); 12 | } else { 13 | var links = htmlparser.DomUtils.getElements({tag_name: 'link', rel: "alternate", type: 'application/rss+xml' }, dom); 14 | 15 | var urls = links.map(function (l) { 16 | return l.attribs.href; 17 | }); 18 | 19 | cb(null, urls); 20 | } 21 | }, { verbose: false, ignoreWhitespace: true }); 22 | var parser = new htmlparser.Parser(handler); 23 | parser.parseComplete(body); 24 | }); 25 | }; 26 | exports.feed_auto_discovery = feed_auto_discovery; 27 | 28 | exports.worker = function (task, config) { 29 | 30 | var input = JSON.parse(task.config.input); 31 | 32 | // fetch must be able to get multiple URLs 33 | var urls = []; 34 | if (Array.isArray(input.URL)) { 35 | input.URL.forEach(function (u) { 36 | urls.push(u); 37 | }); 38 | } else { 39 | urls.push(input.URL); 40 | } 41 | 42 | var feed_urls = []; 43 | async.forEachSeries(urls, function (url, cb) { 44 | 45 | feed_auto_discovery(url, function (err, results) { 46 | if (results.length > 0) { 47 | feed_urls.push(results[0]); 48 | } 49 | cb(err, results); 50 | }); 51 | 52 | }, function () { 53 | 54 | var fetch_feeds = require('../fetch/fetch').fetch_feeds; 55 | 56 | console.log(feed_urls); 57 | 58 | fetch_feeds(feed_urls, function (err, items) { 59 | task.respondCompleted({ 60 | _OUTPUT: items 61 | }); 62 | }); 63 | 64 | }); 65 | 66 | }; 67 | 68 | -------------------------------------------------------------------------------- /modules/fetchsitefeed/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "fetchsitefeed", 3 | "main" : "./fetchsitefeed.js", 4 | "version": "0.0.1", 5 | "description": "fetchsitefeed module", 6 | "dependencies": { 7 | "xml2js-expat": "0.2.2", 8 | "request": "2.11.4", 9 | "htmlparser": "1.7.6" 10 | } 11 | } -------------------------------------------------------------------------------- /modules/filter/filter.js: -------------------------------------------------------------------------------- 1 | 2 | var subkey = require('../../lib/subkey').subkey; 3 | 4 | var apply_rule = function (item, rule) { 5 | 6 | var a = subkey(item, rule.field), 7 | b = rule.value; 8 | 9 | if (rule.op === 'contains') { 10 | return a.match(b); 11 | } else if (rule.op === 'doesnotcontain') { 12 | return !a.match(b); 13 | } else if (rule.op === 'matches') { 14 | return a.match(new RegExp(b)); 15 | } else if (rule.op === 'greater') { 16 | return a > b; 17 | } else if (rule.op == 'is') { 18 | return a == b; 19 | } else if (rule.op === 'less') { 20 | return a < b; 21 | } else { 22 | // TODO: after 23 | // before 24 | throw "unknown op "+rule.op; 25 | } 26 | 27 | }; 28 | 29 | exports.worker = function (task, config) { 30 | 31 | var input = JSON.parse(task.config.input); 32 | 33 | var results = []; 34 | var combineFcts = { 35 | 'or': function(a,b) { return !!a || !!b; }, 36 | 'and': function(a,b) { return !!a && !!b; } 37 | }; 38 | 39 | input._INPUT.forEach(function (item) { 40 | 41 | var rules_matches = input.RULE.map(function (rule) { 42 | return apply_rule(item, rule); 43 | }); 44 | 45 | var matching = rules_matches.reduce(combineFcts[input.COMBINE]); 46 | 47 | if ( (matching && input.MODE === "permit") || (input.MODE === 'block' && !matching) ) { 48 | results.push(item); 49 | } 50 | }); 51 | 52 | task.respondCompleted({ 53 | _OUTPUT: results 54 | }); 55 | 56 | }; 57 | -------------------------------------------------------------------------------- /modules/filter/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "filter", 3 | "main" : "./filter.js", 4 | "version": "0.0.1", 5 | "description": "filter module" 6 | } -------------------------------------------------------------------------------- /modules/itembuilder/itembuilder.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | var item = {}, k; 7 | 8 | for (k in input.attrs) { 9 | if (input.attrs.hasOwnProperty(k)) { 10 | item[input.attrs[k].key] = input.attrs[k].value; 11 | } 12 | } 13 | 14 | task.respondCompleted({ 15 | _OUTPUT: [item] 16 | }); 17 | }; 18 | 19 | -------------------------------------------------------------------------------- /modules/itembuilder/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "itembuilder", 3 | "main" : "./itembuilder.js", 4 | "version": "0.0.1", 5 | "description": "itembuilder module" 6 | } -------------------------------------------------------------------------------- /modules/itembuilder/test.js: -------------------------------------------------------------------------------- 1 | var vows = require('vows'), 2 | assert = require('assert'); 3 | 4 | var worker = require('./itembuilder').worker; 5 | 6 | vows.describe('Test itembuilder module').addBatch({ 7 | 8 | 'should create an item': { 9 | topic: function () { 10 | 11 | var that = this; 12 | worker({ 13 | config: { 14 | input: JSON.stringify({ 15 | attrs: [ 16 | { 17 | key: 'foo', 18 | value: 'bar' 19 | } 20 | ] 21 | }) 22 | }, 23 | respondCompleted: function (results) { 24 | that.callback(null, results); 25 | } 26 | }); 27 | 28 | }, 29 | 30 | 'itembuilder results': function (err, result) { 31 | assert.equal(result._OUTPUT[0].foo, 'bar'); 32 | } 33 | } 34 | 35 | }).export(module); -------------------------------------------------------------------------------- /modules/locationbuilder/locationbuilder.js: -------------------------------------------------------------------------------- 1 | var querystring = require('querystring'), 2 | request = require('request'); 3 | 4 | var search_location = function (address, cb) { 5 | 6 | 7 | var url = 'http://maps.googleapis.com/maps/api/geocode/json'; 8 | 9 | var q = { 10 | sensor: 'false', 11 | address: address 12 | }; 13 | 14 | url += '?' + querystring.stringify(q); 15 | 16 | request(url, function (error, response, body) { 17 | 18 | var gmapsResults = JSON.parse(body).results[0]; 19 | 20 | var h = {}; 21 | gmapsResults.address_components.forEach(function (c) { 22 | c.types.forEach(function (t) { 23 | if (t === 'country') { 24 | h[t] = c.long_name; 25 | } else { 26 | h[t] = c.short_name; 27 | } 28 | }); 29 | }); 30 | 31 | cb({ 32 | "street_number": h.street_number, 33 | "street": h.route, 34 | "country": h.country, 35 | "postal": h.postal_code, 36 | "state": h.administrative_area_level_1, 37 | "city": h.locality, 38 | "lat": gmapsResults.geometry.location.lat, 39 | "lon": gmapsResults.geometry.location.lng, 40 | "quality": "50" 41 | }); 42 | 43 | }); 44 | 45 | }; 46 | 47 | exports.search_location = search_location; 48 | 49 | exports.worker = function (task, config) { 50 | 51 | var input = JSON.parse(task.config.input); 52 | 53 | search_location(input.LOCATION, function (response) { 54 | 55 | task.respondCompleted({ 56 | _OUTPUT: response 57 | }); 58 | 59 | }); 60 | 61 | }; 62 | 63 | -------------------------------------------------------------------------------- /modules/locationbuilder/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "locationbuilder", 3 | "main" : "./locationbuilder.js", 4 | "version": "0.0.1", 5 | "description": "locationbuilder module", 6 | "dependencies": { 7 | "request": "2.11.4" 8 | } 9 | } -------------------------------------------------------------------------------- /modules/locationinput/locationinput.js: -------------------------------------------------------------------------------- 1 | var querystring = require('querystring'), 2 | request = require('request'); 3 | 4 | exports.worker = function (task, config) { 5 | 6 | var input = JSON.parse(task.config.input); 7 | 8 | var search_location = require('../locationbuilder/locationbuilder').search_location; 9 | 10 | search_location(input.default || input.debug, function (response) { 11 | 12 | task.respondCompleted({ 13 | _OUTPUT: response 14 | }); 15 | 16 | }); 17 | 18 | }; 19 | 20 | -------------------------------------------------------------------------------- /modules/locationinput/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "locationinput", 3 | "main" : "./locationinput.js", 4 | "version": "0.0.1", 5 | "description": "locationinput module", 6 | "dependencies": { 7 | "request": "2.11.4" 8 | } 9 | } -------------------------------------------------------------------------------- /modules/locationinput/test.js: -------------------------------------------------------------------------------- 1 | var vows = require('vows'), 2 | assert = require('assert'); 3 | 4 | var worker = require('./locationinput').worker; 5 | 6 | vows.describe('Test locationinput module').addBatch({ 7 | 8 | 'should get a location': { 9 | topic: function () { 10 | 11 | var that = this; 12 | worker({ 13 | config: { 14 | input: JSON.stringify({ 15 | "name": "locationinput1", 16 | "prompt": "ZipCode:", 17 | "position": "2", 18 | "default": "44114", 19 | "debug": "44114" 20 | }) 21 | }, 22 | respondCompleted: function (results) { 23 | that.callback(null, results); 24 | } 25 | }); 26 | 27 | }, 28 | 29 | 'locationinput results': function (err, result) { 30 | 31 | assert.equal(result._OUTPUT.country, 'United States'); 32 | // TODO 33 | /*country United States 34 | lat 41.508315 35 | postal 44114 36 | state OH 37 | city Cleveland 38 | lon -81.681363 39 | quality 60 40 | */ 41 | } 42 | } 43 | 44 | }).export(module); -------------------------------------------------------------------------------- /modules/loop/loop.js: -------------------------------------------------------------------------------- 1 | var async = require('async'), 2 | path = require('path'), 3 | subkey = require('../../lib/subkey').subkey; 4 | 5 | var walk_subkey_values = function (conf, item) { 6 | var i, k; 7 | if (Array.isArray(conf)) { 8 | for (i = 0; i < conf.length; i += 1) { 9 | if (conf[i].hasOwnProperty('subkey')) { 10 | conf[i] = subkey(item, conf[i].subkey); 11 | } else { 12 | walk_subkey_values(conf[i], item); 13 | } 14 | } 15 | } else if (typeof conf === 'object') { 16 | for (k in conf) { 17 | if (conf.hasOwnProperty(k)) { 18 | if (conf[k].hasOwnProperty('subkey')) { 19 | conf[k] = subkey(item, conf[k].subkey); 20 | } else { 21 | walk_subkey_values(conf[k], item); 22 | } 23 | } 24 | } 25 | } 26 | }; 27 | 28 | exports.worker = function (task, config) { 29 | 30 | var input = JSON.parse(task.config.input); 31 | var submodule = input.embed; 32 | var activityType = submodule.type; 33 | var worker = require(path.join(process.cwd(), 'modules', activityType)).worker; 34 | 35 | /*"emit_part": { 36 | "type": "text", 37 | "value": "all" 38 | }, 39 | "mode": { 40 | "type": "text", 41 | "value": "assign" 42 | }, 43 | "assign_part": { 44 | "type": "text", 45 | "value": "all" 46 | }, 47 | "assign_to": { 48 | "value": "title", 49 | "type": "text" 50 | }, 51 | */ 52 | 53 | var submoduleJsonConf = JSON.stringify(submodule.conf); 54 | var loop_results = []; 55 | 56 | async.forEachSeries(input._INPUT, function (item, cb) { 57 | 58 | var submoduleInput = JSON.parse(submoduleJsonConf); 59 | 60 | walk_subkey_values(submoduleInput, item); 61 | 62 | // Handling "with" attribute 63 | if (input.with) { 64 | submoduleInput._INPUT = item[input.with]; 65 | } else { 66 | submoduleInput._INPUT = item; 67 | } 68 | 69 | worker({ 70 | config: { 71 | input: JSON.stringify(submoduleInput) 72 | }, 73 | 74 | respondCompleted: function (results) { 75 | 76 | if (Array.isArray(results._OUTPUT)) { 77 | loop_results = loop_results.concat(results._OUTPUT); 78 | } else { 79 | loop_results.push(results._OUTPUT); 80 | } 81 | 82 | item[input.assign_to] = results._OUTPUT; 83 | cb(null, results); 84 | } 85 | }); 86 | 87 | 88 | }, function () { 89 | task.respondCompleted({ 90 | _OUTPUT: (input.mode === 'EMIT') ? loop_results : input._INPUT 91 | }); 92 | }); 93 | 94 | }; 95 | 96 | -------------------------------------------------------------------------------- /modules/loop/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "loop", 3 | "main" : "./loop.js", 4 | "version": "0.0.1", 5 | "description": "loop module", 6 | "dependencies": { 7 | "async": "0.1.22" 8 | } 9 | } -------------------------------------------------------------------------------- /modules/numberinput/numberinput.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | task.respondCompleted({ 7 | _OUTPUT: parseFloat(input.default || input.debug) 8 | }); 9 | 10 | }; 11 | 12 | -------------------------------------------------------------------------------- /modules/numberinput/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "numberinput", 3 | "main" : "./numberinput.js", 4 | "version": "0.0.1", 5 | "description": "numberinput module" 6 | } -------------------------------------------------------------------------------- /modules/output/output.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | task.respondCompleted({ 7 | _OUTPUT: input._INPUT 8 | }); 9 | 10 | }; 11 | 12 | -------------------------------------------------------------------------------- /modules/output/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "output", 3 | "main" : "./output.js", 4 | "version": "0.0.1", 5 | "description": "output module" 6 | } -------------------------------------------------------------------------------- /modules/output/test.js: -------------------------------------------------------------------------------- 1 | var vows = require('vows'), 2 | assert = require('assert'); 3 | 4 | var worker = require('./output').worker; 5 | 6 | vows.describe('Test output module').addBatch({ 7 | 8 | 'should return the same value': { 9 | topic: function () { 10 | 11 | var that = this; 12 | worker({ 13 | config: { 14 | input: JSON.stringify({ 15 | _INPUT: "foo-bar" 16 | }) 17 | }, 18 | respondCompleted: function (results) { 19 | that.callback(null, results); 20 | } 21 | }); 22 | 23 | }, 24 | 25 | 'output results': function (err, result) { 26 | assert.equal(result._OUTPUT, 'foo-bar'); 27 | } 28 | } 29 | 30 | }).export(module); -------------------------------------------------------------------------------- /modules/privateinput/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "privateinput", 3 | "main" : "./privateinput.js", 4 | "version": "0.0.1", 5 | "description": "privateinput module" 6 | } -------------------------------------------------------------------------------- /modules/privateinput/privateinput.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | task.respondCompleted({ 7 | _OUTPUT: input.default || input.debug 8 | }); 9 | 10 | }; 11 | 12 | -------------------------------------------------------------------------------- /modules/privatestring/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "privatestring", 3 | "main" : "./privatestring.js", 4 | "version": "0.0.1", 5 | "description": "privatestring module" 6 | } -------------------------------------------------------------------------------- /modules/privatestring/privatestring.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | task.respondCompleted({ 7 | _OUTPUT: input.private_value 8 | }); 9 | 10 | }; 11 | -------------------------------------------------------------------------------- /modules/regex/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "regex", 3 | "main" : "./regex.js", 4 | "version": "0.0.1", 5 | "description": "regex module" 6 | } -------------------------------------------------------------------------------- /modules/regex/regex.js: -------------------------------------------------------------------------------- 1 | 2 | var apply_rule = function (item, rule) { 3 | 4 | var destPath = rule.field.split('.'); 5 | var src = item, i; 6 | 7 | if (destPath.length > 1) { 8 | for (i = 0; i < destPath.length - 1; i += 1) { 9 | src = src[destPath[i]]; 10 | } 11 | } 12 | 13 | var lastKey = destPath[destPath.length - 1]; 14 | 15 | if (rule.replace) { 16 | src[lastKey] = src[lastKey].replace(new RegExp(rule.match), rule.replace); 17 | } 18 | 19 | }; 20 | 21 | exports.worker = function (task, config) { 22 | 23 | var input = JSON.parse(task.config.input); 24 | 25 | if (!Array.isArray(input._INPUT)) { 26 | return; 27 | } 28 | 29 | input._INPUT.forEach(function (item) { 30 | input.RULE.forEach(function (rule) { 31 | apply_rule(item, rule); 32 | }); 33 | }); 34 | 35 | 36 | task.respondCompleted({ 37 | _OUTPUT: input._INPUT 38 | }); 39 | 40 | }; 41 | -------------------------------------------------------------------------------- /modules/rename/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "rename", 3 | "main" : "./rename.js", 4 | "version": "0.0.1", 5 | "description": "rename module" 6 | } -------------------------------------------------------------------------------- /modules/rename/rename.js: -------------------------------------------------------------------------------- 1 | 2 | var apply_rule = function (item, rule) { 3 | 4 | var destPath = rule.newval.split('.'); 5 | var val = item[rule.field]; 6 | 7 | var dest = item, i; 8 | if (destPath.length > 1) { 9 | for (i = 0; i < destPath.length - 1; i += 1) { 10 | if (!dest[destPath[i]]) { 11 | dest[destPath[i]] = {}; 12 | } 13 | dest = dest[destPath[i]]; 14 | } 15 | } 16 | 17 | dest[destPath[destPath.length - 1]] = val; 18 | 19 | if (rule.op === 'copy') { 20 | //dest[destPath[destPath.length - 1]] = val; 21 | } else if (rule.op === 'rename') { 22 | delete dest[rule.field]; 23 | } else { 24 | throw 'unknown op ' + rule.op; 25 | } 26 | 27 | }; 28 | 29 | 30 | exports.worker = function (task, config) { 31 | 32 | var input = JSON.parse(task.config.input); 33 | 34 | /* 35 | "RULE": [ 36 | { 37 | "field": "title", 38 | "op": "copy", 39 | "newval": "extra.originaltitle" 40 | }, 41 | { 42 | "field": "title", 43 | "op": "copy", 44 | "newval": "extra.creator" 45 | } 46 | ] 47 | */ 48 | 49 | input._INPUT.forEach(function (item) { 50 | input.RULE.forEach(function (rule) { 51 | apply_rule(item, rule); 52 | }); 53 | }); 54 | 55 | 56 | task.respondCompleted({ 57 | _OUTPUT: input._INPUT 58 | }); 59 | 60 | }; 61 | -------------------------------------------------------------------------------- /modules/reverse/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "reverse", 3 | "main" : "./reverse.js", 4 | "version": "0.0.1", 5 | "description": "reverse module" 6 | } -------------------------------------------------------------------------------- /modules/reverse/reverse.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | task.respondCompleted({ 7 | _OUTPUT: input._INPUT.reverse() 8 | }); 9 | 10 | }; 11 | -------------------------------------------------------------------------------- /modules/reverse/test.js: -------------------------------------------------------------------------------- 1 | var vows = require('vows'), 2 | assert = require('assert'); 3 | 4 | var worker = require('./reverse').worker; 5 | 6 | vows.describe('Test reverse module').addBatch({ 7 | 8 | 'should reverse the array': { 9 | topic: function () { 10 | 11 | var that = this; 12 | worker({ 13 | config: { 14 | input: JSON.stringify({ 15 | _INPUT: [1, 2, 3] 16 | }) 17 | }, 18 | respondCompleted: function (results) { 19 | that.callback(null, results); 20 | } 21 | }); 22 | 23 | }, 24 | 25 | 'reverse results': function (err, result) { 26 | assert.equal(result._OUTPUT[0], 3); 27 | assert.equal(result._OUTPUT[1], 2); 28 | assert.equal(result._OUTPUT[2], 1); 29 | } 30 | } 31 | 32 | }).export(module); -------------------------------------------------------------------------------- /modules/rssitembuilder/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "rssitembuilder", 3 | "main" : "./rssitembuilder.js", 4 | "version": "0.0.1", 5 | "description": "rssitembuilder module" 6 | } -------------------------------------------------------------------------------- /modules/rssitembuilder/rssitembuilder.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | var item = {}; 7 | 8 | if (input.title) { 9 | item.title = input.title; 10 | item["y:title"] = input.title; 11 | } 12 | if (input.description) { 13 | item.description = input.description; 14 | } 15 | if (input.link) { 16 | item.link = input.link; 17 | } 18 | if (input.pubdate) { 19 | item.pubDate = input.pubdate; 20 | } 21 | // TODO: y:published 22 | if (input.guid) { 23 | item.guid = input.guid; 24 | item["y:id"] = {value: input.guid}; 25 | } 26 | if (input.author) { 27 | item.author = input.author; 28 | } 29 | if (input.mediaContentURL) { 30 | item["media:content"] = { 31 | "url": input.mediaContentURL, 32 | "type": input.mediaContentType, 33 | "width": input.mediaContentWidth, 34 | "height": input.mediaContentHeight 35 | }; 36 | } 37 | if (input.mediaThumbURL) { 38 | item["media:thumbnail"] = { 39 | "url": input.mediaThumbURL, 40 | "width": input.mediaThumbWidth, 41 | "height": input.mediaThumbHeight 42 | }; 43 | } 44 | 45 | task.respondCompleted({ 46 | _OUTPUT: [item] 47 | }); 48 | 49 | }; 50 | -------------------------------------------------------------------------------- /modules/simplemath/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "simplemath", 3 | "main" : "./simplemath.js", 4 | "version": "0.0.1", 5 | "description": "simplemath module" 6 | } -------------------------------------------------------------------------------- /modules/simplemath/simplemath.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | var a = input._INPUT, 7 | b = parseFloat(input.OTHER), 8 | op = input.OP; 9 | 10 | var result; 11 | 12 | if (op === 'modulo') { 13 | result = a % b; 14 | } else if (op === 'subtract') { 15 | result = a - b; 16 | } else if (op === 'divide') { 17 | result = a / b; 18 | } else if (op === 'add') { 19 | result = a + b; 20 | } else if (op === 'multiply') { 21 | result = a * b; 22 | } else if (op === 'power') { 23 | result = Math.pow(a, b); 24 | } else { 25 | throw "OP not known: " + op; 26 | } 27 | 28 | task.respondCompleted({ 29 | _OUTPUT: result 30 | }); 31 | 32 | }; 33 | -------------------------------------------------------------------------------- /modules/sort/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "sort", 3 | "main" : "./sort.js", 4 | "version": "0.0.1", 5 | "description": "sort module" 6 | } -------------------------------------------------------------------------------- /modules/sort/sort.js: -------------------------------------------------------------------------------- 1 | var subkey = require('../../lib/subkey').subkey; 2 | 3 | exports.worker = function (task, config) { 4 | 5 | var input = JSON.parse(task.config.input), 6 | results = input._INPUT; 7 | 8 | //input.KEY.forEach(function (sortKey) { 9 | 10 | // TODO: currently handling only first sort field 11 | 12 | var sortKey = input.KEY[0]; 13 | 14 | var sort_order = (sortKey.dir === 'ASC') ? -1 : 1; 15 | results = results.sort(function (a, b) { 16 | var valA = subkey(a, sortKey.field), 17 | valB = subkey(b, sortKey.field); 18 | return (valA < valB ? 1 : -1) * sort_order; 19 | }); 20 | 21 | //}); 22 | 23 | 24 | task.respondCompleted({ 25 | _OUTPUT: results 26 | }); 27 | 28 | }; 29 | -------------------------------------------------------------------------------- /modules/split/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "split", 3 | "main" : "./split.js", 4 | "version": "0.0.1", 5 | "description": "split module" 6 | } -------------------------------------------------------------------------------- /modules/split/split.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | task.respondCompleted({ 7 | _OUTPUT: input._INPUT, 8 | _OUTPUT2: input._INPUT, 9 | _OUTPUT3: input._INPUT, 10 | _OUTPUT4: input._INPUT, 11 | _OUTPUT5: input._INPUT 12 | }); 13 | 14 | }; 15 | 16 | -------------------------------------------------------------------------------- /modules/strconcat/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "strconcat", 3 | "main" : "./strconcat.js", 4 | "version": "0.0.1", 5 | "description": "strconcat module" 6 | } -------------------------------------------------------------------------------- /modules/strconcat/strconcat.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | task.respondCompleted({ 7 | _OUTPUT: Array.isArray(input.part) ? input.part.join('') : input.part 8 | }); 9 | 10 | }; 11 | 12 | -------------------------------------------------------------------------------- /modules/stringtokenizer/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "stringtokenizer", 3 | "main" : "./stringtokenizer.js", 4 | "version": "0.0.1", 5 | "description": "stringtokenizer module" 6 | } -------------------------------------------------------------------------------- /modules/stringtokenizer/stringtokenizer.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | task.respondCompleted({ 7 | _OUTPUT: input._INPUT.split(input["to-str"]).map(function (i) { return {content: i}; }) 8 | }); 9 | 10 | }; 11 | -------------------------------------------------------------------------------- /modules/strregex/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "strregex", 3 | "main" : "./strregex.js", 4 | "version": "0.0.1", 5 | "description": "strregex module" 6 | } -------------------------------------------------------------------------------- /modules/strregex/strregex.js: -------------------------------------------------------------------------------- 1 | 2 | var apply_rule = function (str, rule) { 3 | str = str.replace(new RegExp(rule.match/*, "g"*/), rule.replace); 4 | return str; 5 | }; 6 | 7 | exports.worker = function (task, config) { 8 | 9 | var input = JSON.parse(task.config.input); 10 | 11 | var str = input._INPUT; 12 | 13 | input.RULE.forEach(function (rule) { 14 | str = apply_rule(str, rule); 15 | }); 16 | 17 | task.respondCompleted({ 18 | _OUTPUT: str 19 | }); 20 | 21 | }; 22 | -------------------------------------------------------------------------------- /modules/strreplace/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "strreplace", 3 | "main" : "./strreplace.js", 4 | "version": "0.0.1", 5 | "description": "strreplace module" 6 | } -------------------------------------------------------------------------------- /modules/strreplace/strreplace.js: -------------------------------------------------------------------------------- 1 | 2 | function replaceLastInstance(find, replace, str) { 3 | var charpos = str.lastIndexOf(find); 4 | if (charpos < 0) { return str; } 5 | var ptone = str.substring(0, charpos); 6 | var pttwo = str.substring(charpos + find.length); 7 | return ptone + replace + pttwo; 8 | } 9 | 10 | var apply_rule = function (str, rule) { 11 | 12 | var modifiers; 13 | if (rule.param === "1") { 14 | str = str.replace(rule.find, rule.replace); 15 | } else if (rule.param === "2") { 16 | str = replaceLastInstance(rule.find, rule.replace, str); 17 | } else if (rule.param === "3") { 18 | str = str.replace(new RegExp(rule.find, "g"), rule.replace); 19 | } 20 | 21 | return str; 22 | }; 23 | 24 | exports.worker = function (task, config) { 25 | 26 | var input = JSON.parse(task.config.input); 27 | 28 | var str = input._INPUT; 29 | 30 | input.RULE.forEach(function (rule) { 31 | str = apply_rule(str, rule); 32 | }); 33 | 34 | task.respondCompleted({ 35 | _OUTPUT: str 36 | }); 37 | 38 | }; 39 | -------------------------------------------------------------------------------- /modules/subelement/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "subelement", 3 | "main" : "./subelement.js", 4 | "version": "0.0.1", 5 | "description": "subelement module" 6 | } -------------------------------------------------------------------------------- /modules/subelement/subelement.js: -------------------------------------------------------------------------------- 1 | 2 | var subkey = require('../../lib/subkey').subkey; 3 | 4 | exports.worker = function (task, config) { 5 | 6 | var input = JSON.parse(task.config.input); 7 | 8 | var items = input._INPUT; 9 | 10 | var results = items.map(function (item) { 11 | return { 12 | content: subkey(item, input.path) 13 | }; 14 | }); 15 | 16 | task.respondCompleted({ 17 | _OUTPUT: results 18 | }); 19 | 20 | }; 21 | -------------------------------------------------------------------------------- /modules/substr/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "substr", 3 | "main" : "./substr.js", 4 | "version": "0.0.1", 5 | "description": "substr module" 6 | } -------------------------------------------------------------------------------- /modules/substr/substr.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | var str = input._INPUT; 7 | 8 | task.respondCompleted({ 9 | _OUTPUT: str.substr(input.from, input.length) 10 | }); 11 | 12 | }; 13 | -------------------------------------------------------------------------------- /modules/tail/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "tail", 3 | "main" : "./tail.js", 4 | "version": "0.0.1", 5 | "description": "tail module" 6 | } -------------------------------------------------------------------------------- /modules/tail/tail.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | var items = input._INPUT; 7 | 8 | task.respondCompleted({ 9 | _OUTPUT: items.slice(items.length - input.count) 10 | }); 11 | 12 | }; 13 | -------------------------------------------------------------------------------- /modules/termextraction/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "termextraction", 3 | "main" : "./termextraction.js", 4 | "version": "0.0.1", 5 | "description": "termextraction module", 6 | "dependencies": { 7 | } 8 | } -------------------------------------------------------------------------------- /modules/termextraction/termextraction.js: -------------------------------------------------------------------------------- 1 | 2 | var request = require('request'), 3 | querystring = require('querystring'); 4 | 5 | exports.worker = function (task, config) { 6 | 7 | var input = JSON.parse(task.config.input); 8 | 9 | console.log(task.config.input); 10 | 11 | var q = { 12 | q: 'select * from search.termextract where context=' + JSON.stringify(input._INPUT), 13 | format: "json", 14 | diagnostics: "false" 15 | }; 16 | 17 | var url = "http://query.yahooapis.com/v1/public/yql?" + querystring.stringify(q); 18 | 19 | request(url, function (error, response, body) { 20 | 21 | console.log("Got response. Code = " + response.statusCode); 22 | 23 | if (!error && response.statusCode === 200) { 24 | 25 | var result = JSON.parse(body).query.results.Result; 26 | var results = []; 27 | if (Array.isArray(result)) { 28 | results = result.map(function (i) { return {content: i}; }); 29 | } 30 | 31 | task.respondCompleted({ 32 | _OUTPUT: results 33 | }); 34 | 35 | } 36 | }); 37 | 38 | 39 | }; 40 | 41 | -------------------------------------------------------------------------------- /modules/textinput/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "textinput", 3 | "main" : "./textinput.js", 4 | "version": "0.0.1", 5 | "description": "textinput module" 6 | } -------------------------------------------------------------------------------- /modules/textinput/test.js: -------------------------------------------------------------------------------- 1 | var vows = require('vows'), 2 | assert = require('assert'); 3 | 4 | var worker = require('./textinput').worker; 5 | 6 | vows.describe('Test textinput module').addBatch({ 7 | 8 | 'should textinput the array': { 9 | topic: function () { 10 | 11 | var that = this; 12 | worker({ 13 | config: { 14 | input: JSON.stringify({ 15 | "name": "Prefix", 16 | "prompt": "Prefix: ", 17 | "position": "", 18 | "default": "This will be legen...", 19 | "debug": "" 20 | }) 21 | }, 22 | respondCompleted: function (results) { 23 | that.callback(null, results); 24 | } 25 | }); 26 | 27 | }, 28 | 29 | 'textinput results': function (err, result) { 30 | assert.equal(result._OUTPUT, "This will be legen..."); 31 | } 32 | } 33 | 34 | }).export(module); -------------------------------------------------------------------------------- /modules/textinput/textinput.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | task.respondCompleted({ 7 | _OUTPUT: input.default || input.debug 8 | }); 9 | 10 | }; 11 | 12 | -------------------------------------------------------------------------------- /modules/truncate/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "truncate", 3 | "main" : "./truncate.js", 4 | "version": "0.0.1", 5 | "description": "truncate module" 6 | } -------------------------------------------------------------------------------- /modules/truncate/truncate.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | exports.worker = function (task, config) { 4 | 5 | var input = JSON.parse(task.config.input); 6 | 7 | task.respondCompleted({ 8 | _OUTPUT: input._INPUT.slice(0, input.count) 9 | }); 10 | 11 | }; 12 | 13 | -------------------------------------------------------------------------------- /modules/union/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "union", 3 | "main" : "./union.js", 4 | "version": "0.0.1", 5 | "description": "union module" 6 | } -------------------------------------------------------------------------------- /modules/union/union.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | var items = input._INPUT || [], k; 7 | 8 | for (k in input) { 9 | if (input.hasOwnProperty(k)) { 10 | if (k.indexOf('_OTHER') === 0) { 11 | items = items.concat(input[k]); 12 | } 13 | } 14 | } 15 | 16 | task.respondCompleted({ 17 | _OUTPUT: items 18 | }); 19 | 20 | }; 21 | 22 | -------------------------------------------------------------------------------- /modules/uniq/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "uniq", 3 | "main" : "./uniq.js", 4 | "version": "0.0.1", 5 | "description": "uniq module" 6 | } -------------------------------------------------------------------------------- /modules/uniq/test.js: -------------------------------------------------------------------------------- 1 | var vows = require('vows'), 2 | assert = require('assert'); 3 | 4 | var worker = require('./uniq').worker; 5 | 6 | vows.describe('Test uniq module').addBatch({ 7 | 8 | 'should uniq the array': { 9 | topic: function () { 10 | 11 | var that = this; 12 | worker({ 13 | config: { 14 | input: JSON.stringify({ 15 | _INPUT: [ 16 | { 17 | guid: '12345', 18 | title: 'First post' 19 | }, 20 | { 21 | guid: '123456', 22 | title: 'Second post' 23 | }, 24 | { 25 | guid: '12345', 26 | title: 'First post bis' 27 | } 28 | ], 29 | field: 'guid' 30 | }) 31 | }, 32 | respondCompleted: function (results) { 33 | that.callback(null, results); 34 | } 35 | }); 36 | 37 | }, 38 | 39 | 'uniq results': function (err, result) { 40 | assert.equal(result._OUTPUT.length, 2); 41 | 42 | 43 | assert.equal(result._OUTPUT[0].guid, '12345'); 44 | assert.equal(result._OUTPUT[0].title, 'First post'); 45 | assert.equal(result._OUTPUT[0]['y:repeatcount'], 2); 46 | 47 | assert.equal(result._OUTPUT[1].guid, '123456'); 48 | assert.equal(result._OUTPUT[1].title, 'Second post'); 49 | assert.equal(result._OUTPUT[1]['y:repeatcount'], 1); 50 | } 51 | } 52 | 53 | }).export(module); -------------------------------------------------------------------------------- /modules/uniq/uniq.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | var uniq_field = input.field; 7 | 8 | var results = []; 9 | var itemsByValue = {}, i; 10 | 11 | for (i = 0; i < input._INPUT.length; i += 1) { 12 | var val = input._INPUT[i][uniq_field]; 13 | 14 | if (!itemsByValue[val]) { 15 | input._INPUT[i]["y:repeatcount"] = 1; 16 | results.push(input._INPUT[i]); 17 | itemsByValue[val] = input._INPUT[i]; 18 | } else { 19 | itemsByValue[val]["y:repeatcount"] += 1; 20 | } 21 | } 22 | 23 | task.respondCompleted({ 24 | _OUTPUT: results 25 | }); 26 | 27 | }; 28 | 29 | -------------------------------------------------------------------------------- /modules/urlbuilder/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "urlbuilder", 3 | "main" : "./urlbuilder.js", 4 | "version": "0.0.1", 5 | "description": "urlbuilder module" 6 | } -------------------------------------------------------------------------------- /modules/urlbuilder/urlbuilder.js: -------------------------------------------------------------------------------- 1 | var querystring = require('querystring'); 2 | 3 | exports.worker = function (task, config) { 4 | 5 | var input = JSON.parse(task.config.input), 6 | base = input.BASE, 7 | url_path = Array.isArray(input.PATH) ? input.PATH.filter(function (i) { return i !== ''; }).join('/') : input.PATH; 8 | 9 | var url = base + ((base.length > 0 && base[base.length - 1] !== '/' && url_path.length > 0 && url_path[0] !== '/') ? '/' : '') + url_path, 10 | q = {}; 11 | 12 | input.PARAM.forEach(function (p) { 13 | q[p.key] = p.value; 14 | }); 15 | 16 | url += '?' + querystring.stringify(q); 17 | 18 | console.log(url); 19 | 20 | task.respondCompleted({ 21 | _OUTPUT: url 22 | }); 23 | 24 | }; 25 | 26 | -------------------------------------------------------------------------------- /modules/urlinput/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "urlinput", 3 | "main" : "./urlinput.js", 4 | "version": "0.0.1", 5 | "description": "urlinput module" 6 | } -------------------------------------------------------------------------------- /modules/urlinput/test.js: -------------------------------------------------------------------------------- 1 | var vows = require('vows'), 2 | assert = require('assert'); 3 | 4 | var worker = require('./urlinput').worker; 5 | 6 | vows.describe('Test urlinput module').addBatch({ 7 | 8 | 'should urlinput the array': { 9 | topic: function () { 10 | 11 | var that = this; 12 | worker({ 13 | config: { 14 | input: JSON.stringify({ 15 | "name": "FeedURL", 16 | "prompt": "FeedURL", 17 | "position": "", 18 | "default": "http://neyric.com/feed", 19 | "debug": "" 20 | }) 21 | }, 22 | respondCompleted: function (results) { 23 | that.callback(null, results); 24 | } 25 | }); 26 | 27 | }, 28 | 29 | 'urlinput results': function (err, result) { 30 | assert.equal(result._OUTPUT, "http://neyric.com/feed"); 31 | } 32 | } 33 | 34 | }).export(module); -------------------------------------------------------------------------------- /modules/urlinput/urlinput.js: -------------------------------------------------------------------------------- 1 | 2 | exports.worker = function (task, config) { 3 | 4 | var input = JSON.parse(task.config.input); 5 | 6 | task.respondCompleted({ 7 | _OUTPUT: input.default || input.debug 8 | }); 9 | 10 | }; 11 | 12 | -------------------------------------------------------------------------------- /modules/webservice/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "webservice", 3 | "main" : "./webservice.js", 4 | "version": "0.0.1", 5 | "description": "webservice module", 6 | "dependencies": { 7 | "request": "2.11.4" 8 | } 9 | } -------------------------------------------------------------------------------- /modules/webservice/webservice.js: -------------------------------------------------------------------------------- 1 | 2 | var request = require('request'); 3 | var subkey = require('../../lib/subkey').subkey; 4 | 5 | exports.worker = function (task, config) { 6 | 7 | var input = JSON.parse(task.config.input); 8 | 9 | request({ 10 | method: 'POST', 11 | url: input.url, 12 | form: {data: JSON.stringify(input._INPUT)} 13 | }, function (error, response, body) { 14 | 15 | var r = JSON.parse(body); 16 | var items = subkey(r, input.path); 17 | 18 | task.respondCompleted({ 19 | _OUTPUT: items 20 | }); 21 | 22 | }); 23 | 24 | }; 25 | 26 | -------------------------------------------------------------------------------- /modules/xpathfetchpage/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "xpathfetchpage", 3 | "main" : "./xpathfetchpage.js", 4 | "version": "0.0.1", 5 | "description": "xpathfetchpage module", 6 | "dependencies": { 7 | "jsdom": "0.2.19" 8 | } 9 | } -------------------------------------------------------------------------------- /modules/xpathfetchpage/xpathfetchpage.js: -------------------------------------------------------------------------------- 1 | var jsdom = require("jsdom"); 2 | 3 | var toStruct = function (node, relativepath, fullpath) { 4 | var h = {}, i; 5 | for (i = 0; i < node.attributes.length; i += 1) { 6 | var attrName = node.attributes[i].name, 7 | attrValue = node.attributes[i].value; 8 | // if href or src 9 | if (attrName === 'src' || attrName === 'href') { 10 | // fix relative or full paths 11 | if (attrValue[0] === '/') { 12 | attrValue = fullpath + attrValue; 13 | } else { 14 | attrValue = relativepath + attrValue; 15 | } 16 | } 17 | h[attrName] = attrValue; 18 | } 19 | 20 | for (i = 0; i < node.childNodes.length; i += 1) { 21 | if (node.childNodes[i].attributes) { 22 | h[node.childNodes[i].tagName.toLowerCase()] = toStruct(node.childNodes[i], relativepath, fullpath); 23 | } 24 | } 25 | 26 | return h; 27 | }; 28 | 29 | 30 | exports.worker = function (task, config) { 31 | 32 | var input = JSON.parse(task.config.input); 33 | 34 | jsdom.env( 35 | input.URL, 36 | [], 37 | function (errors, window) { 38 | 39 | var result = window.document.evaluate(input.xpath, window.document, null, 7, null); 40 | 41 | var l = result.snapshotLength, i; 42 | 43 | var relativepath = String(window.document.location), 44 | fullpath = window.document.location.protocol + '//' + window.document.location.host; 45 | 46 | var results = []; 47 | for (i = 0; i < l; i += 1) { 48 | var r = result.snapshotItem(i); 49 | results.push(toStruct(r, relativepath, fullpath)); 50 | } 51 | 52 | task.respondCompleted({ 53 | _OUTPUT: results 54 | }); 55 | 56 | } 57 | ); 58 | 59 | }; 60 | -------------------------------------------------------------------------------- /modules/yql/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "yql", 3 | "main" : "./yql.js", 4 | "version": "0.0.1", 5 | "description": "YQL module", 6 | "dependencies": { 7 | "request": "2.11.4" 8 | } 9 | } -------------------------------------------------------------------------------- /modules/yql/yql.js: -------------------------------------------------------------------------------- 1 | 2 | var request = require('request'), 3 | querystring = require('querystring'); 4 | 5 | exports.worker = function (task, config) { 6 | 7 | var input = JSON.parse(task.config.input); 8 | 9 | var q = { 10 | q: input.yqlquery, 11 | format: "json", 12 | diagnostics: (input.raw === "results") ? "false" : "true", 13 | callback: "", 14 | env: input.envURL 15 | }; 16 | 17 | var url = "http://query.yahooapis.com/v1/public/yql?" + querystring.stringify(q); 18 | 19 | request(url, function (error, response, body) { 20 | 21 | console.log("Got response. Code = " + response.statusCode); 22 | 23 | if (!error && response.statusCode === 200) { 24 | 25 | var results = JSON.parse(body); 26 | if (input.raw === 'results') { 27 | results = results.query.results; 28 | results = results[Object.keys(results)[0]]; 29 | results = results.map(function (i) { 30 | if (typeof i === 'object' && i.hasOwnProperty('content')) { 31 | return {content: i.content}; 32 | } 33 | return {content: i}; 34 | }); 35 | } 36 | 37 | task.respondCompleted({ 38 | _OUTPUT: results 39 | }); 40 | 41 | } 42 | }); 43 | 44 | 45 | }; 46 | 47 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "pipes2js", 3 | "version": "0.0.1", 4 | "description": "Compile Yahoo! Pipes to Javascript", 5 | "author": "Eric Abouaf ", 6 | "preferGlobal": "true", 7 | "main" : "index.js", 8 | "directories" : { "lib" : "./lib" }, 9 | "repository": { 10 | "type":"git", 11 | "url":"http://github.com/neyric/pipes2js.git" 12 | }, 13 | "bin": { 14 | "pipes2js": "./cli/pipes2js.js" 15 | }, 16 | "dependencies": { 17 | "async": "0.1.22", 18 | "xml2js-expat": "0.2.2", 19 | "request": "2.11.4", 20 | "htmlparser": "1.7.6", 21 | "strftime": "0.4.7", 22 | "jsdom": "0.2.19" 23 | }, 24 | "devDependencies": { 25 | "vows": "0.6.4" 26 | }, 27 | "scripts": { 28 | "test": "vows --spec modules/*/test.js tests/*" 29 | }, 30 | "licenses" : [ 31 | { 32 | "type" : "MIT", 33 | "url" : "https://raw.github.com/neyric/pipes2js/master/LICENSE.txt" 34 | } 35 | ] 36 | } -------------------------------------------------------------------------------- /tests/test-import.js: -------------------------------------------------------------------------------- 1 | var vows = require('vows'), 2 | assert = require('assert'); 3 | 4 | var pipe2js = require('../index'); 5 | 6 | vows.describe('Test conf2input method').addBatch({ 7 | 8 | 'should simplify the config': { 9 | topic: function () { 10 | 11 | return pipe2js.conf2input({ 12 | "attrs": [ 13 | { 14 | "key": { 15 | "value": "guid", 16 | "type": "text" 17 | }, 18 | "value": { 19 | "value": "1932", 20 | "type": "text" 21 | } 22 | }, 23 | { 24 | "key": { 25 | "value": "title", 26 | "type": "text" 27 | }, 28 | "value": { 29 | "value": "First post", 30 | "type": "text" 31 | } 32 | } 33 | ], 34 | "field": { 35 | "value": "name", 36 | "type": "text" 37 | } 38 | }); 39 | 40 | }, 41 | 42 | 'conf2input results': function (input) { 43 | assert.equal(input.attrs[0].key, 'guid'); 44 | assert.equal(input.attrs[0].value, '1932'); 45 | 46 | assert.equal(input.attrs[1].key, 'title'); 47 | assert.equal(input.attrs[1].value, 'First post'); 48 | 49 | assert.equal(input.field, 'name'); 50 | } 51 | } 52 | 53 | }).export(module); -------------------------------------------------------------------------------- /tests/test-subkey.js: -------------------------------------------------------------------------------- 1 | var vows = require('vows'), 2 | assert = require('assert'); 3 | 4 | var subkey = require('../lib/subkey').subkey; 5 | 6 | vows.describe('Test subkey method').addBatch({ 7 | 8 | 'should get the subkey': { 9 | topic: function () { 10 | 11 | return subkey({ 12 | "attrs": { 13 | "key": { 14 | "value": "guid", 15 | "type": "text" 16 | }, 17 | "value": { 18 | "value": "1932", 19 | "type": "text" 20 | } 21 | } 22 | }, "attrs.key.value"); 23 | 24 | }, 25 | 26 | 'subkey results': function (result) { 27 | assert.equal(result, 'guid'); 28 | } 29 | } 30 | 31 | }).export(module); --------------------------------------------------------------------------------