├── index.js ├── .gitignore ├── .travis.yml ├── lib ├── replace-error.js ├── pooler.js ├── task.js └── worker.js ├── test ├── utils │ ├── clean-up.js │ └── create-activity.js ├── scenarios │ ├── error-worker.js │ ├── ready.js │ ├── failed-fn.js │ ├── sync-fn.js │ ├── issue-16.js │ └── test.js ├── examples │ └── abort.js └── scripts │ └── cleanup-states.js ├── package.json └── README.md /index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./lib/worker.js'); 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | .nyc_output 4 | coverage 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '12' 4 | - '14' 5 | install: 6 | - npm install -g codecov nyc 7 | - npm install 8 | 9 | script: 10 | - npm run lint 11 | - npm test 12 | 13 | after_success: 14 | - npm run report-coverage 15 | 16 | deploy: 17 | provider: script 18 | skip_cleanup: true 19 | script: 20 | - npm run semantic-release 21 | -------------------------------------------------------------------------------- /lib/replace-error.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Coming from https://stackoverflow.com/questions/18391212/is-it-not-possible-to-stringify-an-error-using-json-stringify 3 | * @example 4 | * console.log(JSON.stringify(error, replaceErrors)); 5 | */ 6 | 7 | module.exports = function (key, value) { 8 | if (value instanceof Error) { 9 | const error = {}; 10 | Object.getOwnPropertyNames(value).forEach(key => { 11 | error[key] = value[key]; 12 | }); 13 | return error; 14 | } 15 | 16 | return value; 17 | }; 18 | -------------------------------------------------------------------------------- /test/utils/clean-up.js: -------------------------------------------------------------------------------- 1 | const AWS = require('aws-sdk'); 2 | 3 | const stepFunction = new AWS.StepFunctions(); 4 | 5 | module.exports = function ({ 6 | activityArn = null, 7 | stateMachineArn = null 8 | }) { 9 | let p1; 10 | let p2; 11 | if (activityArn) { 12 | p1 = stepFunction.deleteActivity({ 13 | activityArn 14 | }).promise(); 15 | } else { 16 | p1 = Promise.resolve(); 17 | } 18 | 19 | if (stateMachineArn) { 20 | p2 = stepFunction.deleteStateMachine({ 21 | stateMachineArn 22 | }).promise(); 23 | } else { 24 | p2 = Promise.resolve(); 25 | } 26 | 27 | return Promise.all([p1, p2]); 28 | }; 29 | -------------------------------------------------------------------------------- /test/scenarios/error-worker.js: -------------------------------------------------------------------------------- 1 | const test = require('ava'); 2 | 3 | process.on('uncaughtException', err => { 4 | console.log('uncaughtException', err); 5 | }); 6 | 7 | const workerName = 'test worker name'; 8 | const StepFunctionWorker = require('../..'); 9 | 10 | test.serial('Step function Activity Worker worker without fn', t => { 11 | const error = t.throws(() => { 12 | const test = new StepFunctionWorker({ // eslint-disable-line no-unused-vars 13 | activityArn: 'fake-actovuty-arn', 14 | workerName: workerName + '-fn' 15 | }); 16 | }); 17 | t.is(error.message, 'fn parameter should be a function (currently undefined)'); 18 | }); 19 | 20 | -------------------------------------------------------------------------------- /test/examples/abort.js: -------------------------------------------------------------------------------- 1 | const AWS = require('aws-sdk'); 2 | 3 | const stepfunction = new AWS.StepFunctions(); 4 | 5 | const activityArn = 'arn:aws:states:eu-central-1:170670752151:activity:test-step-function-worker-94'; 6 | const stateMachineArn = 'arn:aws:states:eu-central-1:170670752151:stateMachine:test-state-machine-253'; 7 | const paramsStartExecution = { 8 | stateMachineArn /* Required */ 9 | }; 10 | const paramsFirstGetActivity = { 11 | activityArn, /* Required */ 12 | workerName: 'worker1' 13 | }; 14 | const paramsSecondGetActivity = { 15 | activityArn, /* Required */ 16 | workerName: 'worker2' 17 | }; 18 | const onFirstActivityTask = function (err, data) { 19 | console.log('in first activity task', err, data); // An error occurred 20 | stepfunction.getActivityTask(paramsSecondGetActivity, (err, data) => { 21 | console.log('in second activity task', err, data); // An error occurred 22 | }); 23 | stepfunction.startExecution(paramsStartExecution, (err, data) => { 24 | console.log('in start execution', err, data); // An error occurred 25 | }); 26 | }; 27 | 28 | const firstGetActivityTaskRequest = stepfunction.getActivityTask(paramsFirstGetActivity, onFirstActivityTask); 29 | setTimeout(() => { 30 | firstGetActivityTaskRequest.abort(); 31 | }, 2000); 32 | -------------------------------------------------------------------------------- /test/utils/create-activity.js: -------------------------------------------------------------------------------- 1 | const AWS = require('aws-sdk'); 2 | 3 | const stepFunction = new AWS.StepFunctions(); 4 | 5 | const stateMachineDefinition = function (options) { 6 | return { 7 | Comment: 'An Example State machine using Activity.', 8 | StartAt: 'FirstState', 9 | States: { 10 | FirstState: { 11 | Type: 'Task', 12 | Resource: options.activityArn, 13 | TimeoutSeconds: 300, 14 | HeartbeatSeconds: 60, 15 | End: true 16 | } 17 | } 18 | }; 19 | }; 20 | 21 | const stateMachineRoleArn = process.env.ROLE_ARN; 22 | if (!stateMachineRoleArn) { 23 | throw (new Error('$ROLE_ARN should be defined to run this test')); 24 | } 25 | 26 | module.exports = function ({context = {}, activityName, workerName, stateMachineName}) { 27 | return stepFunction 28 | .createActivity({ 29 | name: activityName 30 | }).promise().then(data => { 31 | context.activityArn = data.activityArn; 32 | context.workerName = workerName; 33 | }).then(() => { 34 | const params = { 35 | definition: JSON.stringify(stateMachineDefinition({activityArn: context.activityArn})), /* Required */ 36 | name: stateMachineName, /* Required */ 37 | roleArn: stateMachineRoleArn /* Required */ 38 | }; 39 | return stepFunction.createStateMachine(params).promise(); 40 | }).then(data => { 41 | context.stateMachineArn = data.stateMachineArn; 42 | }).then(() => { 43 | return context; 44 | }); 45 | }; 46 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "step-function-worker", 3 | "version": "0.0.3", 4 | "description": "Easy AWS step function activity worker in node.js", 5 | "main": "index.js", 6 | "scripts": { 7 | "lint": "xo", 8 | "test": "nyc ava test/scenarios/* --timeout 5m", 9 | "semantic-release": "semantic-release", 10 | "report-coverage": "nyc report --reporter=text-lcov > coverage.lcov && codecov" 11 | }, 12 | "engines": { 13 | "node": ">=6.0" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "https://github.com/piercus/step-function-worker.git" 18 | }, 19 | "keywords": [ 20 | "step-function", 21 | "worker", 22 | "aws", 23 | "stepfunction", 24 | "activity" 25 | ], 26 | "author": "Pierre Colle ", 27 | "license": "MIT", 28 | "bugs": { 29 | "url": "https://github.com/piercus/step-function-worker/issues" 30 | }, 31 | "xo": { 32 | "rules": { 33 | "ava/use-test": 1 34 | } 35 | }, 36 | "ava": { 37 | "timeout": 120000 38 | }, 39 | "homepage": "https://github.com/piercus/step-function-worker#readme", 40 | "devDependencies": { 41 | "ava": "^3.15.0", 42 | "bluebird": "^3.5.3", 43 | "nyc": "^15.1.0", 44 | "semantic-release": "^17.2.3", 45 | "winston": "^2.4.1", 46 | "xo": "^0.24.0" 47 | }, 48 | "dependencies": { 49 | "aws-arn-parser": "^1.0.0", 50 | "aws-sdk": "^2.82.0" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /test/scripts/cleanup-states.js: -------------------------------------------------------------------------------- 1 | const AWS = require('aws-sdk'); 2 | const PromiseBlue = require('bluebird'); 3 | const winston = require('winston'); 4 | 5 | const logger = new winston.Logger({ 6 | transports: [new winston.transports.Console({ 7 | level: 'info' 8 | })] 9 | }); 10 | 11 | if (typeof (process.env.AWS_REGION) !== 'string') { 12 | throw (new TypeError('$AWS_REGION must be defined')); 13 | } 14 | 15 | const stepfunctions = new AWS.StepFunctions({ 16 | region: process.env.AWS_REGION 17 | }); 18 | 19 | const reg = new RegExp('stateMachine:test-state-machine'); 20 | 21 | const removeStateMachines = function (reg) { 22 | const params = {}; 23 | return stepfunctions.listStateMachines(params).promise().then(data => { 24 | const stateMachinesArn = []; 25 | data.stateMachines.forEach(stm => { 26 | if (reg.test(stm.stateMachineArn)) { 27 | stateMachinesArn.push(stm.stateMachineArn); 28 | } 29 | }); 30 | return stateMachinesArn; 31 | }).then(stateMachinesArn => { 32 | return PromiseBlue.map(stateMachinesArn, stateMachineArn => { 33 | const params = { 34 | stateMachineArn 35 | }; 36 | return stepfunctions.describeStateMachine(params).promise().then(data => { 37 | const definition = JSON.parse(data.definition); 38 | const activityArn = definition.States.FirstState.Resource; 39 | const params = { 40 | activityArn 41 | }; 42 | return stepfunctions.deleteActivity(params).promise().then(() => { 43 | logger.info('Activity ', params.activityArn, ' was deleted'); 44 | }); 45 | }).then(() => { 46 | return stepfunctions.deleteStateMachine(params).promise().then(() => { 47 | logger.info('StateMachine ', params.stateMachineArn, ' was deleted'); 48 | }); 49 | }); 50 | }, {concurrency: 1}); 51 | }); 52 | }; 53 | 54 | removeStateMachines(reg).catch(error => { 55 | logger.error(error); 56 | }); 57 | -------------------------------------------------------------------------------- /test/scenarios/ready.js: -------------------------------------------------------------------------------- 1 | const test = require('ava'); 2 | const winston = require('winston'); 3 | 4 | const StepFunctionWorker = require('../..'); 5 | const createActivity = require('../utils/create-activity'); 6 | const cleanUp = require('../utils/clean-up'); 7 | 8 | const logger = new winston.Logger({ 9 | transports: [new winston.transports.Console({ 10 | level: 'debug' 11 | })] 12 | }); 13 | 14 | const workerName = 'test worker name'; 15 | const stateMachineName = 'test-state-machine-' + Math.floor(Math.random() * 100000); 16 | const activityName = 'test-step-function-worker-' + Math.floor(Math.random() * 100000); 17 | 18 | process.on('uncaughtException', err => { 19 | console.log('uncaughtException', err); 20 | }); 21 | /* 22 | { 23 | definition: '{"Comment":"An Example State machine using Activity.","StartAt":"FirstState","States":{"FirstState":{"Type":"Task","Resource":"arn:aws:states:eu-central-1:170670752151:activity:test-step-function-worker","TimeoutSeconds":300,"HeartbeatSeconds":60,"Next":"End"}}}', 24 | name: 'test-state-machine', 25 | roleArn: 'arn:aws:iam::170670752151:role/service-role/StatesExecutionRole-eu-central-1' 26 | } 27 | */ 28 | 29 | const context = {}; 30 | 31 | const before = createActivity.bind(null, {context, activityName, stateMachineName, workerName}); 32 | const after = cleanUp.bind(null, {context, activityName, stateMachineName, workerName}); 33 | 34 | const fn = function (event, callback, heartbeat) { 35 | heartbeat(); 36 | setTimeout(() => { 37 | // Assert.equal(event, sentInput); 38 | callback(null, event); 39 | }, 2000); 40 | }; 41 | 42 | test.before(before); 43 | 44 | test.serial('Step function Activity Workerhas a ready event', t => { 45 | const {activityArn} = context; 46 | 47 | return new Promise((resolve, reject) => { 48 | const worker = new StepFunctionWorker({ 49 | activityArn, 50 | workerName: workerName + '-fn', 51 | fn, 52 | logger 53 | }); 54 | let ready = false; 55 | worker.on('ready', () => { 56 | t.pass(); 57 | ready = true; 58 | resolve(); 59 | }); 60 | 61 | setTimeout(() => { 62 | if (!ready) { 63 | t.fail(); 64 | reject(); 65 | } 66 | }, 1000); 67 | }); 68 | }); 69 | 70 | test.after(after); 71 | 72 | -------------------------------------------------------------------------------- /test/scenarios/failed-fn.js: -------------------------------------------------------------------------------- 1 | const test = require('ava'); 2 | const AWS = require('aws-sdk'); 3 | const winston = require('winston'); 4 | 5 | const StepFunctionWorker = require('../..'); 6 | const createActivity = require('../utils/create-activity'); 7 | const cleanUp = require('../utils/clean-up'); 8 | 9 | const stepFunction = new AWS.StepFunctions(); 10 | 11 | const logger = new winston.Logger({ 12 | transports: [new winston.transports.Console({ 13 | level: 'debug' 14 | })] 15 | }); 16 | 17 | const workerName = 'test worker name'; 18 | const stateMachineName = 'test-state-machine-' + Math.floor(Math.random() * 100000); 19 | const activityName = 'test-step-function-worker-' + Math.floor(Math.random() * 100000); 20 | 21 | process.on('uncaughtException', err => { 22 | console.log('uncaughtException', err); 23 | }); 24 | /* 25 | { 26 | definition: '{"Comment":"An Example State machine using Activity.","StartAt":"FirstState","States":{"FirstState":{"Type":"Task","Resource":"arn:aws:states:eu-central-1:170670752151:activity:test-step-function-worker","TimeoutSeconds":300,"HeartbeatSeconds":60,"Next":"End"}}}', 27 | name: 'test-state-machine', 28 | roleArn: 'arn:aws:iam::170670752151:role/service-role/StatesExecutionRole-eu-central-1' 29 | } 30 | */ 31 | 32 | const context = {}; 33 | 34 | const before = createActivity.bind(null, {context, activityName, stateMachineName, workerName}); 35 | const after = cleanUp.bind(null, {context, activityName, stateMachineName, workerName}); 36 | 37 | const sentInput = {foo: 'bar'}; 38 | 39 | const fnError = function (event, callback, heartbeat) { 40 | heartbeat(); 41 | setTimeout(() => { 42 | const err = new Error('custom error'); 43 | // Assert.equal(event, sentInput); 44 | callback(err); 45 | }, 2000); 46 | }; 47 | 48 | test.before(before); 49 | 50 | test.serial('Step function Activity Worker with A failing worker', t => { 51 | const {activityArn, stateMachineArn} = context; 52 | 53 | const worker = new StepFunctionWorker({ 54 | activityArn, 55 | workerName: workerName + '-fn', 56 | fn: fnError, 57 | logger 58 | }); 59 | 60 | return new Promise((resolve, reject) => { 61 | let expectedTaskToken; 62 | const params = { 63 | stateMachineArn, 64 | input: JSON.stringify(sentInput) 65 | }; 66 | worker.once('task', task => { 67 | // Task.taskToken 68 | // task.input 69 | t.deepEqual(task.input, sentInput); 70 | t.is(typeof (task.taskToken), 'string'); 71 | expectedTaskToken = task.taskToken; 72 | }); 73 | 74 | worker.once('failure', out => { 75 | t.is(out.taskToken, expectedTaskToken); 76 | t.is(out.error.message, 'custom error'); 77 | worker.close(() => { 78 | resolve(); 79 | }); 80 | }); 81 | 82 | worker.on('success', reject); 83 | stepFunction.startExecution(params).promise(); 84 | }); 85 | }); 86 | 87 | test.after(after); 88 | 89 | -------------------------------------------------------------------------------- /lib/pooler.js: -------------------------------------------------------------------------------- 1 | const crypto = require('crypto'); 2 | 3 | /** 4 | * @class Pooler 5 | * @param {object} options 6 | * @param {string} options.activityArn 7 | * @param {string} [options.workerName=null] 8 | * @param {string} options.logger 9 | * */ 10 | 11 | function Pooler(options) { 12 | this.id = crypto.randomBytes(3).toString('hex'); 13 | this.logger = options.logger; 14 | this.startTime = new Date(); 15 | this.activityArn = options.activityArn; 16 | this.worker = options.worker; 17 | this.index = options.index; 18 | this.workerName = options.workerName && (options.workerName + '-' + this.index); 19 | this.logger.debug(`new pooler ${this.id}`); 20 | this.getActivityTask(); 21 | } 22 | 23 | Pooler.prototype.stop = function () { 24 | this.logger.debug(`Pooler (${this.id}): Stop`); 25 | 26 | if (!this._stoppingPromise) { 27 | this._stoppingPromise = (this._requestPromise || Promise.resolve()).then(() => { 28 | this._stopped = true; 29 | }); 30 | } 31 | 32 | return this._stoppingPromise; 33 | }; 34 | 35 | /** 36 | * @typedef {object} PoolerReport 37 | * @param {String} workerName 38 | * @param {String} status, can be 'Task under going', 'Waiting for Tasks' or 'Paused' 39 | * @param {TaskReport | null} task 40 | */ 41 | /** 42 | * Get a report on the actual situation of the pooler 43 | * @return {PoolerReport} list of poolers 44 | */ 45 | Pooler.prototype.report = function () { 46 | return { 47 | id: this.id, 48 | startTime: this.startTime, 49 | status: (this._stopped ? 'Stopped' : 'Running') 50 | }; 51 | }; 52 | 53 | Pooler.prototype.restart = function () { 54 | return this.stop().then(() => { 55 | this._stopped = false; 56 | this.getActivityTask(); 57 | return null; 58 | }); 59 | }; 60 | 61 | Pooler.prototype.getActivityTask = function () { 62 | // This.logger.info('getActivityTask'); 63 | 64 | // this.logger.debug(this.workerName + ' getActivityTask ' + this.activityArn); 65 | if (this._stopped) { 66 | return Promise.reject(new Error(`Pooler (${this.id}) is stopped`)); 67 | } 68 | 69 | if (!this._requestPromise) { 70 | this.logger.debug(`Pooler (${this.id}): getActivityTask`); 71 | 72 | this._requestPromise = this.worker.stepfunction.getActivityTask({ 73 | activityArn: this.activityArn, 74 | workerName: this.workerName 75 | }).promise() 76 | .then(data => { 77 | if (data.taskToken && typeof (data.taskToken) === 'string' && data.taskToken.length > 1) { 78 | this.logger.debug(`Pooler (${this.id}): Activity task received (${data.taskToken.slice(0, 10)})`); 79 | const params = Object.assign({}, data, { 80 | input: JSON.parse(data.input), 81 | workerName: this.workerName, 82 | poolerId: this.id 83 | }); 84 | return this.worker.addTask(params); 85 | } 86 | 87 | this.logger.debug(`Pooler (${this.id}): No activity task received`); 88 | return null; 89 | }) 90 | .then(() => { 91 | this._requestPromise = null; 92 | const renewal = this.worker.renewPooler(this); 93 | if (!renewal) { 94 | this.stop(); 95 | this.worker.removePooler(this); 96 | return null; 97 | } 98 | 99 | return this.getActivityTask(); 100 | }) 101 | .catch(error => { 102 | // Console.log(err); 103 | this.logger.error(`Pooler (${this.id}):`, error); 104 | if (error.code === 'RequestAbortedError') { 105 | // In case of abort, close silently 106 | } else { 107 | this.worker.emit('error', error); 108 | } 109 | 110 | // Return Promise.reject(err); 111 | }); 112 | } 113 | 114 | return this._requestPromise; 115 | }; 116 | 117 | module.exports = Pooler; 118 | -------------------------------------------------------------------------------- /test/scenarios/sync-fn.js: -------------------------------------------------------------------------------- 1 | const test = require('ava'); 2 | const AWS = require('aws-sdk'); 3 | const StepFunctionWorker = require('../..'); 4 | const createActivity = require('../utils/create-activity'); 5 | const cleanUp = require('../utils/clean-up'); 6 | 7 | const stepFunction = new AWS.StepFunctions(); 8 | const workerName = 'test worker name'; 9 | const stateMachineName = 'test-state-machine-' + Math.floor(Math.random() * 100000); 10 | const activityName = 'test-step-function-worker-' + Math.floor(Math.random() * 100000); 11 | 12 | process.on('uncaughtException', err => { 13 | console.log('uncaughtException', err); 14 | }); 15 | /* 16 | { 17 | definition: '{"Comment":"An Example State machine using Activity.","StartAt":"FirstState","States":{"FirstState":{"Type":"Task","Resource":"arn:aws:states:eu-central-1:170670752151:activity:test-step-function-worker","TimeoutSeconds":300,"HeartbeatSeconds":60,"Next":"End"}}}', 18 | name: 'test-state-machine', 19 | roleArn: 'arn:aws:iam::170670752151:role/service-role/StatesExecutionRole-eu-central-1' 20 | } 21 | */ 22 | 23 | const context = {}; 24 | 25 | const before = createActivity.bind(null, {context, activityName, stateMachineName, workerName}); 26 | const after = cleanUp.bind(null, context); 27 | 28 | const sentInput = {foo: 'bar'}; 29 | const sentOutput = {foo2: 'bar2'}; 30 | 31 | const fn = function (event, callback) { 32 | callback(null, sentOutput); 33 | }; 34 | 35 | const fnError = function () { 36 | throw (new Error('custom error')); 37 | }; 38 | 39 | test.before(before); 40 | 41 | test.serial('Step function Activity Worker with 2 consecutive synchronous tasks', t => { 42 | const {activityArn, stateMachineArn} = context; 43 | 44 | const worker = new StepFunctionWorker({ 45 | activityArn, 46 | workerName: workerName + '-fn', 47 | fn 48 | }); 49 | 50 | return new Promise((resolve, reject) => { 51 | let expectedTaskToken; 52 | const params = { 53 | stateMachineArn, 54 | input: JSON.stringify(sentInput) 55 | }; 56 | worker.once('task', task => { 57 | // Task.taskToken 58 | // task.input 59 | t.deepEqual(task.input, sentInput); 60 | t.is(typeof (task.taskToken), 'string'); 61 | expectedTaskToken = task.taskToken; 62 | }); 63 | worker.on('error', reject); 64 | worker.once('success', out => { 65 | t.is(out.taskToken, expectedTaskToken); 66 | 67 | let expectedTaskToken2; 68 | worker.once('task', task => { 69 | // Task.taskToken 70 | // task.input 71 | expectedTaskToken2 = task.taskToken; 72 | }); 73 | 74 | worker.once('success', out => { 75 | t.is(out.taskToken, expectedTaskToken2); 76 | worker.close(() => { 77 | resolve(); 78 | }); 79 | }); 80 | 81 | stepFunction.startExecution(params).promise(); 82 | }); 83 | 84 | stepFunction.startExecution(params).promise(); 85 | }); 86 | }); 87 | 88 | test.serial('Step function Activity Worker with synchronous failing task', t => { 89 | const {activityArn, stateMachineArn} = context; 90 | 91 | const worker = new StepFunctionWorker({ 92 | activityArn, 93 | workerName: workerName + '-fn', 94 | fn: fnError 95 | }); 96 | 97 | return new Promise((resolve, reject) => { 98 | let expectedTaskToken; 99 | const params = { 100 | stateMachineArn, 101 | input: JSON.stringify(sentInput) 102 | }; 103 | worker.once('task', task => { 104 | // Task.taskToken 105 | // task.input 106 | t.deepEqual(task.input, sentInput); 107 | t.is(typeof (task.taskToken), 'string'); 108 | expectedTaskToken = task.taskToken; 109 | }); 110 | worker.once('failure', out => { 111 | t.is(out.taskToken, expectedTaskToken); 112 | t.is(out.error.message, 'custom error'); 113 | worker.close(() => { 114 | resolve(); 115 | }); 116 | }); 117 | worker.once('success', reject); 118 | stepFunction.startExecution(params).promise(); 119 | }); 120 | }); 121 | test.after(after); 122 | -------------------------------------------------------------------------------- /lib/task.js: -------------------------------------------------------------------------------- 1 | const {EventEmitter} = require('events'); 2 | const util = require('util'); 3 | const replaceError = require('./replace-error.js'); 4 | 5 | /** 6 | * @class StepFunctionWorker 7 | * @param {object} options 8 | * @param {object} options.worker 9 | * @param {string} options.taskToken 10 | * @param {string} options.logger 11 | * @param {string} options.workerName - this.pooler workerName 12 | * @param {object} options.input 13 | * */ 14 | 15 | function Task(options) { 16 | EventEmitter.call(this); 17 | 18 | this.logger = options.logger; 19 | this.worker = options.worker; 20 | this.stepfunction = this.worker.stepfunction; 21 | this.input = options.input; 22 | this.taskToken = options.taskToken; 23 | this.workerName = options.workerName; 24 | this.startTime = new Date(); 25 | this._finished = false; 26 | this._execute(this.input, this.taskCallback.bind(this), this.heartbeat.bind(this)); 27 | } 28 | 29 | Task.prototype.taskCallback = function (err, res) { 30 | if (err) { 31 | this.logger.debug('task fail'); 32 | this.fail(err); 33 | } else { 34 | this.logger.debug('task succeed'); 35 | this.succeed(res); 36 | } 37 | }; 38 | /** 39 | * @typedef {object} TaskReport 40 | * @param {String} taskToken 41 | * @param {object} input 42 | * @param {Date} startTime 43 | */ 44 | 45 | /** 46 | * Get a report on the actual situation of the task 47 | * @return {TaskReport} 48 | */ 49 | 50 | Task.prototype.report = function () { 51 | return { 52 | taskToken: this.taskToken, 53 | input: this.input, 54 | startTime: this.startTime 55 | }; 56 | }; 57 | 58 | Task.prototype.succeed = function (res) { 59 | this.logger.debug(`Succeed (${this.input.index})`); 60 | this._succeed({ 61 | input: this.input, 62 | output: res, 63 | taskToken: this.taskToken, 64 | workerName: this.workerName 65 | }); 66 | this._finished = true; 67 | this.emit('finish'); 68 | }; 69 | 70 | Task.prototype.fail = function (err) { 71 | this._fail({ 72 | error: err, 73 | input: this.input, 74 | taskToken: this.taskToken, 75 | workerName: this.workerName 76 | }); 77 | this._finished = true; 78 | this.emit('finish'); 79 | }; 80 | 81 | Task.prototype.heartbeat = function () { 82 | this.logger.debug(`Heartbeat (${this.input.index})`); 83 | 84 | this._heartbeat({ 85 | input: this.input, 86 | taskToken: this.taskToken, 87 | workerName: this.workerName 88 | }); 89 | }; 90 | 91 | Task.prototype._execute = function (input, cb, heartbeat) { 92 | setImmediate(() => { 93 | try { 94 | this.worker.fn(input, cb, heartbeat); 95 | } catch (error) { 96 | cb(error); 97 | } 98 | }); 99 | }; 100 | 101 | Task.prototype._succeed = function (res) { 102 | const params = Object.assign({}, res, {output: JSON.stringify(res.output)}); 103 | delete params.workerName; 104 | delete params.input; 105 | this.stepfunction.sendTaskSuccess(params, err => { 106 | if (err) { 107 | this.logger.error('Cannot sendTaskSuccess', err); 108 | this.worker.emit('error', {err, input: res.input}); 109 | } else { 110 | this.worker.emit('success', res); 111 | } 112 | }); 113 | }; 114 | 115 | Task.prototype._fail = function (res) { 116 | let error = JSON.stringify(res.error, replaceError); 117 | 118 | if (error.length > 256) { 119 | // Otherwise aws sdk will tell 120 | // failed to satisfy constraint: Member must have length less than or equal to 256 121 | error = error.slice(0, 253) + '...'; 122 | } 123 | 124 | const params = Object.assign({}, res, {error}); 125 | delete params.workerName; 126 | delete params.input; 127 | // This.logger.debug('sendTaskFailure', res.error); 128 | this.stepfunction.sendTaskFailure(params, err => { 129 | if (err) { 130 | this.worker.emit('error', {err, input: res.input}); 131 | } else { 132 | this.worker.emit('failure', res); 133 | } 134 | }); 135 | }; 136 | 137 | Task.prototype._heartbeat = function (res) { 138 | const params = Object.assign({}, res); 139 | delete params.workerName; 140 | delete params.input; 141 | // This.logger.debug('sendTaskHeartbeat'); 142 | 143 | this.stepfunction.sendTaskHeartbeat(params, err => { 144 | if (err) { 145 | if (err.code === 'TaskTimedOut' && this._finished) { 146 | this.logger.warn( 147 | `Heartbeat response received after task is finished (succeed or failed) 148 | To remove this warning make sure to not send heartbeat() just before calling cb()` 149 | ); 150 | } else { 151 | this.worker.emit('error', {err, input: res.input}); 152 | } 153 | } else { 154 | this.worker.emit('heartbeat', res); 155 | } 156 | }); 157 | }; 158 | 159 | util.inherits(Task, EventEmitter); 160 | 161 | module.exports = Task; 162 | -------------------------------------------------------------------------------- /test/scenarios/issue-16.js: -------------------------------------------------------------------------------- 1 | const test = require('ava'); 2 | const AWS = require('aws-sdk'); 3 | const winston = require('winston'); 4 | const StepFunctionWorker = require('../..'); 5 | const createActivity = require('../utils/create-activity'); 6 | const cleanUp = require('../utils/clean-up'); 7 | 8 | const stepFunction = new AWS.StepFunctions(); 9 | const workerName = 'test worker name'; 10 | const stateMachineName = 'test-state-machine-' + Math.floor(Math.random() * 100000); 11 | const activityName = 'test-step-function-worker-' + Math.floor(Math.random() * 100000); 12 | 13 | process.on('uncaughtException', err => { 14 | console.log('uncaughtException', err); 15 | }); 16 | /* 17 | { 18 | definition: '{"Comment":"An Example State machine using Activity.","StartAt":"FirstState","States":{"FirstState":{"Type":"Task","Resource":"arn:aws:states:eu-central-1:170670752151:activity:test-step-function-worker","TimeoutSeconds":300,"HeartbeatSeconds":60,"Next":"End"}}}', 19 | name: 'test-state-machine', 20 | roleArn: 'arn:aws:iam::170670752151:role/service-role/StatesExecutionRole-eu-central-1' 21 | } 22 | */ 23 | 24 | const context = {}; 25 | 26 | const before = createActivity.bind(null, { 27 | context, 28 | activityName, 29 | stateMachineName, 30 | workerName 31 | }); 32 | const after = cleanUp.bind(null, context); 33 | 34 | const sentInput = function (i) { 35 | return { 36 | foo: 'bar', 37 | index: i 38 | }; 39 | }; 40 | 41 | const sentOutput = {foo2: 'bar2'}; 42 | 43 | const taskDurationBase = 2000; 44 | const fn = function (event, callback, heartbeat) { 45 | heartbeat(); 46 | 47 | const totalDuration = Math.ceil(Math.random() * taskDurationBase); 48 | setTimeout(() => { 49 | // Assert.equal(event, sentInput); 50 | heartbeat(); 51 | }, totalDuration); 52 | setTimeout(() => { 53 | // Assert.equal(event, sentInput); 54 | heartbeat(); 55 | }, 2 * totalDuration); 56 | setTimeout(() => { 57 | // Assert.equal(event, sentInput); 58 | heartbeat(); 59 | }, 3 * totalDuration); 60 | setTimeout(() => { 61 | // Assert.equal(event, sentInput); 62 | heartbeat(); 63 | }, 4 * totalDuration); 64 | setTimeout(() => { 65 | // Assert.equal(event, sentInput); 66 | heartbeat(); 67 | }, 5 * totalDuration); 68 | setTimeout(() => { 69 | // Assert.equal(event, sentInput); 70 | callback(null, sentOutput); 71 | }, 6 * totalDuration); 72 | }; 73 | 74 | test.before(before); 75 | 76 | test.serial('Step function Activity Worker with 200 parallel tasks and heartbeat', t => { 77 | const {activityArn, stateMachineArn} = context; 78 | const startDate = new Date(); 79 | const totalTasks = 10; 80 | const poolConcurrency = 3; 81 | const taskConcurrency = 5; 82 | const worker = new StepFunctionWorker({ 83 | activityArn, 84 | workerName: workerName + '-fn', 85 | fn, 86 | logger: new winston.Logger({ 87 | level: 'debug', 88 | transports: [ 89 | new (winston.transports.Console)({ 90 | timestamp() { 91 | return (new Date()).toISOString().slice(11); 92 | }, 93 | formatter(options) { 94 | // - Return string will be passed to logger. 95 | // - Optionally, use options.colorize(options.level, ) to 96 | // colorize output based on the log level. 97 | return options.timestamp() + ' ' + 98 | winston.config.colorize(options.level, options.level.toUpperCase()) + ' ' + 99 | (options.message ? options.message : '') + 100 | (options.meta && Object.keys(options.meta).length > 0 ? '\n\t' + JSON.stringify(options.meta) : ''); 101 | } 102 | }) 103 | ] 104 | }), 105 | poolConcurrency, 106 | taskConcurrency 107 | }); 108 | 109 | const params = function (i) { 110 | return { 111 | stateMachineArn, 112 | input: JSON.stringify(sentInput(i)) 113 | }; 114 | }; 115 | 116 | let count = 0; 117 | let countFull = 0; 118 | worker.on('task', () => { 119 | count++; 120 | }); 121 | worker.on('full', () => { 122 | countFull++; 123 | const report = worker.report(); 124 | t.is(report.tasks.length, taskConcurrency); 125 | }); 126 | const promises = []; 127 | for (let i = 0; i < totalTasks; i++) { 128 | promises.push(stepFunction.startExecution(params(i)).promise()); 129 | } 130 | 131 | return new Promise((resolve, reject) => { 132 | worker.once('empty', () => { 133 | t.is(count, totalTasks); 134 | t.true(countFull > 0); 135 | // T.is(Math.abs(countFull - (totalTasks-taskConcurrency))/totalTasks) 136 | const endDate = new Date(); 137 | worker.logger.info(`Spent ${(endDate - startDate) / 1000} seconds`); 138 | worker.close(() => { 139 | resolve(); 140 | }); 141 | }); 142 | worker.on('error', reject); 143 | 144 | return Promise.all(promises); 145 | }); 146 | }); 147 | 148 | test.after(after); 149 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://app.travis-ci.com/piercus/step-function-worker.svg?branch=master)](https://app.travis-ci.com/piercus/step-function-worker) 2 | [![codecov](https://codecov.io/gh/piercus/step-function-worker/branch/master/graph/badge.svg)](https://codecov.io/gh/piercus/step-function-worker) 3 | 4 | # step-function-worker 5 | 6 | Create a nodejs aws step-function worker/pooler easily :-) 7 | 8 | ## install 9 | 10 | ``` 11 | npm install step-function-worker 12 | ``` 13 | 14 | ### Example usage 15 | 16 | #### Basic example 17 | 18 | ```javascript 19 | const fn = function(input, cb, heartbeat){ 20 | // do something 21 | doSomething(input) 22 | 23 | // call heartbeat to avoid timeout 24 | heartbeat() 25 | 26 | // call callback in the end 27 | cb(null, {"foo" : "bar"}); // output must be compatible with JSON.stringify 28 | }; 29 | 30 | const worker = new StepFunctionWorker({ 31 | activityArn : '', 32 | workerName : 'workerName', 33 | fn : fn, 34 | taskConcurrency : 22, // default is null = Infinity 35 | poolConcurrency : 2 // default is 1 36 | }); 37 | ``` 38 | 39 | ### Concurrency management 40 | 41 | Since version **3.0**, `concurrency` has been replaced by `poolConcurrency` and `taskConcurrency`. 42 | 43 | see more information in https://github.com/piercus/step-function-worker/issues/16#issuecomment-486971866 44 | 45 | * `poolConcurrency` is the maximum number of parallel getActivity, http request (see [`sdk.getActivity`](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/StepFunctions.html#getActivityTask-property)) (default: `1`) Increase this to have a more responsive worker, decrease this to consume less http connections. 46 | 47 | * `taskConcurrency` (`null` means Infinite) represents the maximum number of parallel tasks done by the worker (default: equals to `poolConcurrency`). 48 | 49 | Anyway, you should always have `poolConcurrency` <= `taskConcurrency`. 50 | 51 | #### Set the Region 52 | 53 | By default, this package is built on top of `aws-sdk` so you should set your AWS Region by changing `AWS_REGION` environment variable. 54 | 55 | If you want to set it in JS code directly you can do it using `awsConfig` (see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html to see all available options) like 56 | 57 | ```javascript 58 | const worker = new StepFunctionWorker({ 59 | activityArn : '', 60 | workerName : 'workerName', 61 | fn : fn, 62 | awsConfig: { 63 | region: '' 64 | } 65 | }); 66 | ``` 67 | 68 | #### Close the worker 69 | 70 | ```javascript 71 | // when finish close the worker with a callback 72 | // this closing process may take up to 60 seconds per concurent worker, to close all connections smoothly without loosing any task 73 | worker.close(function(){ 74 | process.exit(); 75 | }) 76 | ``` 77 | 78 | #### Get info on current worker 79 | 80 | ```javascript 81 | // A worker as multiple poolers and multiple running tasks 82 | // You can have infos about it by doing 83 | const {poolers, tasks} = worker.report(); 84 | 85 | // poolers is an array of { 86 | // startTime: , 87 | // workerName: , 88 | // status: 89 | // } 90 | // 91 | // tasks is an array of { 92 | // taskToken: , 93 | // input: , 94 | // startTime: 95 | // } 96 | // 97 | ``` 98 | 99 | #### Custom logging with winston 100 | 101 | You can customize logging by using a [winston](https://www.npmjs.com/package/winston) logger (or winston-like logger) as input 102 | 103 | ```javascript 104 | const winston = require('winston'); 105 | 106 | const logger = winston.createLogger({ 107 | level: 'debug', 108 | format: winston.format.json(), 109 | defaultMeta: { service: 'user-service' }, 110 | transports: [ 111 | // 112 | // - Write to all logs with level `info` and below to `combined.log` 113 | // - Write all logs error (and below) to `error.log`. 114 | // 115 | new winston.transports.File({ filename: 'error.log', level: 'error' }), 116 | new winston.transports.File({ filename: 'combined.log' }) 117 | ] 118 | }); 119 | 120 | const worker = new StepFunctionWorker({ 121 | activityArn : '', 122 | workerName : 'workerName', 123 | fn : fn, 124 | logger 125 | }); 126 | ``` 127 | 128 | Alternatively, you can just use a winston-like logger 129 | 130 | ```javascript 131 | const logger = console; 132 | 133 | const worker = new StepFunctionWorker({ 134 | activityArn : '', 135 | workerName : 'workerName', 136 | fn : fn, 137 | logger 138 | }); 139 | ``` 140 | 141 | #### Events 142 | 143 | 144 | ```javascript 145 | // when a task starts 146 | worker.on('task', function(task){ 147 | // task.taskToken 148 | // task.input 149 | console.log("task ", task.input) 150 | }); 151 | 152 | // when a task fails 153 | worker.on('failure', function(failure){ 154 | // out.error 155 | // out.taskToken 156 | console.log("Failure :",failure.error) 157 | }); 158 | 159 | // when a heartbeat signal is sent 160 | worker.on('heartbeat', function(beat){ 161 | // out.taskToken 162 | console.log("Heartbeat"); 163 | }); 164 | 165 | // when a task succeed 166 | worker.on('success', function(out){ 167 | // out.output 168 | // out.taskToken 169 | console.log("Success :",out.output) 170 | }); 171 | 172 | // when an error happens 173 | worker.on('error', function(err){ 174 | console.log("error ", err) 175 | }); 176 | 177 | // when the worker has no more task to process 178 | worker.on('empty', function(){ 179 | console.log("error ", err) 180 | }); 181 | 182 | // when the worker reaches taskConcurrency tasks 183 | worker.on('full', function(err){ 184 | console.log("error ", err) 185 | }); 186 | ``` 187 | 188 | ### Documentation 189 | 190 | See JSDoc in the code. 191 | 192 | -------------------------------------------------------------------------------- /test/scenarios/test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava'); 2 | const winston = require('winston'); 3 | const AWS = require('aws-sdk'); 4 | const StepFunctionWorker = require('../..'); 5 | const createActivity = require('../utils/create-activity'); 6 | const cleanUp = require('../utils/clean-up'); 7 | 8 | const stepFunction = new AWS.StepFunctions(); 9 | const workerName = 'test worker name'; 10 | const stateMachineName = 'test-state-machine-' + Math.floor(Math.random() * 100000); 11 | const activityName = 'test-step-function-worker-' + Math.floor(Math.random() * 100000); 12 | 13 | process.on('uncaughtException', err => { 14 | console.log('uncaughtException', err); 15 | }); 16 | const logger = new winston.Logger({ 17 | transports: [new winston.transports.Console({ 18 | level: 'debug' 19 | })] 20 | }); 21 | /* 22 | { 23 | definition: '{"Comment":"An Example State machine using Activity.","StartAt":"FirstState","States":{"FirstState":{"Type":"Task","Resource":"arn:aws:states:eu-central-1:170670752151:activity:test-step-function-worker","TimeoutSeconds":300,"HeartbeatSeconds":60,"Next":"End"}}}', 24 | name: 'test-state-machine', 25 | roleArn: 'arn:aws:iam::170670752151:role/service-role/StatesExecutionRole-eu-central-1' 26 | } 27 | */ 28 | 29 | const context = {}; 30 | 31 | const before = createActivity.bind(null, {context, activityName, stateMachineName, workerName}); 32 | const after = cleanUp.bind(null, context); 33 | 34 | const sentInput = {foo: 'bar'}; 35 | const sentOutput = {foo2: 'bar2'}; 36 | 37 | const fn = function (event, callback, heartbeat) { 38 | heartbeat(); 39 | setTimeout(() => { 40 | // Assert.equal(event, sentInput); 41 | callback(null, sentOutput); 42 | }, 2000); 43 | }; 44 | 45 | const fn2 = function (event, callback, heartbeat) { 46 | heartbeat(); 47 | setTimeout(() => { 48 | // Assert.equal(event, sentInput); 49 | callback(null, Object.assign({}, event, sentOutput)); 50 | }, 2000); 51 | }; 52 | 53 | test.before(before); 54 | 55 | test.serial('Step function Activity Worker with 2 consecutive tasks', t => { 56 | const {activityArn, stateMachineArn} = context; 57 | 58 | const worker = new StepFunctionWorker({ 59 | activityArn, 60 | logger, 61 | workerName: workerName + '-fn', 62 | fn 63 | }); 64 | 65 | return new Promise((resolve, reject) => { 66 | let expectedTaskToken; 67 | const params = { 68 | stateMachineArn, 69 | input: JSON.stringify(sentInput) 70 | }; 71 | worker.once('task', task => { 72 | // Task.taskToken 73 | // task.input 74 | t.deepEqual(task.input, sentInput); 75 | t.is(typeof (task.taskToken), 'string'); 76 | expectedTaskToken = task.taskToken; 77 | }); 78 | worker.on('error', reject); 79 | worker.once('success', out => { 80 | t.is(out.taskToken, expectedTaskToken); 81 | 82 | let expectedTaskToken2; 83 | worker.once('task', task => { 84 | // Task.taskToken 85 | // task.input 86 | expectedTaskToken2 = task.taskToken; 87 | }); 88 | 89 | worker.once('success', out => { 90 | t.is(out.taskToken, expectedTaskToken2); 91 | worker.close(() => { 92 | resolve(); 93 | }); 94 | }); 95 | 96 | stepFunction.startExecution(params).promise(); 97 | }); 98 | 99 | stepFunction.startExecution(params).promise(); 100 | }); 101 | }); 102 | 103 | test.serial('Step function with 3 poolConcurrency worker', t => { 104 | const {activityArn, stateMachineArn} = context; 105 | 106 | const worker = new StepFunctionWorker({ 107 | activityArn, 108 | logger, 109 | workerName: workerName + '-poolConcurrency', 110 | fn: fn2, 111 | poolConcurrency: 3 112 | }); 113 | const params1 = { 114 | stateMachineArn, 115 | input: JSON.stringify({inputNumber: '0'}) 116 | }; 117 | const params2 = { 118 | stateMachineArn, 119 | input: JSON.stringify({inputNumber: '1'}) 120 | }; 121 | const params3 = { 122 | stateMachineArn, 123 | input: JSON.stringify({inputNumber: '2'}) 124 | }; 125 | 126 | return new Promise((resolve, reject) => { 127 | let countTask = 0; 128 | let countSuccess = 0; 129 | const workerNames = []; 130 | const startDate = new Date(); 131 | const onTask = function (task) { 132 | // Task.taskToken 133 | // task.input 134 | // task.workerName 135 | countTask++; 136 | 137 | if (workerNames.indexOf(task.workerName) === -1) { 138 | workerNames.push(task.workerName); 139 | } 140 | 141 | if (countTask === 3) { 142 | worker.removeListener('task', onTask); 143 | t.is(workerNames.length, 3); 144 | } 145 | }; 146 | 147 | const onSuccess = function (out) { 148 | countSuccess++; 149 | if (workerNames.indexOf(out.workerName) === -1) { 150 | t.fail('workerName should have been seen on task event before'); 151 | } 152 | 153 | if (countSuccess === 1) { 154 | const report = worker.report(); 155 | t.is(report.poolers.length, 3); 156 | t.is(report.tasks.length, 0); 157 | } 158 | 159 | if (countSuccess === 3) { 160 | worker.removeListener('success', onSuccess); 161 | const endDate = new Date(); 162 | t.true((endDate - startDate) / 1000 < 3.9); 163 | t.true((endDate - startDate) / 1000 > 2); 164 | worker.close(() => { 165 | t.is(worker._poolers.length, 0); 166 | resolve(); 167 | }); 168 | } 169 | }; 170 | 171 | worker.on('success', onSuccess); 172 | worker.on('task', onTask); 173 | worker.on('error', reject); 174 | stepFunction.startExecution(params1).promise(); 175 | stepFunction.startExecution(params2).promise(); 176 | stepFunction.startExecution(params3).promise(); 177 | }); 178 | }); 179 | 180 | test.serial('Step function with deprecated concurrency worker', t => { 181 | const {activityArn, stateMachineArn} = context; 182 | 183 | const worker = new StepFunctionWorker({ 184 | activityArn, 185 | logger, 186 | workerName: workerName + '-concurrency', 187 | fn: fn2, 188 | concurrency: 3 189 | }); 190 | const params1 = { 191 | stateMachineArn, 192 | input: JSON.stringify({inputNumber: '0'}) 193 | }; 194 | const params2 = { 195 | stateMachineArn, 196 | input: JSON.stringify({inputNumber: '1'}) 197 | }; 198 | const params3 = { 199 | stateMachineArn, 200 | input: JSON.stringify({inputNumber: '2'}) 201 | }; 202 | 203 | return new Promise((resolve, reject) => { 204 | let countTask = 0; 205 | let countSuccess = 0; 206 | const workerNames = []; 207 | const startDate = new Date(); 208 | const onTask = function (task) { 209 | // Task.taskToken 210 | // task.input 211 | // task.workerName 212 | countTask++; 213 | 214 | if (workerNames.indexOf(task.workerName) === -1) { 215 | workerNames.push(task.workerName); 216 | } 217 | 218 | if (countTask === 3) { 219 | worker.removeListener('task', onTask); 220 | t.is(workerNames.length, 3); 221 | } 222 | }; 223 | 224 | const onSuccess = function (out) { 225 | countSuccess++; 226 | if (workerNames.indexOf(out.workerName) === -1) { 227 | t.fail('workerName should have been seen on task event before'); 228 | } 229 | 230 | if (countSuccess === 1) { 231 | const report = worker.report(); 232 | t.is(report.poolers.length, 3); 233 | t.is(report.tasks.length, 0); 234 | } 235 | 236 | if (countSuccess === 3) { 237 | worker.removeListener('success', onSuccess); 238 | const endDate = new Date(); 239 | t.true((endDate - startDate) / 1000 < 3.9); 240 | t.true((endDate - startDate) / 1000 > 2); 241 | worker.close(() => { 242 | t.is(worker._poolers.length, 0); 243 | resolve(); 244 | }); 245 | } 246 | }; 247 | 248 | worker.on('success', onSuccess); 249 | worker.on('task', onTask); 250 | worker.on('error', reject); 251 | stepFunction.startExecution(params1).promise(); 252 | stepFunction.startExecution(params2).promise(); 253 | stepFunction.startExecution(params3).promise(); 254 | }); 255 | }); 256 | test.serial('Restart the worker', t => { 257 | const {activityArn, stateMachineArn} = context; 258 | 259 | const worker = new StepFunctionWorker({ 260 | activityArn, 261 | logger, 262 | workerName: workerName + '-restart', 263 | fn: fn2, 264 | poolConcurrency: 1 265 | }); 266 | const params1 = { 267 | stateMachineArn, 268 | input: JSON.stringify({inputNumber: '0'}) 269 | }; 270 | const params2 = { 271 | stateMachineArn, 272 | input: JSON.stringify({inputNumber: '1'}) 273 | }; 274 | return new Promise((resolve, reject) => { 275 | let countSuccess = 0; 276 | 277 | const onSuccess = function (out) { 278 | countSuccess++; 279 | if (out.workerName === worker.workerName) { 280 | t.fail('workerName should be same than in worker'); 281 | } 282 | 283 | if (countSuccess === 1) { 284 | const beforeRestartLength = worker._poolers.length; 285 | logger.debug('restart'); 286 | worker.restart(() => { 287 | logger.debug('restarted'); 288 | t.is(worker._poolers.length, beforeRestartLength); 289 | stepFunction.startExecution(params2).promise(); 290 | }); 291 | } 292 | 293 | if (countSuccess === 2) { 294 | resolve(); 295 | } 296 | }; 297 | 298 | worker.on('success', onSuccess); 299 | worker.on('error', reject); 300 | stepFunction.startExecution(params1).promise(); 301 | }); 302 | }); 303 | 304 | test.after(after); 305 | 306 | -------------------------------------------------------------------------------- /lib/worker.js: -------------------------------------------------------------------------------- 1 | const {EventEmitter} = require('events'); 2 | const util = require('util'); 3 | const AWS = require('aws-sdk'); 4 | const parser = require('aws-arn-parser'); 5 | 6 | const Pooler = require('./pooler.js'); 7 | const Task = require('./task.js'); 8 | 9 | /** 10 | * @typedef {Object} AWSConfig see https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html 11 | */ 12 | /** 13 | * @class Worker 14 | * @param {object} options 15 | * @param {string} options.activityArn 16 | * @param {string} [options.workerName=null] 17 | * @param {function} [options.fn=null] 18 | * @param {boolean} [options.autoStart=true] 19 | * @param {boolean} [options.logger=null] winston-like logger 20 | * @param {string} [options.concurrency=1] 21 | * @param {AWSConfig} [options.awsConfig={}] 22 | * */ 23 | 24 | function Worker(options) { 25 | EventEmitter.call(this); 26 | const awsConfig = options.awsConfig || {}; 27 | this.stepfunction = new AWS.StepFunctions(awsConfig); 28 | 29 | this.autoStart = typeof (options.autoStart) === 'boolean' ? options.autoStart : true; 30 | 31 | if (!options.activityArn) { 32 | throw (new Error('activityArn is mandatory inside Worker')); 33 | } 34 | 35 | this.logger = options.logger || { 36 | debug() {}, 37 | info() {}, 38 | warn: console.warn, 39 | error: console.error 40 | }; 41 | 42 | if (typeof (options.concurrency) === 'number') { 43 | // To uncomment in the future 44 | // throw (new TypeError('step-function-worker `concurrency` parameter is deprectated since version 3.0, see README.md')); 45 | this.logger.warn('[step-function-worker] `concurrency` parameter is deprectated since version 3.0 please use `poolConcurrency` instead, see README.md'); 46 | if (typeof (options.poolConcurrency) !== 'number') { 47 | options.poolConcurrency = options.concurrency; 48 | } 49 | } 50 | 51 | this.poolConcurrency = typeof (options.poolConcurrency) === 'number' ? options.poolConcurrency : 1; 52 | this.taskConcurrency = typeof (options.taskConcurrency) === 'number' ? options.taskConcurrency : this.poolConcurrency; 53 | 54 | if (this.poolConcurrency > this.taskConcurrency) { 55 | throw (new Error(`poolConcurrency (${this.poolConcurrency}) < taskConcurrency (${this.taskConcurrency}) is invalid`)); 56 | } 57 | 58 | this.workerName = options.workerName; 59 | 60 | this.activityArn = options.activityArn; 61 | 62 | this.fn = options.fn; 63 | this._poolers = []; 64 | this._tasks = []; 65 | 66 | if (typeof (this.fn) !== 'function') { 67 | throw (new TypeError(`fn parameter should be a function (currently ${typeof (this.fn)})`)); 68 | } 69 | 70 | const {region} = parser(options.activityArn); 71 | 72 | if (typeof (region) === 'string' && (this.stepfunction.config.region !== region)) { 73 | throw (new Error(`activity ARN region (${region}) should match with AWS Region (${this.stepfunction.config.region})`)); 74 | } 75 | 76 | if (this.autoStart) { 77 | setImmediate(() => { 78 | this.start() 79 | .then(() => { 80 | // Do nothing 81 | this.emit('ready'); 82 | }) 83 | .catch(error => { 84 | this.logger.error('Worker failed to start', error); 85 | this.emit('error', error); 86 | }); 87 | }); 88 | } 89 | } 90 | 91 | /** 92 | * Start the worker pooling for new tasks 93 | * @param {function} cb callback(err) 94 | * @returns {Promise} empty Promise 95 | */ 96 | Worker.prototype.start = function () { 97 | this.increasePool(); 98 | this.logger.info('Worker started'); 99 | return Promise.resolve(); 100 | }; 101 | 102 | /** 103 | * Get a report of the actual situation of the worker 104 | * @return {Array.} list of poolers 105 | */ 106 | Worker.prototype.report = function () { 107 | return { 108 | poolers: this._poolers.map(pooler => { 109 | return pooler.report(); 110 | }), 111 | tasks: this._tasks.map(task => { 112 | return task.report(); 113 | }) 114 | }; 115 | }; 116 | 117 | Worker.prototype.renewPooler = function (pooler) { 118 | const maxNumberOfPools = this.getMaxNumberOfPools(); 119 | 120 | if (this._poolers.length > maxNumberOfPools) { 121 | const index = this._poolers.indexOf(pooler); 122 | if (index === -1) { 123 | throw (new Error('cannot removed non-listed pooler')); 124 | } 125 | 126 | return false; 127 | } 128 | 129 | this.increasePool(); 130 | return true; 131 | }; 132 | 133 | Worker.prototype.getMaxNumberOfPools = function () { 134 | let maxNumberOfPools = this.poolConcurrency; 135 | if (typeof (this.taskConcurrency) === 'number') { 136 | maxNumberOfPools = Math.min(this.taskConcurrency - this._tasks.length, this.poolConcurrency); 137 | } 138 | 139 | if (maxNumberOfPools < 0) { 140 | throw (new Error(`maxNumberOfPools (${maxNumberOfPools}) should be positive`)); 141 | } 142 | 143 | return maxNumberOfPools; 144 | }; 145 | 146 | Worker.prototype.increasePool = function () { 147 | const maxNumberOfPools = this.getMaxNumberOfPools(); 148 | this.logger.debug('increasePool started', maxNumberOfPools, this._poolers.length); 149 | 150 | if (this._poolers.length < maxNumberOfPools) { 151 | this.addPooler(this._poolers.length); 152 | return this.increasePool(); 153 | } 154 | 155 | if (this._poolers.length > maxNumberOfPools) { 156 | return false; 157 | } 158 | 159 | return true; 160 | }; 161 | 162 | Worker.prototype.addTask = function (params) { 163 | // This.logger.count('addTask'); 164 | const task = new Task(Object.assign({}, params, {worker: this, logger: this.logger})); 165 | this._tasks.push(task); 166 | this.emit('task', params); 167 | task.on('finish', () => { 168 | // This.logger.count('finishTask'); 169 | const index = this._tasks.indexOf(task); 170 | if (index === -1) { 171 | throw (new Error('tasks is not registered in _tasks')); 172 | } 173 | 174 | this._tasks.splice(index, 1); 175 | this.updateTasks(); 176 | this.increasePool(); 177 | }); 178 | this.updateTasks(); 179 | }; 180 | 181 | Worker.prototype.updateTasks = function () { 182 | if (typeof (this.taskConcurrency) === 'number') { 183 | if (this._tasks.length === this.taskConcurrency) { 184 | this.emit('full'); 185 | } else if (this._tasks.length > this.taskConcurrency) { 186 | throw (new Error(`Should not reach ${this._tasks.length} tasks`)); 187 | } 188 | } 189 | 190 | if (this._tasks.length === 0) { 191 | this.logger.info('empty'); 192 | this.emit('empty'); 193 | } 194 | }; 195 | 196 | Worker.prototype.addPooler = function (index) { 197 | this.logger.debug('addPooler'); 198 | const pooler = new Pooler({ 199 | activityArn: this.activityArn, 200 | workerName: this.workerName, 201 | worker: this, 202 | logger: this.logger, 203 | index 204 | }); 205 | 206 | this._poolers.push(pooler); 207 | }; 208 | 209 | Worker.prototype.removePooler = function (pooler) { 210 | this.logger.debug('removePooler'); 211 | 212 | const index = this._poolers.indexOf(pooler); 213 | if (index === -1) { 214 | throw (new Error(`pooler ${pooler} is not in the pooler list`)); 215 | } 216 | 217 | this._poolers.splice(index, 1); 218 | 219 | if (this._poolers.length === 0) { 220 | this.emit('empty-poolers'); 221 | } 222 | }; 223 | 224 | Worker.prototype.removeTask = function (pooler) { 225 | this.logger.debug('removePooler'); 226 | 227 | const index = this._poolers.indexOf(pooler); 228 | if (index === -1) { 229 | throw (new Error(`pooler ${pooler} is not in the pooler list`)); 230 | } 231 | 232 | this._poolers.splice(index, 1); 233 | 234 | if (this._poolers.length === 0) { 235 | this.emit('empty-poolers'); 236 | } 237 | }; 238 | 239 | // Worker.prototype.removePooler = function () { 240 | // if(!this._poolerRemovalPromise){ 241 | // this._poolerRemovalPromise = Promise.resolve() 242 | // .then(() => { 243 | // this.logger.debug('removePooler started') 244 | // const removedPooler = this._poolers[this._poolers.length -1]; 245 | // const id = Math.random(); 246 | // const _this = this; 247 | // return removedPooler.stop() 248 | // }).then(() => { 249 | // const index = _this._poolers.indexOf(removedPooler); 250 | // if(index === -1){ 251 | // throw(new Error('cross poolers removal is not expected')) 252 | // } 253 | // _this._poolers.splice(index, 1); 254 | // return _this._poolers 255 | // }).then(r => { 256 | // this.logger.debug('removePooler ended') 257 | // 258 | // this._poolerRemovalPromise = null 259 | // return r; 260 | // }) 261 | // } 262 | // 263 | // return this._poolerRemovalPromise; 264 | // }; 265 | 266 | /** 267 | * Close the worker, this function might take 60 seconds to finish to do step function design 268 | * remove all the events attached to the worker 269 | * @param {function} callback 270 | */ 271 | 272 | Worker.prototype.close = function (cb) { 273 | this.removeAllListeners(); 274 | const promise = this.stop(); 275 | 276 | if (cb) { 277 | promise.then(() => cb()).catch(cb); 278 | } else { 279 | return promise; 280 | } 281 | }; 282 | 283 | /** 284 | * Stop the worker 285 | * But does not remove all the events attached to it 286 | * NB: worker.concurrency is set to 0 287 | * @param {function} callback 288 | */ 289 | 290 | Worker.prototype.stop = function () { 291 | this.logger.info('Stopping the worker ... this might take 60 seconds'); 292 | this.poolConcurrency = 0; 293 | if (!this._stoppingPromise) { 294 | this._stoppingPromise = new Promise((resolve, reject) => { 295 | const onEmpty = () => { 296 | this.logger.info('Worker stopped'); 297 | if (this._tasks.length > 0) { 298 | const err = new Error('Some tasks are still ongoing, please make sure all the tasks are finished before stopping the worker'); 299 | return reject(err); 300 | } 301 | 302 | return resolve(); 303 | }; 304 | 305 | if (this._poolers.length === 0) { 306 | onEmpty(); 307 | } 308 | 309 | this.once('empty-poolers', () => { 310 | onEmpty(); 311 | }); 312 | }); 313 | } 314 | 315 | return this._stoppingPromise; 316 | }; 317 | 318 | Worker.prototype.restart = function (cb) { 319 | const oldPoolConcurrency = this.poolConcurrency; 320 | 321 | const promise = this.stop().then(() => { 322 | this.poolConcurrency = oldPoolConcurrency; 323 | return this.start(); 324 | }); 325 | if (cb) { 326 | promise.catch(cb).then(() => cb()); 327 | } else { 328 | return promise; 329 | } 330 | }; 331 | 332 | util.inherits(Worker, EventEmitter); 333 | 334 | module.exports = Worker; 335 | --------------------------------------------------------------------------------