├── .editorconfig ├── .eslintignore ├── .eslintrc.json ├── .gitattributes ├── .gitignore ├── .prettierrc ├── CHANGELOG.md ├── LICENSE ├── README.md ├── circle.yml ├── demo ├── Dockerfile ├── docker-compose.yml ├── handler.js ├── package-lock.json ├── package.json ├── serverless.yml └── webpack.config.js ├── package-lock.json ├── package.json ├── src ├── DynamoDBStreamReadable.js ├── FunctionExecutable.js ├── executeFunctions.js └── index.js └── test ├── executeFunctions.test.js ├── handler.js └── test.js /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | end_of_line = lf 7 | indent_size = 2 8 | indent_style = space 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true 11 | 12 | [*.md] 13 | max_line_length = off 14 | trim_trailing_whitespace = false 15 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "prettier", 4 | "plugin:prettier/recommended", 5 | "plugin:jest/recommended" 6 | ], 7 | "env": { 8 | "es6": true, 9 | "jest": true 10 | }, 11 | "plugins": ["prettier", "json", "jest"] 12 | } 13 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | .webpack -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "arrowParens": "always", 3 | "singleQuote": true 4 | } 5 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v1.0.0 4 | 5 | * Initial release 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Orchestrated System 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # serverless-plugin-offline-dynamodb-stream 2 | 3 | [![CircleCI Status][circleci-badge]][circleci-url] 4 | [![NPM Version][npm-badge]][npm-url] 5 | [![License][license-badge]][license-url] 6 | 7 | > Serverless framework offline plugin to support dynamodb stream 8 | 9 | This plugin pull from dynamodb stream and trigger serverless function if any records detected. 10 | 11 | # Installation 12 | 13 | Install package 14 | 15 | ```bash 16 | $ npm install --save serverless-plugin-offline-dynamodb-stream 17 | ``` 18 | 19 | # Usage 20 | 21 | Add following config to serverless.yml file. 22 | 23 | ```yml 24 | plugins: 25 | - serverless-plugin-offline-dynamodb-stream 26 | - serverless-plugin-offline-kinesis-stream 27 | custom: 28 | dynamodbStream: 29 | host: {LOCAL_DYNAMODB_HOST} 30 | port: {LOCAL_DYNAMODB_PORT} 31 | pollForever: boolean 32 | streams: 33 | - table: {TABLE_NAME} 34 | functions: 35 | - {FUNCTION_NAME} 36 | kinesisStream: 37 | host: {LOCAL_KINESIS_HOST} 38 | port: {LOCAL_KINESIS_PORT} 39 | intervalMillis: 5000 40 | streams: 41 | - streamName: {STREAM_NAME} 42 | functions: 43 | - {FUNCTION_NAME} 44 | ``` 45 | 46 | #### pollForever 47 | * pollForever can be set to `true` to indicate that this plugin should continue to poll for dynamodbstreams events indefinity. If 48 | `pollForever` is not set, or is set to false, the plugin will stop polling for events once the end of the 49 | stream is reached (when dynamodbstreams.getRecords => data.NextShardIterator === null), or an error occurs. 50 | 51 | * With `pollForever` set to `true` the following events will trigger a restart instead of exiting as would happen with `pollForever` set to `false`: 52 | * The end of a Dynamodb Stream is reached (when dynamodbstreams.getRecords => data.NextShardIterator === null) 53 | * [ExpiredIteratorException](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_streams_GetRecords.html) is thrown from `dynamodbstreams.getRecords`. 54 | 55 | * This can be useful in scenarios where you have a lambda function as part of a larger service struture, and the other services depend on the functinality in the lambda. 56 | 57 | Ensure your local dynamodb is up and running, or you could also consider using [serverless-dynamodb-local](https://github.com/99xt/serverless-dynamodb-local) plugin before start your serverless offline process. 58 | 59 | ```bash 60 | $ serverless offline start 61 | ``` 62 | 63 | # Development 64 | 65 | * Cloning the repo 66 | 67 | ```bash 68 | $ git clone https://github.com/orchestrated-io/serverless-plugin-offline-dynamodb-stream.git 69 | ``` 70 | 71 | * Installing dependencies 72 | 73 | ```bash 74 | $ npm install 75 | ``` 76 | 77 | * Running scripts 78 | 79 | | Action | Usage | 80 | | ---------------------------------------- | ------------------- | 81 | | Linting code | `npm run lint` | 82 | | Running unit tests | `npm run jest` | 83 | | Running code coverage | `npm run coverage` | 84 | | Running lint + tests | `npm test` | 85 | 86 | # Demo 87 | 88 | ``` 89 | > cd demo 90 | > docker-compose up --build 91 | ``` 92 | 93 | * open [dynamodb admin](http://localhost:8001/tables/items/items) in browser. 94 | * adding new item on items table will result event detail printed out in console. 95 | 96 | # Author 97 | 98 | [Emmanuel Kong](https://github.com/emmkong) 99 | 100 | # License 101 | 102 | [MIT](https://github.com/orchestrated-io/serverless-plugin-offline-dynamodb-stream/blob/master/LICENSE) 103 | 104 | [circleci-badge]: https://circleci.com/gh/orchestrated-io/serverless-plugin-offline-dynamodb-stream/tree/master.svg?style=shield 105 | [circleci-url]: https://circleci.com/gh/orchestrated-io/serverless-plugin-offline-dynamodb-stream 106 | 107 | [npm-badge]: https://img.shields.io/npm/v/serverless-plugin-offline-dynamodb-stream.svg 108 | [npm-url]: https://www.npmjs.com/package/serverless-plugin-offline-dynamodb-stream 109 | 110 | [license-badge]: https://img.shields.io/github/license/orchestrated-io/serverless-plugin-offline-dynamodb-stream.svg 111 | [license-url]: https://opensource.org/licenses/MIT 112 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | defaults: &defaults 4 | working_directory: ~/repo 5 | docker: 6 | - image: circleci/node:14 7 | 8 | jobs: 9 | test: 10 | <<: *defaults 11 | steps: 12 | - checkout 13 | - restore_cache: 14 | keys: 15 | - node-v1-{{ .Branch }}-{{ checksum "package-lock.json" }} 16 | - node-v1-{{ .Branch }}- 17 | - node-v1- 18 | - run: 19 | name: install packages 20 | command: npm install 21 | - run: 22 | name: test 23 | command: npm test 24 | - save_cache: 25 | paths: 26 | - ~/repo/node_modules 27 | key: node-v1-{{ .Branch }}-{{ checksum "package-lock.json" }} 28 | - persist_to_workspace: 29 | root: ~/repo 30 | paths: . 31 | deploy: 32 | <<: *defaults 33 | steps: 34 | - attach_workspace: 35 | at: ~/repo 36 | - run: 37 | name: Set NPM authentication. 38 | command: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/repo/.npmrc 39 | - run: 40 | name: Publish package 41 | command: npm publish 42 | workflows: 43 | version: 2 44 | test-deploy: 45 | jobs: 46 | - test: 47 | filters: 48 | tags: 49 | only: /^v.*/ 50 | - deploy: 51 | requires: 52 | - test 53 | filters: 54 | tags: 55 | only: /^v.*/ 56 | branches: 57 | ignore: /.*/ 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /demo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:8.10 2 | 3 | # Create app directory 4 | RUN mkdir -p /usr/src/app 5 | WORKDIR /usr/src/app -------------------------------------------------------------------------------- /demo/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.3' 2 | 3 | services: 4 | dynamodb: 5 | image: cnadiminti/dynamodb-local:latest 6 | admin: 7 | image: boogak/dynamodb-admin 8 | ports: 9 | - 8001:8001 10 | environment: 11 | - DYNAMO_ENDPOINT=http://dynamodb:8000 12 | serverless: 13 | build: . 14 | volumes: 15 | - .:/usr/src/app 16 | environment: 17 | # - DEBUG=* 18 | - AWS_REGION=localhost 19 | - AWS_ACCESS_KEY_ID=XXXXXXXXXXXXXXXXXXXX 20 | - AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/xxx+ 21 | command: yarn start 22 | links: 23 | - dynamodb 24 | -------------------------------------------------------------------------------- /demo/handler.js: -------------------------------------------------------------------------------- 1 | exports.processItem = (event) => { 2 | console.log('[new dynamodb event received] :=> ', JSON.stringify(event)); 3 | }; 4 | -------------------------------------------------------------------------------- /demo/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "demo", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "license": "MIT", 6 | "scripts": { 7 | "start": "serverless offline start", 8 | "build": "serverless package", 9 | "print": "serverless print", 10 | "deploy": "serverless deploy" 11 | }, 12 | "dependencies": { 13 | "babel-core": "^6.26.3", 14 | "babel-loader": "^7.1.5", 15 | "babel-preset-env": "^1.7.0", 16 | "babel-preset-stage-0": "^6.24.1", 17 | "require-without-cache": "^0.0.6", 18 | "serverless": "^1.61.3", 19 | "serverless-dynamodb-local": "^0.2.39", 20 | "serverless-offline": "^5.12.1", 21 | "serverless-plugin-offline-dynamodb-stream": "^1.0.19", 22 | "serverless-webpack": "^5.3.1", 23 | "webpack": "^4.41.5" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /demo/serverless.yml: -------------------------------------------------------------------------------- 1 | service: demo 2 | provider: 3 | name: aws 4 | runtime: nodejs8.10 5 | environment: 6 | region: localhost 7 | plugins: 8 | - serverless-webpack 9 | - serverless-dynamodb-local 10 | - serverless-plugin-offline-dynamodb-stream 11 | - serverless-offline 12 | custom: 13 | serverless-offline: 14 | host: 0.0.0.0 15 | port: 4000 16 | dontPrintOutput: true 17 | dynamodb: 18 | start: 19 | host: dynamodb 20 | port: 8000 21 | migrate: true 22 | noStart: true 23 | dynamodbStream: 24 | host: dynamodb 25 | port: 8000 26 | streams: 27 | - table: items 28 | functions: 29 | - processItem 30 | functions: 31 | processItem: 32 | handler: handler.processItem 33 | resources: 34 | Resources: 35 | EventTable: 36 | Type: 'AWS::DynamoDB::Table' 37 | Properties: 38 | AttributeDefinitions: 39 | - AttributeName: id 40 | AttributeType: S 41 | - AttributeName: name 42 | AttributeType: S 43 | KeySchema: 44 | - AttributeName: id 45 | KeyType: HASH 46 | - AttributeName: name 47 | KeyType: RANGE 48 | TableName: items 49 | ProvisionedThroughput: 50 | ReadCapacityUnits: 1 51 | WriteCapacityUnits: 1 52 | StreamSpecification: 53 | StreamViewType: NEW_IMAGE 54 | -------------------------------------------------------------------------------- /demo/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | // eslint-disable-next-line import/no-extraneous-dependencies 3 | const slsw = require('serverless-webpack'); 4 | 5 | module.exports = { 6 | entry: './handler.js', 7 | target: 'node', 8 | devtool: 'eval-source-map', 9 | module: { 10 | rules: [ 11 | { 12 | test: /\.js$/, 13 | exclude: /node_modules/, 14 | use: [ 15 | { 16 | loader: 'babel-loader', 17 | options: { 18 | presets: [['env', { targets: { node: '8.10' } }], 'stage-0'] 19 | } 20 | } 21 | ] 22 | } 23 | ] 24 | }, 25 | output: { 26 | libraryTarget: 'commonjs', 27 | path: path.join(__dirname, '.webpack'), 28 | filename: 'handler.js' 29 | } 30 | }; 31 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "serverless-plugin-offline-dynamodb-stream", 3 | "version": "1.0.20", 4 | "description": "Serverless framework offline plugin to support dynamodb stream", 5 | "author": "Emmanuel Kong ", 6 | "homepage": "https://github.com/orchestrated-io/serverless-plugin-offline-dynamodb-stream#readme", 7 | "keywords": [ 8 | "node" 9 | ], 10 | "main": "src/index.js", 11 | "files": [ 12 | "src" 13 | ], 14 | "scripts": { 15 | "coverage": "jest --coverage ", 16 | "jest": "jest", 17 | "lint": "eslint src/**/*.js test/**/*.js --fix", 18 | "test": "npm run lint && npm run jest" 19 | }, 20 | "repository": { 21 | "type": "git", 22 | "url": "https://github.com/orchestrated-io/serverless-plugin-offline-dynamodb-stream.git" 23 | }, 24 | "bugs": { 25 | "url": "https://github.com/orchestrated-io/serverless-plugin-offline-dynamodb-stream/issues" 26 | }, 27 | "dependencies": { 28 | "aws-sdk": "^2.896.0", 29 | "debug": "^4.3.1", 30 | "lodash": "^4.17.21", 31 | "path": "0.12.7", 32 | "require-without-cache": "^0.0.6" 33 | }, 34 | "devDependencies": { 35 | "eslint": "^7.25.0", 36 | "eslint-config-prettier": "^8.3.0", 37 | "eslint-config-standard": "^16.0.2", 38 | "eslint-plugin-import": "^2.22.1", 39 | "eslint-plugin-jest": "^24.3.6", 40 | "eslint-plugin-json": "^2.1.2", 41 | "eslint-plugin-node": "^11.1.0", 42 | "eslint-plugin-prettier": "^3.4.0", 43 | "eslint-plugin-promise": "^5.1.0", 44 | "eslint-plugin-standard": "^5.0.0", 45 | "jest": "^26.6.3", 46 | "prettier": "2.2.1" 47 | }, 48 | "jest": { 49 | "testEnvironment": "node" 50 | }, 51 | "license": "MIT" 52 | } 53 | -------------------------------------------------------------------------------- /src/DynamoDBStreamReadable.js: -------------------------------------------------------------------------------- 1 | const Readable = require('stream').Readable; 2 | const debug = require('debug')('DynamoDBStream-streams:readable'); 3 | 4 | function sleep(timeout, shouldWait, ...args) { 5 | if (shouldWait || timeout === 0) { 6 | return Promise.resolve(...args); 7 | } 8 | 9 | return new Promise((resolve) => { 10 | setTimeout(() => resolve(...args), timeout); 11 | }); 12 | } 13 | 14 | function timeout(func, timeout, ...args) { 15 | return new Promise((resolve) => { 16 | setTimeout(() => resolve(func(...args)), timeout); 17 | }); 18 | } 19 | 20 | class DynamoDBStreamReadable extends Readable { 21 | constructor(client, streamArn, pollForever, options = {}) { 22 | if (!client) { 23 | throw new Error('client is required'); 24 | } 25 | if (!streamArn) { 26 | throw new Error('streamArn is required'); 27 | } 28 | 29 | super(Object.assign({ objectMode: true }, options)); 30 | 31 | this.client = client; 32 | this.streamArn = streamArn; 33 | this.options = { 34 | interval: 2000, 35 | parser: JSON.parse, 36 | }; 37 | this.pollForever = !!pollForever; 38 | this._started = 0; 39 | } 40 | 41 | getShard() { 42 | const params = { 43 | StreamArn: this.streamArn, 44 | }; 45 | return this.client 46 | .describeStream(params) 47 | .promise() 48 | .then((data) => { 49 | if (!data.StreamDescription.Shards.length) { 50 | throw new Error('No shards!'); 51 | } 52 | 53 | debug('getShard found %d shards', data.StreamDescription.Shards.length); 54 | 55 | const [openShard] = data.StreamDescription.Shards.filter( 56 | (shard) => !shard.SequenceNumberRange.EndingSequenceNumber 57 | ); 58 | 59 | return openShard && openShard.ShardId; 60 | }); 61 | } 62 | 63 | getShardIterator(shardId, options) { 64 | const params = Object.assign( 65 | { 66 | ShardId: shardId, 67 | ShardIteratorType: 'LATEST', 68 | StreamArn: this.streamArn, 69 | }, 70 | options || {} 71 | ); 72 | return this.client 73 | .getShardIterator(params) 74 | .promise() 75 | .then((data) => { 76 | debug('getShardIterator got iterator id: %s', data.ShardIterator); 77 | return data.ShardIterator; 78 | }); 79 | } 80 | 81 | _startDynamoDBStream(size, shardIteratorOptions = {}) { 82 | return this.getShard() 83 | .then((shardId) => this.getShardIterator(shardId, shardIteratorOptions)) 84 | .then((shardIterator) => this.readShard(shardIterator, size)) 85 | .then((shardIterator) => { 86 | if (!shardIterator && this.pollForever) { 87 | debug('stream ended -- pollForever enabled -- restarting'); 88 | return timeout(this._startDynamoDBStream.bind(this), 2000, size); 89 | } 90 | 91 | return shardIterator; 92 | }) 93 | .catch((err) => { 94 | if (err.code === 'ExpiredIteratorException') { 95 | if (this.pollForever) { 96 | debug( 97 | 'readShard - ExpiredIteratorException -- pollForever enabled -- restarting' 98 | ); 99 | return this._startDynamoDBStream(size); 100 | } 101 | this.emit('error', err) || console.log(err, err.stack); 102 | } else if (err.code === 'TrimmedDataAccessException') { 103 | debug( 104 | 'readShard - TrimmedDataAccessException -> restart dynamodb stream' 105 | ); 106 | const refetchShardIteratorOptions = { 107 | ShardIteratorType: 'TRIM_HORIZON', 108 | }; 109 | return this._startDynamoDBStream(size, refetchShardIteratorOptions); 110 | } else { 111 | this.emit('error', err) || console.log(err, err.stack); 112 | } 113 | }); 114 | } 115 | 116 | readShard(shardIterator, size) { 117 | const params = { 118 | ShardIterator: shardIterator, 119 | Limit: size, 120 | }; 121 | return this.client 122 | .getRecords(params) 123 | .promise() 124 | .then((data) => { 125 | if (data.MillisBehindLatest > 60 * 1000) { 126 | debug('behind by %d milliseconds', data.MillisBehindLatest); 127 | } 128 | if (data.Records.length) { 129 | for (const record in data.Records) { 130 | data.Records[record].eventSourceARN = this.streamArn; 131 | } 132 | this.push(data); 133 | this.emit( 134 | 'checkpoint', 135 | data.Records[data.Records.length - 1].SequenceNumber 136 | ); 137 | } 138 | if (!data.NextShardIterator) { 139 | debug('readShard.closed %s', shardIterator); 140 | } 141 | 142 | return { 143 | nextShardIterator: data.NextShardIterator, 144 | hasData: data.Records.length > 0, 145 | }; 146 | }) 147 | .then(({ nextShardIterator, hasData }) => { 148 | if (nextShardIterator) { 149 | return sleep(this.options.interval, hasData, nextShardIterator); 150 | } 151 | 152 | return null; 153 | }) 154 | .then((nextShardIterator) => { 155 | if (nextShardIterator) { 156 | return this.readShard(nextShardIterator, size); 157 | } 158 | 159 | return null; 160 | }); 161 | } 162 | 163 | _read(size) { 164 | if (this._started) { 165 | return; 166 | } 167 | 168 | this._startDynamoDBStream(size) 169 | .then(() => { 170 | this._started = 2; 171 | }) 172 | .catch((err) => { 173 | this.emit('error', err) || console.log(err, err.stack); 174 | }); 175 | this._started = 1; 176 | } 177 | } 178 | 179 | module.exports = DynamoDBStreamReadable; 180 | -------------------------------------------------------------------------------- /src/FunctionExecutable.js: -------------------------------------------------------------------------------- 1 | const { Writable } = require('stream'); 2 | const executeFunctions = require('./executeFunctions'); 3 | 4 | const FunctionExecutable = (location, functions) => 5 | new Writable({ 6 | write(chunk = [], encoding, callback) { 7 | executeFunctions(chunk, location, functions).then(() => { 8 | callback(); 9 | }); 10 | }, 11 | objectMode: true, 12 | }); 13 | 14 | module.exports = FunctionExecutable; 15 | -------------------------------------------------------------------------------- /src/executeFunctions.js: -------------------------------------------------------------------------------- 1 | const { isNil, isFunction, map } = require('lodash'); 2 | const requireWithoutCache = require('require-without-cache'); 3 | 4 | const promisify = (foo) => 5 | new Promise((resolve, reject) => { 6 | foo((error, result) => { 7 | if (error) { 8 | reject(error); 9 | } else { 10 | resolve(result); 11 | } 12 | }); 13 | }); 14 | 15 | const createHandler = (location, fn) => { 16 | const originalEnv = Object.assign({}, process.env); 17 | process.env = Object.assign({}, originalEnv, fn.environment); 18 | 19 | const handler = requireWithoutCache( 20 | location + '/' + fn.handler.substring(0, fn.handler.lastIndexOf('.')), 21 | require 22 | )[fn.handler.split('/').pop().split('.')[1]]; 23 | return (event, context = {}) => 24 | promisify((cb) => { 25 | const maybeThennable = handler(event, context, cb); 26 | if (!isNil(maybeThennable) && isFunction(maybeThennable.then)) { 27 | maybeThennable 28 | .then((result) => { 29 | process.env = originalEnv; 30 | return cb(null, result); 31 | }) 32 | .catch((err) => cb(err)); 33 | } 34 | }); 35 | }; 36 | 37 | const executeFunctions = (events = [], location, functions) => { 38 | return Promise.all( 39 | map(functions, (fn) => { 40 | const handler = createHandler(location, fn); 41 | return handler(events); 42 | }) 43 | ); 44 | }; 45 | 46 | module.exports = executeFunctions; 47 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const AWS = require('aws-sdk'); 3 | const requireWithoutCache = require('require-without-cache'); 4 | const DynamoDBStreamReadable = require('./DynamoDBStreamReadable'); 5 | const FunctionExecutable = require('./FunctionExecutable'); 6 | 7 | class ServerlessPluginOfflineDynamodbStream { 8 | constructor(serverless, options) { 9 | this.serverless = serverless; 10 | this.config = 11 | (serverless.service.custom && serverless.service.custom.dynamodbStream) || 12 | {}; 13 | this.options = options; 14 | this.provider = 'aws'; 15 | this.commands = {}; 16 | this.hooks = { 17 | 'before:offline:start:init': this.startReadableStreams.bind(this), 18 | }; 19 | } 20 | 21 | createHandler(location, fn) { 22 | const handler = requireWithoutCache( 23 | location + '/' + fn.handler.split('.')[0], 24 | require 25 | )[fn.handler.split('/').pop().split('.')[1]]; 26 | return (event, context = {}) => handler(event, context); 27 | } 28 | 29 | startReadableStreams() { 30 | const { 31 | config: { 32 | host: hostname, 33 | port, 34 | region, 35 | batchSize, 36 | pollForever = false, 37 | } = {}, 38 | serverless: { service: { provider: { environment } = {} } = {} } = {}, 39 | } = this; 40 | const endpoint = new AWS.Endpoint(`http://${hostname}:${port}`); 41 | const offlineConfig = 42 | this.serverless.service.custom['serverless-offline'] || {}; 43 | const fns = this.serverless.service.functions; 44 | 45 | process.env = Object.assign({}, process.env, environment); 46 | 47 | let location = process.cwd(); 48 | if (offlineConfig.location) { 49 | location = process.cwd() + '/' + offlineConfig.location; 50 | } else if (this.serverless.config.servicePath) { 51 | location = this.serverless.config.servicePath; 52 | } 53 | 54 | const streams = (this.config.streams || []).map( 55 | ({ table, functions = [] }) => ({ 56 | table, 57 | functions: functions.map((functionName) => _.get(fns, functionName)), 58 | }) 59 | ); 60 | 61 | streams.forEach(({ table, functions }) => { 62 | const dynamo = endpoint 63 | ? new AWS.DynamoDB({ region, endpoint }) 64 | : new AWS.DynamoDB({ region }); 65 | dynamo.describeTable({ TableName: table }, (err, tableDescription) => { 66 | if (err) { 67 | throw err; 68 | } 69 | if ( 70 | tableDescription && 71 | tableDescription.Table && 72 | tableDescription.Table.LatestStreamArn 73 | ) { 74 | const streamArn = tableDescription.Table.LatestStreamArn; 75 | 76 | const ddbStream = endpoint 77 | ? new AWS.DynamoDBStreams({ 78 | region, 79 | endpoint, 80 | }) 81 | : new AWS.DynamoDBStreams({ region }); 82 | 83 | const readable = new DynamoDBStreamReadable( 84 | ddbStream, 85 | streamArn, 86 | pollForever, 87 | { 88 | highWaterMark: batchSize, 89 | } 90 | ); 91 | 92 | const functionExecutable = FunctionExecutable(location, functions); 93 | 94 | readable.on('error', (error) => { 95 | console.log( 96 | `DynamoDBStreamReadable error... terminating stream... Error => ${error}` 97 | ); 98 | functionExecutable.destroy(error); 99 | }); 100 | 101 | readable.pipe(functionExecutable).on('end', () => { 102 | console.log(`stream for table [${table}] closed!`); 103 | }); 104 | } 105 | }); 106 | }); 107 | } 108 | } 109 | 110 | module.exports = ServerlessPluginOfflineDynamodbStream; 111 | -------------------------------------------------------------------------------- /test/executeFunctions.test.js: -------------------------------------------------------------------------------- 1 | const executeFunctions = require('../src/executeFunctions'); 2 | 3 | test('should able to handle async function', () => { 4 | return executeFunctions('result', `${process.cwd()}/test`, [ 5 | { 6 | handler: 'handler.functionA', 7 | }, 8 | ]).then(([result]) => { 9 | expect(result).toBe('resultA'); 10 | }); 11 | }); 12 | 13 | test('should able to handle return promise', () => { 14 | return executeFunctions('result', `${process.cwd()}/test`, [ 15 | { 16 | handler: 'handler.functionB', 17 | }, 18 | ]).then(([result]) => { 19 | expect(result).toBe('resultB'); 20 | }); 21 | }); 22 | 23 | test('should able to handle via callback', () => { 24 | return executeFunctions('result', `${process.cwd()}/test`, [ 25 | { 26 | handler: 'handler.functionC', 27 | }, 28 | ]).then(([result]) => { 29 | expect(result).toBe('resultC'); 30 | }); 31 | }); 32 | 33 | test('should able to handle multiple functions', () => { 34 | return executeFunctions('result', `${process.cwd()}/test`, [ 35 | { 36 | handler: 'handler.functionA', 37 | }, 38 | { 39 | handler: 'handler.functionB', 40 | }, 41 | { 42 | handler: 'handler.functionC', 43 | }, 44 | ]).then(([result1, result2, result3, result4]) => { 45 | expect(result1).toBe('resultA'); 46 | expect(result2).toBe('resultB'); 47 | expect(result3).toBe('resultC'); 48 | }); 49 | }); 50 | 51 | test('should able to handle any functions with a dot in path', () => { 52 | return executeFunctions('result', `${process.cwd()}/test`, [ 53 | { 54 | handler: './handler.functionA' 55 | }, 56 | { 57 | handler: './handler.functionB' 58 | }, 59 | { 60 | handler: './handler.functionC' 61 | } 62 | ]).then(([result1, result2, result3, result4]) => { 63 | expect(result1).toBe('resultA'); 64 | expect(result2).toBe('resultB'); 65 | expect(result3).toBe('resultC'); 66 | }); 67 | }); 68 | -------------------------------------------------------------------------------- /test/handler.js: -------------------------------------------------------------------------------- 1 | function functionA(event) { 2 | return Promise.resolve(event + 'A'); 3 | } 4 | function functionB(event) { 5 | return new Promise((resolve, reject) => { 6 | setTimeout(() => { 7 | resolve(event + 'B'); 8 | }, 500); 9 | }); 10 | } 11 | function functionC(event, context, callback) { 12 | setTimeout(() => { 13 | callback(null, event + 'C'); 14 | }, 300); 15 | } 16 | 17 | module.exports = { 18 | functionA, 19 | functionB, 20 | functionC, 21 | }; 22 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | jest.mock('../src/DynamoDBStreamReadable'); 2 | jest.mock('./handler'); 3 | const ServerlessPluginOfflineDynamodbStream = require('../src'); 4 | 5 | const handler = require('./handler'); 6 | const serverless = { 7 | config: { 8 | servicePath: '../test', 9 | }, 10 | service: { 11 | custom: { 12 | 'serverless-offline': {}, 13 | dynamodbStream: { 14 | streams: [{ table: 'table-name', functions: ['funtionA'] }], 15 | }, 16 | functions: { funtionA: { handler: 'handler.funtionA' } }, 17 | }, 18 | }, 19 | }; 20 | const options = {}; 21 | 22 | describe('Serverless Plugin Offline Dynamodb Stream', () => { 23 | test('Meet serverless plugin interface', () => { 24 | const plugin = new ServerlessPluginOfflineDynamodbStream( 25 | serverless, 26 | options 27 | ); 28 | expect(plugin.hooks).toEqual({ 29 | 'before:offline:start:init': expect.any(Function), 30 | }); 31 | }); 32 | 33 | test('should create handler function', () => { 34 | const plugin = new ServerlessPluginOfflineDynamodbStream( 35 | serverless, 36 | options 37 | ); 38 | const hanler = plugin.createHandler(`${process.cwd()}/test`, { 39 | handler: 'handler.funtionA', 40 | }); 41 | expect(hanler).toEqual(expect.any(Function)); 42 | }); 43 | 44 | test('should has correct hook', () => { 45 | const plugin = new ServerlessPluginOfflineDynamodbStream( 46 | serverless, 47 | options 48 | ); 49 | expect(plugin.hooks['before:offline:start:init']).toBeTruthy(); 50 | }); 51 | }); 52 | --------------------------------------------------------------------------------