├── .github └── workflows │ └── codeql-analysis.yml ├── .gitignore ├── .npmignore ├── LICENSE ├── README.md ├── cli.js ├── examples ├── localReplica.js └── statePersistence.js ├── index.js ├── package-lock.json ├── package.json └── test.js /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '32 22 * * 2' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | 28 | strategy: 29 | fail-fast: false 30 | matrix: 31 | language: [ 'javascript' ] 32 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 33 | # Learn more: 34 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 35 | 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@v2 39 | 40 | # Initializes the CodeQL tools for scanning. 41 | - name: Initialize CodeQL 42 | uses: github/codeql-action/init@v1 43 | with: 44 | languages: ${{ matrix.language }} 45 | # If you wish to specify custom queries, you can do so here or in a config file. 46 | # By default, queries listed here will override any specified in a config file. 47 | # Prefix the list here with "+" to use these queries and those in the config file. 48 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 49 | 50 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 51 | # If this step fails, then you should remove it and run the build manually (see below) 52 | - name: Autobuild 53 | uses: github/codeql-action/autobuild@v1 54 | 55 | # ℹ️ Command-line programs to run using the OS shell. 56 | # 📚 https://git.io/JvXDl 57 | 58 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 59 | # and modify them (or add more) to build your code if your project 60 | # uses a compiled language 61 | 62 | #- run: | 63 | # make bootstrap 64 | # make release 65 | 66 | - name: Perform CodeQL Analysis 67 | uses: github/codeql-action/analyze@v1 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Dependency directory 17 | # Commenting this out is preferred by some people, see 18 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 19 | node_modules 20 | 21 | # Mac stuff 22 | .DS_Store 23 | 24 | # rc config files 25 | .*rc 26 | 27 | # see https://github.com/substack/browserify-handbook#organizing-modules 28 | node_modules/* 29 | !node_modules/app 30 | 31 | index2.js 32 | test2.js 33 | shardState.json 34 | oldShardState.json 35 | playground -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Dependency directory 17 | # Commenting this out is preferred by some people, see 18 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 19 | node_modules 20 | 21 | # Mac stuff 22 | .DS_Store 23 | 24 | # rc config files 25 | .*rc 26 | 27 | # see https://github.com/substack/browserify-handbook#organizing-modules 28 | node_modules/* 29 | !node_modules/app 30 | 31 | index2.js 32 | test2.js 33 | shardState.json 34 | oldShardState.json 35 | playground 36 | 37 | .github -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 ironSource 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DynamoDBStream 2 | 3 | A wrapper around low level aws sdk that makes it easy to consume a dynamodb-stream, even in a browser. 4 | 5 | _update: serious overhaul with this [commit](https://github.com/ironSource/node-dynamodb-stream/commit/a418f5b4fc8e3948f279c6a2e57974051025c13c) and a few smaller ones after. Major version is bumped to 1.x.x_ 6 | 7 | ### Example: Replicating small tables 8 | 9 | fetchStreamState() should be invoked whenever the consumer wishes to get the updates. 10 | 11 | When a consumer needs to maintain a replica of the table data, fetchStreamState() is invoked on regular intervals. 12 | 13 | The current best practice for replication is to manage the state of the stream as it relates to the consumer in a separate dynamodb table (shard iterators/sequence numbers etc), so if a failure occurs, that consumer can get back to the point he was in the stream. However for small or even medium tables this is not necessary. One can simply reread the entire table on startup. 14 | 15 | This different approach make things more "stateless" and slightly simpler (in my view): 16 | 17 | - call fetchStreamState() first, one may safely disregard any events that are emitted at this stage. Under the hood DynamoDBStream uses ```ShardIteratorType: LATEST``` to get shard iterators for all the current shards of the stream. These iterators act as a "bookmark" in the stream. 18 | - Obtain an initial copy of the table's data (via a dynamodb scan api call for example) and store it locally 19 | - call fetchStreamState() again, at this point some of the events might already be included in the initial local copy of the data and some won't. Depending on the data structure thats houses the local copy of data, some filtering might be needed. 20 | - start polling on fetchStreamState() and blindly mutate the local copy according to the updates 21 | 22 | Wrapping the initial data scan with fetchStreamState() calls insures that no changes will be missed. At worst, the second call might yield some duplicates. 23 | 24 | ```js 25 | const DynamoDBStream = require('dynamodb-stream') 26 | const { DynamoDB } = require('@aws-sdk/client-dynamodb') 27 | const { DynamoDBStreams } = require('@aws-sdk/client-dynamodb-streams') 28 | const { unmarshall } = require('@aws-sdk/util-dynamodb') 29 | 30 | const STREAM_ARN = 'your stream ARN' 31 | const TABLE_NAME = 'testDynamoDBStream' 32 | 33 | async function main() { 34 | 35 | // table primary key is "pk" 36 | 37 | const ddb = new DynamoDB() 38 | const ddbStream = new DynamoDBStream( 39 | new DynamoDBStreams(), 40 | STREAM_ARN, 41 | unmarshall 42 | ) 43 | 44 | const localState = new Map() 45 | await ddbStream.fetchStreamState() 46 | const { Items } = await ddb.scan({ TableName: TABLE_NAME }) 47 | Items.map(unmarshall).forEach(item => localState.set(item.pk, item)) 48 | 49 | // parse results and store in local state 50 | const watchStream = () => { 51 | console.log(localState) 52 | setTimeout(() => ddbStream.fetchStreamState().then(watchStream), 10 * 1000) 53 | } 54 | 55 | watchStream() 56 | 57 | ddbStream.on('insert record', (data, keys) => { 58 | localState.set(data.pk, data) 59 | }) 60 | 61 | ddbStream.on('remove record', (data, keys) => { 62 | localState.remove(data.pk) 63 | }) 64 | 65 | ddbStream.on('modify record', (newData, oldData, keys) => { 66 | localState.set(newData.pk, newData) 67 | }) 68 | 69 | ddbStream.on('new shards', (shardIds) => {}) 70 | ddbStream.on('remove shards', (shardIds) => {}) 71 | } 72 | 73 | main() 74 | ``` 75 | 76 | ### Example: shards / iterator persistence 77 | 78 | If your program crash and you want to pick up where you left off then `setShardsState()` and `getShardState()` are here for the rescue (though, I haven't tested them yet but they should work... :) ) 79 | 80 | ```js 81 | const DynamoDBStream = require('dynamodb-stream') 82 | const { DynamoDBStreams } = require('@aws-sdk/client-dynamodb-streams') 83 | const { unmarshall } = require('@aws-sdk/util-dynamodb') 84 | const fs = require('fs').promises 85 | 86 | const STREAM_ARN = 'your stream ARN' 87 | const FILE = 'shardState.json' 88 | 89 | async function main() { 90 | const ddbStream = new DynamoDBStream( 91 | new DynamoDBStreams(), 92 | STREAM_ARN, 93 | unmarshall 94 | ) 95 | 96 | // update the state so it will pick up from where it left last time 97 | // remember this has a limit of 24 hours or something along these lines 98 | // https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Streams.html 99 | ddbStream.setShardState(await loadShardState()) 100 | 101 | const fetchStreamState = () => { 102 | setTimeout(async () => { 103 | await ddbStream.fetchStreamState() 104 | const shardState = ddbStream.getShardState() 105 | await fs.writeFile(FILE, JSON.stringify(shardState)) 106 | fetchStreamState() 107 | }, 1000 * 20) 108 | } 109 | 110 | fetchStreamState() 111 | } 112 | 113 | async function loadShardState() { 114 | try { 115 | return JSON.parse(await fs.readFile(FILE, 'utf8')) 116 | } catch (e) { 117 | if (e.code === 'ENOENT') return {} 118 | throw e 119 | } 120 | } 121 | 122 | main() 123 | ``` 124 | 125 | #### TODO 126 | - make sure the aggregation of records is in order - the metadata from the stream might be helpful (order by sequence number?) 127 | - what about sequence numbers and other types of iterators? (TRIM_HORIZON | LATEST | AT_SEQUENCE_NUMBER | AFTER_SEQUENCE_NUMBE) 128 | 129 | #### Wishlist to DynamoDB team: 130 | 1. expose push interface so one won't need to poll the stream api 131 | 2. obtain a sequence number from a dynamodb api scan operation 132 | 133 | [MIT](http://opensource.org/licenses/MIT) © ironSource ltd. 134 | -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | module.exports = function () { 4 | 5 | } 6 | 7 | if (require.main === module) { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /examples/localReplica.js: -------------------------------------------------------------------------------- 1 | const DynamoDBStream = require('dynamodb-stream') 2 | const { DynamoDB } = require('@aws-sdk/client-dynamodb') 3 | const { DynamoDBStreams } = require('@aws-sdk/client-dynamodb-streams') 4 | const { unmarshall } = require('@aws-sdk/util-dynamodb') 5 | 6 | const STREAM_ARN = 'your stream ARN' 7 | const TABLE_NAME = 'your table name' 8 | 9 | async function main() { 10 | 11 | // table primary key is "pk" 12 | 13 | const ddb = new DynamoDB() 14 | const ddbStream = new DynamoDBStream( 15 | new DynamoDBStreams(), 16 | STREAM_ARN, 17 | unmarshall 18 | ) 19 | 20 | const localState = new Map() 21 | await ddbStream.fetchStreamState() 22 | const { Items } = await ddb.scan({ TableName: TABLE_NAME }) 23 | Items.map(unmarshall).forEach(item => localState.set(item.pk, item)) 24 | 25 | // parse results and store in local state 26 | const watchStream = async () => { 27 | await ddbStream.fetchStreamState() 28 | setTimeout(watchStream, 10 * 1000) 29 | } 30 | 31 | watchStream() 32 | 33 | ddbStream.on('insert record', (data, keys) => { 34 | localState.set(data.pk, data) 35 | }) 36 | 37 | ddbStream.on('remove record', (data, keys) => { 38 | localState.remove(data.pk) 39 | }) 40 | 41 | ddbStream.on('modify record', (newData, oldData, keys) => { 42 | localState.set(newData.pk, newData) 43 | }) 44 | 45 | ddbStream.on('new shards', (shardIds) => {}) 46 | ddbStream.on('remove shards', (shardIds) => {}) 47 | } 48 | 49 | main() -------------------------------------------------------------------------------- /examples/statePersistence.js: -------------------------------------------------------------------------------- 1 | const DynamoDBStream = require('dynamodb-stream') 2 | const { DynamoDBStreams } = require('@aws-sdk/client-dynamodb-streams') 3 | const { unmarshall } = require('@aws-sdk/util-dynamodb') 4 | const fs = require('fs').promises 5 | 6 | const STREAM_ARN = 'your stream arn' 7 | const FILE = 'shardState.json' 8 | 9 | async function main() { 10 | const ddbStream = new DynamoDBStream( 11 | new DynamoDBStreams(), 12 | STREAM_ARN, 13 | unmarshall 14 | ) 15 | 16 | // update the state so it will pick up from where it left last time 17 | // remember this has a limit of 24 hours or something along these lines 18 | // https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Streams.html 19 | ddbStream.setShardState(await loadShardState()) 20 | 21 | const fetchStreamState = async () => { 22 | await ddbStream.fetchStreamState() 23 | const shardState = ddbStream.getShardState() 24 | await fs.writeFile(FILE, JSON.stringify(shardState)) 25 | setTimeout(fetchStreamState, 1000 * 20) 26 | } 27 | 28 | fetchStreamState() 29 | } 30 | 31 | async function loadShardState() { 32 | try { 33 | return JSON.parse(await fs.readFile(FILE, 'utf8')) 34 | } catch (e) { 35 | if (e.code === 'ENOENT') return {} 36 | throw e 37 | } 38 | } 39 | 40 | main() -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const debug = require('debug')('DynamoDBStream') 2 | const map = require('@kessler/async-map-limit') 3 | const { EventEmitter } = require('events') 4 | 5 | class DynamoDBStream extends EventEmitter { 6 | 7 | /** 8 | * @param {object} ddbStreams - an instance of DynamoDBStreams 9 | * @param {string} streamArn - the arn of the stream we're consuming 10 | * @param {function} unmarshall - directly from 11 | * ```js 12 | * const { unmarshall } = require('@aws-sdk/util-dynamodb') 13 | * ``` 14 | * if not provided then records will be returned using low level api/shape 15 | */ 16 | constructor(ddbStreams, streamArn, unmarshall) { 17 | super() 18 | if (!typeof ddbStreams === 'object') { 19 | throw new Error('missing or invalid ddbStreams argument') 20 | } 21 | 22 | if (!typeof streamArn === 'string') { 23 | throw new Error('missing or invalid streamArn argument') 24 | } 25 | 26 | this._ddbStreams = ddbStreams 27 | this._streamArn = streamArn 28 | this._shards = new Map() 29 | this._unmarshall = unmarshall 30 | } 31 | 32 | /** 33 | * this will update the stream, shards and records included 34 | * 35 | */ 36 | async fetchStreamState() { 37 | debug('fetchStreamState') 38 | 39 | await this.fetchStreamShards() 40 | await this.fetchStreamRecords() 41 | } 42 | 43 | /** 44 | * update the shard state of the stream 45 | * this will emit new shards / remove shards events 46 | */ 47 | async fetchStreamShards() { 48 | debug('fetchStreamShards') 49 | 50 | this._trimShards() 51 | 52 | const params = { 53 | StreamArn: this._streamArn 54 | } 55 | 56 | const { StreamDescription } = await this._ddbStreams.describeStream(params) 57 | 58 | const shards = StreamDescription.Shards 59 | const newShardIds = [] 60 | 61 | // collect all the new shards of this stream 62 | for (const newShardEntry of shards) { 63 | const existingShardEntry = this._shards.get(newShardEntry.ShardId) 64 | 65 | if (!existingShardEntry) { 66 | this._shards.set(newShardEntry.ShardId, { 67 | shardId: newShardEntry.ShardId 68 | }) 69 | 70 | newShardIds.push(newShardEntry.ShardId) 71 | } 72 | } 73 | 74 | if (newShardIds.length > 0) { 75 | debug('Added %d new shards', newShardIds.length) 76 | this._emitNewShardsEvent(newShardIds) 77 | } 78 | } 79 | 80 | /** 81 | * get latest updates from the underlying stream 82 | * 83 | */ 84 | async fetchStreamRecords() { 85 | debug('fetchStreamRecords') 86 | 87 | if (this._shards.size === 0) { 88 | debug('no shards found, this is ok') 89 | return 90 | } 91 | 92 | await this._getShardIterators() 93 | const records = await this._getRecords() 94 | 95 | debug('fetchStreamRecords', records) 96 | 97 | this._trimShards() 98 | this._emitRecordEvents(records) 99 | 100 | return records 101 | } 102 | 103 | /** 104 | * get a COPY of the current/internal shard state. 105 | * this, in conjuction with setShardState is used to 106 | * persist the stream state locally. 107 | * 108 | * @returns {object} 109 | */ 110 | getShardState() { 111 | const state = {} 112 | for (const [shardId, shardData] of this._shards) { 113 | state[shardId] = { ...shardData } 114 | } 115 | return state 116 | } 117 | 118 | /** 119 | * @param {object} shards 120 | */ 121 | setShardState(shards) { 122 | 123 | this._shards = new Map() 124 | for (const [shardId, shardData] of Object.entries(shards)) { 125 | this._shards.set(shardId, shardData) 126 | } 127 | } 128 | 129 | _getShardIterators() { 130 | debug('_getShardIterators') 131 | return map(this._shards.values(), shardData => this._getShardIterator(shardData), 10) 132 | } 133 | 134 | async _getShardIterator(shardData) { 135 | debug('_getShardIterator') 136 | debug(shardData) 137 | 138 | // no need to get an iterator if this shard already has NextShardIterator 139 | if (shardData.nextShardIterator) { 140 | debug('shard %s already has an iterator, skipping', shardData.shardId) 141 | return 142 | } 143 | 144 | const params = { 145 | ShardId: shardData.shardId, 146 | ShardIteratorType: 'LATEST', 147 | StreamArn: this._streamArn 148 | } 149 | 150 | const { ShardIterator } = await this._ddbStreams.getShardIterator(params) 151 | shardData.nextShardIterator = ShardIterator 152 | } 153 | 154 | async _getRecords() { 155 | debug('_getRecords') 156 | 157 | const results = await map(this._shards.values(), shardData => this._getShardRecords(shardData), 10) 158 | 159 | return results.flat() 160 | } 161 | 162 | async _getShardRecords(shardData) { 163 | debug('_getShardRecords') 164 | 165 | const params = { ShardIterator: shardData.nextShardIterator } 166 | 167 | try { 168 | const { Records, NextShardIterator } = await this._ddbStreams.getRecords(params) 169 | if (NextShardIterator) { 170 | shardData.nextShardIterator = NextShardIterator 171 | } else { 172 | shardData.nextShardIterator = null 173 | } 174 | 175 | return Records 176 | } catch (e) { 177 | if (e.name === 'ExpiredIteratorException') { 178 | debug('_getShardRecords expired iterator', shardData) 179 | shardData.nextShardIterator = null 180 | } else { 181 | throw e 182 | } 183 | } 184 | } 185 | 186 | _trimShards() { 187 | debug('_trimShards') 188 | 189 | const removedShards = [] 190 | 191 | for (const [shardId, shardData] of this._shards) { 192 | if (shardData.nextShardIterator === null) { 193 | debug('deleting shard %s', shardId) 194 | this._shards.delete(shardId) 195 | removedShards.push(shardId) 196 | } 197 | } 198 | 199 | if (removedShards.length > 0) { 200 | this._emitRemoveShardsEvent(removedShards) 201 | } 202 | } 203 | 204 | /** 205 | * you may override in subclasses to change record transformation behavior 206 | * for records emitted during _emitRecordEvents() 207 | */ 208 | _transformRecord(record) { 209 | if (this._unmarshall && record) { 210 | return this._unmarshall(record) 211 | } 212 | } 213 | 214 | _emitRecordEvents(events) { 215 | debug('_emitRecordEvents') 216 | 217 | for (const event of events) { 218 | const keys = this._transformRecord(event.dynamodb.Keys) 219 | const newRecord = this._transformRecord(event.dynamodb.NewImage) 220 | const oldRecord = this._transformRecord(event.dynamodb.OldImage) 221 | 222 | switch (event.eventName) { 223 | case 'INSERT': 224 | this.emit('insert record', newRecord, keys) 225 | break 226 | 227 | case 'MODIFY': 228 | this.emit('modify record', newRecord, oldRecord, keys) 229 | break 230 | 231 | case 'REMOVE': 232 | this.emit('remove record', oldRecord, keys) 233 | break 234 | 235 | default: 236 | throw new Error(`unknown dynamodb event ${event.eventName}`) 237 | } 238 | } 239 | } 240 | 241 | _emitRemoveShardsEvent(shardIds) { 242 | this.emit('remove shards', shardIds) 243 | } 244 | 245 | 246 | _emitNewShardsEvent(shardIds) { 247 | this.emit('new shards', shardIds) 248 | } 249 | } 250 | 251 | module.exports = DynamoDBStream -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynamodb-stream", 3 | "version": "1.1.4", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "env AWS_REGION=us-east-1 AWS_PROFILE=test DEBUG=DynamoDBStream* npx ava test.js" 8 | }, 9 | "keywords": [ 10 | "stream", 11 | "amazon", 12 | "aws", 13 | "dynamodb", 14 | "replication" 15 | ], 16 | "dependencies": { 17 | "@kessler/async-map-limit": "^3.0.0", 18 | "debug": "^2.2.0" 19 | }, 20 | "author": "Yaniv Kessler", 21 | "bugs": { 22 | "url": "https://github.com/ironSource/node-dynamodb-stream/issues" 23 | }, 24 | "repository": { 25 | "type": "git", 26 | "url": "https://github.com/ironSource/node-dynamodb-stream" 27 | }, 28 | "homepage": "https://github.com/ironSource/node-dynamodb-stream", 29 | "license": "MIT", 30 | "devDependencies": { 31 | "@aws-sdk/client-dynamodb": "^3.4.1", 32 | "@aws-sdk/client-dynamodb-streams": "^3.4.1", 33 | "@aws-sdk/util-dynamodb": "^3.4.1", 34 | "ava": "^3.15.0", 35 | "ulid": "^2.3.0" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | const DynamoDBStream = require('./index') 3 | const { DynamoDB } = require('@aws-sdk/client-dynamodb') 4 | const { DynamoDBStreams } = require('@aws-sdk/client-dynamodb-streams') 5 | const { unmarshall } = require('@aws-sdk/util-dynamodb') 6 | const { ulid } = require('ulid') 7 | const debug = require('debug')('DynamoDBStream:test') 8 | const ddbStreams = new DynamoDBStreams() 9 | const ddb = new DynamoDB() 10 | 11 | const TABLE_NAME = 'testDynamoDBStream' 12 | 13 | test('reports the correct stream of changes', async t => { 14 | const { eventLog, ddbStream } = t.context 15 | const pk = ulid() 16 | const pkA = ulid() 17 | const pkB = ulid() 18 | 19 | await ddbStream.fetchStreamState() 20 | await putItem({ pk, data: '1' }) 21 | await ddbStream.fetchStreamState() 22 | await putItem({ pk, data: '2' }) 23 | await ddbStream.fetchStreamState() 24 | await putItem({ pk: pkA, data: '2' }) 25 | await putItem({ pk: pkB, data: '2' }) 26 | await ddbStream.fetchStreamState() 27 | await deleteItem(pkA) 28 | await ddbStream.fetchStreamState() 29 | 30 | t.deepEqual(eventLog, [ 31 | { eventName: 'insert record', record: { pk, data: '1' } }, 32 | { 33 | eventName: 'modify record', 34 | newRecord: { pk, data: '2' }, 35 | oldRecord: { pk, data: '1' } 36 | }, 37 | { eventName: 'insert record', record: { pk: pkA, data: '2' } }, 38 | { eventName: 'insert record', record: { pk: pkB, data: '2' } }, 39 | { eventName: 'remove record', record: { pk: pkA, data: '2' } } 40 | ]) 41 | }) 42 | 43 | // this test will only work if you have a proper shardState set a side 44 | test.skip('ExpiredIteratorException', async t => { 45 | const shardState = require('./oldShardState.json') 46 | const { ddbStream } = t.context 47 | ddbStream.setShardState(shardState) 48 | await t.notThrowsAsync(ddbStream.fetchStreamState()) 49 | }) 50 | 51 | test.beforeEach(async t => { 52 | 53 | t.context = { 54 | eventLog: [] 55 | } 56 | 57 | await createTable() 58 | const arn = await findStreamArn() 59 | const ddbStream = t.context.ddbStream = new DynamoDBStream(ddbStreams, arn, unmarshall) 60 | 61 | ddbStream.on('insert record', (record) => { 62 | t.context.eventLog.push({ eventName: 'insert record', record }) 63 | }) 64 | 65 | ddbStream.on('modify record', (newRecord, oldRecord) => { 66 | t.context.eventLog.push({ 67 | eventName: 'modify record', 68 | newRecord, 69 | oldRecord 70 | }) 71 | }) 72 | 73 | ddbStream.on('remove record', (record) => { 74 | t.context.eventLog.push({ eventName: 'remove record', record }) 75 | }) 76 | 77 | ddbStream.on('new shards', (newShards) => { 78 | t.context.shards = newShards 79 | }) 80 | }) 81 | 82 | /** 83 | * create the test table and wait for it to become active 84 | * 85 | */ 86 | async function createTable() { 87 | debug('creating table...') 88 | 89 | const params = { 90 | TableName: TABLE_NAME, 91 | KeySchema: [{ 92 | AttributeName: 'pk', 93 | KeyType: 'HASH', 94 | }], 95 | AttributeDefinitions: [{ 96 | AttributeName: 'pk', 97 | AttributeType: 'S', // (S | N | B) for string, number, binary 98 | }], 99 | ProvisionedThroughput: { // required provisioned throughput for the table 100 | ReadCapacityUnits: 1, 101 | WriteCapacityUnits: 1, 102 | }, 103 | StreamSpecification: { 104 | StreamEnabled: true, 105 | StreamViewType: 'NEW_AND_OLD_IMAGES' 106 | } 107 | } 108 | try { 109 | await ddb.createTable(params) 110 | debug('table created.') 111 | await waitForTable(true) 112 | } catch (e) { 113 | if (!isTableExistError(e)) { 114 | throw e 115 | } 116 | 117 | debug('table already exists, skipping creation.') 118 | } 119 | } 120 | 121 | async function findStreamArn() { 122 | debug('finding the right stream arn') 123 | const { Streams } = await ddbStreams.listStreams({ TableName: TABLE_NAME }) 124 | 125 | debug('found %d streams', Streams.length) 126 | 127 | const stream = Streams.filter(item => item.TableName === TABLE_NAME)[0] 128 | 129 | debug(stream) 130 | 131 | if (!stream) { 132 | throw new Error('cannot find stream arn') 133 | } 134 | 135 | debug('stream arn for table %s was found', TABLE_NAME) 136 | return stream.StreamArn 137 | } 138 | 139 | /** 140 | * delete the test table and wait for its disappearance 141 | * 142 | */ 143 | async function deleteTable() { 144 | const params = { 145 | TableName: TABLE_NAME 146 | } 147 | debug('deleting table %s', TABLE_NAME) 148 | await ddb.deleteTable(params) 149 | await waitForTable(false) 150 | } 151 | 152 | /** 153 | * wait for a table's state (exist/dont exist) 154 | * if the table is already in that state this function should return quickly 155 | * 156 | */ 157 | async function waitForTable(exists) { 158 | debug('waiting for table %s...', exists ? 'to become available' : 'deletion') 159 | 160 | // Waits for table to become ACTIVE. 161 | // Useful for waiting for table operations like CreateTable to complete. 162 | const params = { 163 | TableName: TABLE_NAME 164 | } 165 | 166 | // Supports 'tableExists' and 'tableNotExists' 167 | await ddb.waitFor(exists ? 'tableExists' : 'tableNotExists', params) 168 | debug('table %s.', exists ? 'available' : 'deleted') 169 | } 170 | 171 | function putItem(data) { 172 | const params = { 173 | TableName: TABLE_NAME, 174 | Item: { 175 | pk: { 176 | S: data.pk 177 | }, 178 | data: { 179 | S: data.data 180 | } 181 | } 182 | } 183 | 184 | debug('putting item %o', params) 185 | 186 | return ddb.putItem(params) 187 | } 188 | 189 | function deleteItem(pk) { 190 | const params = { 191 | TableName: TABLE_NAME, 192 | Key: { pk: { S: pk } } 193 | } 194 | 195 | debug('deleting item %o', params) 196 | 197 | return ddb.deleteItem(params) 198 | } 199 | 200 | function isTableExistError(err) { 201 | return err && err.name === 'ResourceInUseException' && err.message && err.message.indexOf('Table already exists') > -1 202 | } --------------------------------------------------------------------------------