├── error.js ├── .gitignore ├── validate.js ├── package.json ├── example.js ├── README.md └── index.js /error.js: -------------------------------------------------------------------------------- 1 | module.exports = function ConfigError(name, message) { 2 | this.message = message; 3 | this.name = name; 4 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pids 8 | *.pid 9 | *.seed 10 | *.pid.lock 11 | lib-cov 12 | coverage 13 | .nyc_output 14 | .grunt 15 | bower_components 16 | .lock-wscript 17 | build/Release 18 | node_modules/ 19 | jspm_packages/ 20 | typings/ 21 | .npm 22 | .eslintcache 23 | .node_repl_history 24 | *.tgz 25 | .yarn-integrity 26 | .env 27 | .DS_Store 28 | test.js -------------------------------------------------------------------------------- /validate.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | var ConfigError = require('./error') 3 | var regex = new RegExp('^[a-zA-Z0-9_.-]{3,255}$'); 4 | 5 | var validate = function (options, field) { 6 | if (!regex.test(options[field].tableName || '')) { 7 | throw new ConfigError('InvalidConfig', field + 8 | '.tableName must follow AWS naming rules (3-255 length, and only the following characters: a-z, A-Z, 0-9, _-.)') 9 | } 10 | return true 11 | } 12 | 13 | module.exports.config = function (options) { 14 | return validate(options, 'source') && validate(options, 'destination') // check both source and destination 15 | } 16 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "copy-dynamodb-table", 3 | "version": "2.2.1", 4 | "description": "Copy Dynamodb table to another in the same or different zone , It is 100% safe , and speed depends on your destination table user defined write provisioned throughput", 5 | "keywords": [ 6 | "dynamodb", 7 | "aws-dynamodb", 8 | "AWS", 9 | "copy-tables" 10 | ], 11 | "main": "index.js", 12 | "scripts": { 13 | "test": "echo \"Error: no test specified\" && exit 1" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "https://github.com/enGMzizo/copy-dynamodb-table" 18 | }, 19 | "author": "Ezzat @enGMzizo", 20 | "license": "ISC", 21 | "dependencies": { 22 | "aws-sdk": "^2.630.0", 23 | "readline": "^1.3.0" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | var copy = require('./index').copy 2 | 3 | var globalAWSConfig = { // AWS Configuration object http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#constructor-property 4 | accessKeyId: 'AKID', 5 | secretAccessKey: 'SECRET', 6 | region: 'eu-west-1' 7 | } 8 | 9 | var sourceAWSConfig = { 10 | accessKeyId: 'AKID', 11 | secretAccessKey: 'SECRET', 12 | region: 'eu-west-1' 13 | } 14 | 15 | var destinationAWSConfig = { 16 | accessKeyId: 'AKID', 17 | secretAccessKey: 'SECRET', 18 | region: 'us-west-2' // support cross zone copying 19 | } 20 | 21 | copy({ 22 | config: globalAWSConfig, 23 | source: { 24 | tableName: 'source_table_name', // required 25 | config: sourceAWSConfig // optional , leave blank to use globalAWSConfig 26 | }, 27 | destination: { 28 | tableName: 'destination_table_name', // required 29 | config: destinationAWSConfig // optional , leave blank to use globalAWSConfig 30 | }, 31 | log: true, // default false 32 | create: false, // create table if not exist 33 | schemaOnly: false, // make it true and it will copy schema only 34 | continuousBackups: true, // default 'copy', true will always enable backups, 'copy' copies the behaviour from the source and false does not enable them 35 | transformDataFn: function(item){ return item } // function to transform data 36 | }, 37 | function (err, result) { 38 | if (err) { 39 | console.log(err) 40 | } 41 | console.log(result) 42 | }) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Safe Copy Dynamodb Table 2 | =================== 3 | 4 | This module will allow you to copy data from one table to another using very simple API, Support cross zone copying and AWS config for each table ( source & destination ) and it can create the destination table using source table schema 5 | 6 | [![Dependencies](https://david-dm.org/enGMzizo/copy-dynamodb-table.png)](https://david-dm.org/enGMzizo/copy-dynamodb-table) [![NPM version](https://badge.fury.io/js/copy-dynamodb-table.png)](http://badge.fury.io/js/copy-dynamodb-table) 7 | 8 | 9 | ## Installation 10 | 11 | npm i copy-dynamodb-table 12 | 13 | ## Usage : 14 | 15 | ```js 16 | var copy = require('copy-dynamodb-table').copy 17 | 18 | copy({ 19 | source: { 20 | tableName: 'source_table_name', // required 21 | }, 22 | destination: { 23 | tableName: 'destination_table_name', // required 24 | }, 25 | log: true, // default false 26 | create : true, // create destination table if not exist 27 | schemaOnly : false, // if true it will copy schema only -- optional 28 | continuousBackups: true, // if true will enable point in time backups 29 | transform: function(item , index){ return item } // function to transform data 30 | }, 31 | function (err, result) { 32 | if (err) { 33 | console.log(err) 34 | } 35 | console.log(result) 36 | }) 37 | ``` 38 | ## Adding AWS Config : 39 | 40 | ```js 41 | var copy = require('copy-dynamodb-table').copy 42 | 43 | var globalAWSConfig = { // AWS Configuration object http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#constructor-property 44 | accessKeyId: 'AKID', 45 | secretAccessKey: 'SECRET', 46 | region: 'eu-west-1' 47 | } 48 | 49 | copy({ 50 | config: globalAWSConfig, // config for AWS 51 | source: { 52 | tableName: 'source_table_name', // required 53 | }, 54 | destination: { 55 | tableName: 'destination_table_name', // required 56 | }, 57 | log: true, // default false 58 | create : true // create destination table if not exist 59 | }, 60 | function (err, result) { 61 | if (err) { 62 | console.log(err) 63 | } 64 | console.log(result) 65 | }) 66 | ``` 67 | 68 | ## AWS Config for each table ( cross region ) : 69 | 70 | ```js 71 | var copy = require('copy-dynamodb-table').copy 72 | 73 | var globalAWSConfig = { // AWS Configuration object http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#constructor-property 74 | accessKeyId: 'AKID', 75 | secretAccessKey: 'SECRET', 76 | region: 'eu-west-1' 77 | } 78 | 79 | var sourceAWSConfig = { 80 | accessKeyId: 'AKID', 81 | secretAccessKey: 'SECRET', 82 | region: 'eu-west-1' 83 | } 84 | 85 | var destinationAWSConfig = { 86 | accessKeyId: 'AKID', 87 | secretAccessKey: 'SECRET', 88 | region: 'us-west-2' // support cross zone copying 89 | } 90 | 91 | copy({ 92 | config: globalAWSConfig, 93 | source: { 94 | tableName: 'source_table_name', // required 95 | config: sourceAWSConfig // optional , leave blank to use globalAWSConfig 96 | }, 97 | destination: { 98 | tableName: 'destination_table_name', // required 99 | config: destinationAWSConfig // optional , leave blank to use globalAWSConfig 100 | }, 101 | log: true,// default false 102 | create : true // create destination table if not exist 103 | }, 104 | function (err, result) { 105 | if (err) { 106 | console.log(err) 107 | } 108 | console.log(result) 109 | }) 110 | ``` 111 | 112 | ## Note : 113 | 114 | - If `source.config` or `destination.config` value is `undefined` , the module will use the `globalAWSConfig`. 115 | - If `globalAWSConfig` value is `undefined` the module will extact `AWS` config from environment variables. 116 | - Increase Write capacity for your dynamodb table temporarily until the copying is finished so you can get the highest copying speed 117 | 118 | ## Promise version : 119 | 120 | Use this if you want to copy using promises, or async / await . 121 | 122 | ```javascript 123 | function promiseCopy(data) { 124 | return new Promise((resolve, reject) => { 125 | copy(data, function (err, result) { 126 | if (err) { 127 | return reject(err) 128 | } 129 | resolve(result) 130 | }) 131 | }) 132 | } 133 | 134 | promiseCopy({ 135 | source: { 136 | tableName: 'source_table_name', // required 137 | }, 138 | destination: { 139 | tableName: 'destination_table_name', // required 140 | }, 141 | log: true, // default false 142 | create: true // create destination table if not exist 143 | }).then(function (results) { 144 | // do stuff 145 | }).catch(function (err) { 146 | //handle error 147 | }) 148 | ``` 149 | 150 | ## Use Case : 151 | With source table read capacity units = 100 & destination table write capacity units = 1000 , I managed to copy ~100,000 items from source to destination within ~175 seconds , with avarage item size of 4 KB. 152 | 153 | ## Contributors : 154 | 155 | - [@enGMzizo](https://twitter.com/enGMzizo) 156 | - [jazarja](https://github.com/jazarja) 157 | - [kevpmoore](https://github.com/kevpmoore) 158 | - [Floby](https://github.com/Floby) 159 | - [jermeo](https://github.com/jermeo) 160 | - [Simon Li](https://github.com/siutsin) 161 | - [Janusz Slota](https://github.com/nixilla) 162 | - [Kyle Watson](https://github.com/kylejwatson) 163 | 164 | ## License : 165 | 166 | [ISC](https://spdx.org/licenses/ISC) 167 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | var AWS = require('aws-sdk') 3 | var validate = require('./validate') 4 | var readline = require('readline') 5 | 6 | function copy(values, fn) { 7 | 8 | try { 9 | validate.config(values) 10 | } catch (err) { 11 | if (err) { 12 | return fn(err, { 13 | count: 0, 14 | status: 'FAIL' 15 | }) 16 | } 17 | } 18 | 19 | var options = { 20 | config: values.config, 21 | source: { 22 | tableName: values.source.tableName, 23 | dynamoClient: values.source.dynamoClient || new AWS.DynamoDB.DocumentClient(values.source.config || values.config), 24 | dynamodb: values.source.dynamodb || new AWS.DynamoDB(values.source.config || values.config), 25 | active: values.source.active 26 | }, 27 | destination: { 28 | tableName: values.destination.tableName, 29 | dynamoClient: values.destination.dynamoClient || new AWS.DynamoDB.DocumentClient(values.destination.config || values.config), 30 | dynamodb: values.destination.dynamodb || new AWS.DynamoDB(values.destination.config || values.config), 31 | active: values.destination.active, 32 | createTableStr: 'Creating Destination Table ' 33 | }, 34 | key: values.key, 35 | counter: values.counter || 0, 36 | retries: 0, 37 | data: {}, 38 | transform: values.transform, 39 | log: values.log, 40 | create: values.create, 41 | schemaOnly: values.schemaOnly, 42 | continuousBackups: values.continuousBackups 43 | } 44 | 45 | if (options.source.active && options.destination.active) { // both tables are active 46 | return startCopying(options, fn) 47 | } 48 | 49 | if (options.create) { // create table if not exist 50 | return options.source.dynamodb.describeTable({ TableName: options.source.tableName }, function (err, data) { 51 | if (err) { 52 | return fn(err, data) 53 | } 54 | 55 | options.source.active = data.Table.TableStatus === 'ACTIVE' 56 | data.Table.TableName = options.destination.tableName 57 | options.destination.dynamodb.createTable(clearTableSchema(data.Table), function (err) { 58 | if (err && err.code !== 'ResourceInUseException') { 59 | return fn(err, data) 60 | } 61 | waitForActive(options, fn) 62 | // wait for TableStatus to be ACTIVE 63 | }) 64 | }) 65 | } 66 | 67 | checkTables(options, function (err, data) { // check if source and destination table exist 68 | if (err) { 69 | return fn(err, data) 70 | } 71 | startCopying(options, fn) 72 | }) 73 | 74 | } 75 | 76 | function enableBackups(options, fn) { 77 | options.destination.dynamodb.updateContinuousBackups({ 78 | PointInTimeRecoverySpecification: { 79 | PointInTimeRecoveryEnabled: true 80 | }, 81 | TableName: options.destination.tableName 82 | }, fn) 83 | } 84 | 85 | function setContinuousBackups(options, fn) { 86 | options.source.dynamodb.describeContinuousBackups({ TableName: options.source.tableName }, function (err, data) { 87 | if (err) { 88 | return fn(err, data); 89 | } 90 | if (data.ContinuousBackupsDescription.ContinuousBackupsStatus === 'ENABLED') { 91 | return enableBackups(options, fn); 92 | } 93 | fn(null) 94 | }) 95 | 96 | } 97 | 98 | function clearTableSchema(table) { 99 | 100 | delete table.TableStatus; 101 | delete table.CreationDateTime; 102 | if (table.ProvisionedThroughput.ReadCapacityUnits === 0 && table.ProvisionedThroughput.WriteCapacityUnits === 0) { 103 | delete table.ProvisionedThroughput 104 | } 105 | else { 106 | delete table.ProvisionedThroughput.LastIncreaseDateTime; 107 | delete table.ProvisionedThroughput.LastDecreaseDateTime; 108 | delete table.ProvisionedThroughput.NumberOfDecreasesToday; 109 | } 110 | 111 | delete table.TableSizeBytes; 112 | delete table.ItemCount; 113 | delete table.TableArn; 114 | delete table.TableId; 115 | delete table.LatestStreamLabel; 116 | delete table.LatestStreamArn; 117 | 118 | if (table.LocalSecondaryIndexes && table.LocalSecondaryIndexes.length > 0) { 119 | for (var i = 0; i < table.LocalSecondaryIndexes.length; i++) { 120 | delete table.LocalSecondaryIndexes[i].IndexStatus; 121 | delete table.LocalSecondaryIndexes[i].IndexSizeBytes; 122 | delete table.LocalSecondaryIndexes[i].ItemCount; 123 | delete table.LocalSecondaryIndexes[i].IndexArn; 124 | delete table.LocalSecondaryIndexes[i].LatestStreamLabel; 125 | delete table.LocalSecondaryIndexes[i].LatestStreamArn; 126 | } 127 | } 128 | 129 | 130 | if (table.GlobalSecondaryIndexes && table.GlobalSecondaryIndexes.length > 0) { 131 | for (var j = 0; j < table.GlobalSecondaryIndexes.length; j++) { 132 | delete table.GlobalSecondaryIndexes[j].IndexStatus; 133 | if (table.GlobalSecondaryIndexes[j].ProvisionedThroughput.ReadCapacityUnits === 0 && table.GlobalSecondaryIndexes[j].ProvisionedThroughput.WriteCapacityUnits === 0) { 134 | delete table.GlobalSecondaryIndexes[j].ProvisionedThroughput 135 | } 136 | else { 137 | delete table.GlobalSecondaryIndexes[j].ProvisionedThroughput.LastIncreaseDateTime; 138 | delete table.GlobalSecondaryIndexes[j].ProvisionedThroughput.LastDecreaseDateTime; 139 | delete table.GlobalSecondaryIndexes[j].ProvisionedThroughput.NumberOfDecreasesToday; 140 | } 141 | delete table.GlobalSecondaryIndexes[j].IndexSizeBytes; 142 | delete table.GlobalSecondaryIndexes[j].ItemCount; 143 | delete table.GlobalSecondaryIndexes[j].IndexArn; 144 | delete table.GlobalSecondaryIndexes[j].LatestStreamLabel; 145 | delete table.GlobalSecondaryIndexes[j].LatestStreamArn; 146 | } 147 | } 148 | 149 | if (table.SSEDescription) { 150 | table.SSESpecification = { 151 | Enabled: (table.SSEDescription.Status === 'ENABLED' || table.SSEDescription.Status === 'ENABLING'), 152 | }; 153 | delete table.SSEDescription; 154 | } 155 | 156 | if (table.BillingModeSummary) { 157 | table.BillingMode = table.BillingModeSummary.BillingMode 158 | } 159 | delete table.BillingModeSummary; 160 | 161 | return table; 162 | } 163 | 164 | 165 | function checkTables(options, fn) { 166 | options.source.dynamodb.describeTable({ TableName: options.source.tableName }, function (err, sourceData) { 167 | if (err) { 168 | return fn(err, sourceData) 169 | } 170 | if (sourceData.Table.TableStatus !== 'ACTIVE') { 171 | return fn(new Error('Source table not active'), null) 172 | } 173 | options.source.active = true 174 | options.destination.dynamodb.describeTable({ TableName: options.destination.tableName }, function (err, destData) { 175 | if (err) { 176 | return fn(err, destData) 177 | } 178 | if (destData.Table.TableStatus !== 'ACTIVE') { 179 | return fn(new Error('Destination table not active'), null) 180 | } 181 | options.destination.active = true 182 | fn(null) 183 | }) 184 | }) 185 | } 186 | 187 | function waitForActive(options, fn) { 188 | setTimeout(function () { 189 | options.destination.dynamodb.describeTable({ TableName: options.destination.tableName }, function (err, data) { 190 | if (err) { 191 | return fn(err, data) 192 | } 193 | if (options.log) { 194 | options.destination.createTableStr += '.' 195 | readline.clearLine(process.stdout) 196 | readline.cursorTo(process.stdout, 0) 197 | process.stdout.write(options.destination.createTableStr) 198 | } 199 | if (data.Table.TableStatus !== 'ACTIVE') { // wait for active 200 | return waitForActive(options, fn) 201 | } 202 | options.create = false 203 | options.destination.active = true 204 | if (options.schemaOnly) { // schema only copied 205 | return fn(err, { 206 | count: options.counter, 207 | schemaOnly: true, 208 | status: 'SUCCESS' 209 | }) 210 | } 211 | if (options.continuousBackups) { // copy backup options 212 | return setContinuousBackups(options, function (err) { 213 | if (err) { 214 | return fn(err, data) 215 | } 216 | startCopying(options, fn); 217 | }) 218 | } 219 | startCopying(options, fn); 220 | }) 221 | }, 1000) // check every second 222 | } 223 | 224 | function startCopying(options, fn) { 225 | getItems(options, function (err, data) { 226 | if (err) { 227 | return fn(err) 228 | } 229 | options.data = data 230 | options.key = data.LastEvaluatedKey 231 | putItems(options, function (err) { 232 | if (err) { 233 | return fn(err) 234 | } 235 | 236 | if (options.log) { 237 | readline.clearLine(process.stdout) 238 | readline.cursorTo(process.stdout, 0) 239 | process.stdout.write('Copied ' + options.counter + ' items') 240 | } 241 | 242 | if (options.key === undefined) { 243 | return fn(err, { 244 | count: options.counter, 245 | status: 'SUCCESS' 246 | }) 247 | } 248 | copy(options, fn) 249 | }) 250 | }) 251 | } 252 | 253 | function getItems(options, fn) { 254 | scan(options, function (err, data) { 255 | if (err) { 256 | return fn(err, data) 257 | } 258 | fn(err, mapItems(options, data)) 259 | }) 260 | } 261 | 262 | 263 | function scan(options, fn) { 264 | options.source.dynamoClient.scan({ 265 | TableName: options.source.tableName, 266 | Limit: 25, 267 | ExclusiveStartKey: options.key 268 | }, fn) 269 | } 270 | 271 | function mapItems(options, data) { 272 | data.Items = data.Items.map(function (item, index) { 273 | return { 274 | PutRequest: { 275 | Item: !!options.transform ? options.transform(item, index) : item 276 | } 277 | } 278 | }) 279 | return data 280 | } 281 | 282 | function putItems(options, fn) { 283 | if (!options.data.Items || options.data.Items.length === 0) { 284 | return fn(null, options) 285 | } 286 | var batchWriteItems = {} 287 | batchWriteItems.RequestItems = {} 288 | batchWriteItems.RequestItems[options.destination.tableName] = options.data.Items 289 | options.destination.dynamoClient.batchWrite(batchWriteItems, function (err, data) { 290 | if (err) { 291 | return fn(err, data) 292 | } 293 | var unprocessedItems = data.UnprocessedItems[options.destination.tableName] 294 | if (unprocessedItems !== undefined) { 295 | 296 | options.retries++ 297 | options.counter += (options.data.Items.length - unprocessedItems.length) 298 | 299 | options.data = { 300 | Items: unprocessedItems 301 | } 302 | return setTimeout(function () { 303 | putItems(options, fn) 304 | }, 2 * options.retries * 100) // from aws http://docs.aws.amazon.com/general/latest/gr/api-retries.html 305 | 306 | } 307 | options.retries = 0 308 | options.counter += options.data.Items.length 309 | fn(err, options) 310 | }) 311 | } 312 | 313 | module.exports.copy = copy 314 | --------------------------------------------------------------------------------