├── .eslintignore ├── .eslintrc.js ├── .github ├── CODEOWNERS └── workflows │ └── deployment.yml ├── .gitignore ├── .npmignore ├── README.md ├── cli ├── Components │ ├── App.js │ ├── BigTextErrorBoundary.js │ ├── CommandForm.js │ ├── Menu.js │ └── Title.js ├── cli-commands.js └── index.js ├── docs └── zero_downtime_data_transformation_process.md ├── examples └── serverless-localstack │ ├── .env.local │ ├── EXERCISE-prepare-data │ └── v4_using_preparation_data.js │ ├── README.md │ ├── data-transformations │ └── UsersExample │ │ ├── v1_insert_users.js │ │ ├── v2_add_random_number_field.js │ │ ├── v3_split_fullname.js │ │ └── v4_using_preparation_data.js │ ├── docker-compose.yml │ ├── package-lock.json │ ├── package.json │ ├── serverless.yml │ └── usersData.js ├── index.js ├── license ├── package-lock.json ├── package.json ├── serverless-plugin ├── commands.js ├── sls-plugin.js └── sls-resources-parser.js └── src ├── clients ├── dynamodb.js └── index.js ├── command-handlers ├── down.js ├── history.js ├── index.js ├── init.js ├── prepare.js └── up.js ├── config ├── commands.js ├── constants.js └── transformation-template-file.js ├── data-transformation-script-explorer.js ├── services ├── dynamodb │ ├── errorHandlers.js │ ├── index.js │ └── transformations-executions-manager.js └── s3.js └── utils ├── batchDeleteItems.js ├── batchWriteItems.js ├── deleteItems.js ├── getItems.js ├── getTableKeySchema.js ├── index.js ├── insertItems.js ├── responseUtils.js ├── transactWriteItems.js └── transformItems.js /.eslintignore: -------------------------------------------------------------------------------- 1 | **/dist 2 | **/node_modules 3 | cli/** 4 | local -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: [ 3 | 'eslint:recommended', 4 | 'eslint-config-airbnb-base', 5 | ], 6 | 7 | env: { 8 | jest: true, 9 | node: true, 10 | }, 11 | 12 | globals: {}, 13 | 14 | parserOptions: { 15 | ecmaVersion: 'latest', 16 | sourceType: 'module', 17 | ecmaFeatures: { 18 | jsx: true, 19 | }, 20 | }, 21 | 22 | rules: { 23 | 'no-await-in-loop': 'off', 24 | 'lines-between-class-members': 'off', 25 | 'no-console': 'off', 26 | 'consistent-return': 'off', 27 | 'arrow-body-style': 'off', 28 | 'import/no-dynamic-require': 'off', 29 | 'global-require': 'off', 30 | 'no-restricted-syntax': 'off', 31 | 'max-len': [ 32 | 'error', 33 | { 34 | code: 120, 35 | ignoreComments: true, 36 | ignoreRegExpLiterals: true, 37 | ignoreStrings: true, 38 | ignoreTemplateLiterals: true, 39 | }, 40 | ], 41 | }, 42 | overrides: [ 43 | { 44 | files: ['*template-file.js'], 45 | rules: { 46 | 'import/no-extraneous-dependencies': 'off', 47 | 'import/no-unresolved': 'off', 48 | }, 49 | }, 50 | { 51 | files: ['*.js'], 52 | rules: { 53 | 'import/no-extraneous-dependencies': 'off', 54 | 'import/no-unresolved': 'off', 55 | }, 56 | }, 57 | ], 58 | 59 | }; 60 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | @Guy7B -------------------------------------------------------------------------------- /.github/workflows/deployment.yml: -------------------------------------------------------------------------------- 1 | name: Publish dynamo-data-transformations npm package 2 | on: 3 | release: 4 | types: [created] 5 | jobs: 6 | 7 | publish-to-npm: 8 | runs-on: ubuntu-latest 9 | permissions: 10 | contents: read 11 | packages: write 12 | steps: 13 | - uses: actions/checkout@v3 14 | # Setup .npmrc file to publish the package to npmjs.com 15 | - uses: actions/setup-node@v3 16 | with: 17 | node-version: '16.x' 18 | registry-url: 'https://registry.npmjs.org' 19 | - run: npm install 20 | - run: npm publish 21 | env: 22 | NODE_AUTH_TOKEN: ${{ secrets.NPMJS_CI_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .envrc 3 | dist 4 | local 5 | .vscode 6 | .DS_Store 7 | .serverless -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | /.eslint* 2 | /.github 3 | /guy 4 | /.vscode 5 | /docs/images 6 | /examples -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![ddt_graphic1x_tl](https://user-images.githubusercontent.com/101042972/172161782-f4a3e15c-fdf2-42f1-a14d-434b109e7d2a.png) 2 |

3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |

14 | 15 | 16 | 17 | Dynamo Data Transform is an easy to use data transformation tool for DynamoDB. 18 | 19 | It allows performing powerful data transformations using simple Javascript commands, without the risk of breaking your database. 20 | Available as a [Serverless plugin](#-serverless-plugin), [npm package](#standalone-npm-package) and even as an [interactive CLI](#-interactive-cli), Dynamo Data Transform saves you time and keeps you safe with features like dry-running a data transformation and even rolling back your last trasnformation if needed. 21 | 22 | **Features** 23 | 24 | - Seemless data transformations management. 25 | - Support for multiple stages. 26 | - History of executed data transformations. 27 | - Dry run option for each command (by suppling --dry flag, the data will be printed instead of stored). 28 | - Safe & Secure preparation data 29 | - Store preparation data in a private s3 bucket. [Prepare data for your data transformation](#usage-and-command-line-options) 30 | 31 | ## Table of contents 32 | 33 | - [Quick Start](#quick-start) 34 | - [Serverless plugin](#-serverless-plugin) 35 | - [Standalone npm package](#standalone-npm-package) 36 | - [Interactive CLI](#-interactive-cli) 37 | - [Creating your first data transformation](#creating-your-first-data-transformation) 38 | - [Usage and command-line options](#usage-and-command-line-options) 39 | - [What happens behind the scenes](#what-happens-behind-the-scenes) 40 | - [Examples](#examples) 41 | - [The data transformation process](https://github.com/jitsecurity/dynamo-data-transform/blob/main/docs/zero_downtime_data_transformation_process.md) 42 | 43 | ## Quick Start 44 | ### ⚡ Serverless plugin 45 | - Install 46 | ```bash 47 | npm install dynamo-data-transform --save-dev 48 | ``` 49 | - Add the tool to your serverless.yml 50 | Run: 51 | ```bash 52 | npx serverless plugin install -n dynamo-data-transform 53 | ``` 54 | Or add manually to your serverless.yml: 55 | ```YML 56 | plugins: 57 | - dynamo-data-transform 58 | ``` 59 | - Run 60 | ```bash 61 | sls dynamodt --help 62 | ``` 63 | 64 | ### Standalone npm package 65 | - Install the tool 66 | ```bash 67 | npm install -g dynamo-data-transform -s 68 | ``` 69 | - Run the tool 70 | ```bash 71 | dynamodt help 72 | ``` 73 | Or with the shortcut 74 | ```bash 75 | ddt help 76 | ``` 77 | ### 💻 Interactive CLI 78 | After installing the npm package, run: 79 | ```bash 80 | dynamodt -i 81 | ``` 82 | ![cli gif](https://user-images.githubusercontent.com/35347793/172045910-d511e735-2d31-4713-bb64-5f55a900941c.gif) 83 | 84 | 85 | 86 | ## Creating your first data transformation 87 | 1. Intialize data-transformations folder 88 | Serverless (the plugin reads the table names from the serverless.yml file): 89 | ```bash 90 | sls dynamodt init --stage 91 | ``` 92 | Standalone: 93 | ```bash 94 | ddt init --tableNames 95 | ``` 96 | 97 | Open the generated data transformation file 'v1_script-name.js' file and implement the following functions: 98 | - transformUp: Executed when running `dynamodt up` 99 | - transformDown: Executed when running `dynamodt down -t ` 100 | - prepare (optional): Executed when running `dynamodt prepare -t
--tNumber ` 101 | 102 | The function parameters: 103 | - ddb: The DynamoDB Document client object see [DynamoDB Client](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-dynamodb) 104 | - isDryRun: Boolean indicating if --dry run supplied. You can use it to print/log the data instead of storing it. 105 | - preparationData: if you stored the preparation data using `dynamodt prepare`, you can use it here. 106 | 107 | 2. Run the data transformation 108 | ```bash 109 | dynamodt up 110 | ``` 111 | 112 | 113 | ## Data Transformation Script Format 114 | Make sure your script name contains the transformation number, for example: v1_transformation_script 115 | ```js 116 | const { utils } = require('dynamo-data-transform') 117 | 118 | const TABLE_NAME = 'UsersExample' 119 | 120 | const transformUp = async ({ ddb, isDryRun, preparationData }) => { 121 | // your code here... 122 | // return { transformed: 50 } // return the number of transformed items 123 | } 124 | 125 | const transformDown = async ({ ddb, isDryRun, preparationData }) => { 126 | // your code here... 127 | // return { transformed: 50 } // return the number of transformed items 128 | } 129 | 130 | const prepare = async ({ ddb, isDryRun }) => { 131 | // your code here... 132 | // return { transformed: 50 } // return the number of transformed items 133 | } 134 | 135 | module.exports = { 136 | transformUp, 137 | transformDown, 138 | prepare, // optional 139 | transformationNumber: 1, 140 | } 141 | ``` 142 | 143 | 144 | 145 | ## Usage and command-line options 146 | 147 | List available commands: 148 | Serverless plugin: 149 | ```bash 150 | sls dynamodt --help 151 | ``` 152 | Standalone npm package: 153 | ```bash 154 | dynamodt help 155 | ``` 156 | 157 | 158 | To list all of the options for a specific command run: 159 | Serverless plugin: 160 | ```bash 161 | sls dynamodt --help 162 | ``` 163 | 164 | Standalone npm package: 165 | ```bash 166 | dynamodt --help 167 | ``` 168 | 169 | ## What happens behind the scenes 170 | - When a data transformation runs for the first time, a record in your table is created. This record is for tracking the executed transformations on a specific table. 171 | 172 | 173 | 174 | ## Examples 175 | [Examples of data transformation code](https://github.com/jitsecurity/dynamo-data-transform/tree/main/examples/serverless-localstack/data-transformations/UsersExample) 176 | 177 | 178 | ### Insert records 179 | 180 | ```js 181 | // Seed users data transformation 182 | const { utils } = require('dynamo-data-transform'); 183 | const { USERS_DATA } = require('../../usersData'); 184 | 185 | const TABLE_NAME = 'UsersExample'; 186 | 187 | /** 188 | * @param {DynamoDBDocumentClient} ddb - dynamo db document client https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-dynamodb 189 | * @param {boolean} isDryRun - true if this is a dry run 190 | */ 191 | const transformUp = async ({ ddb, isDryRun }) => { 192 | return utils.insertItems(ddb, TABLE_NAME, USERS_DATA, isDryRun); 193 | }; 194 | 195 | const transformDown = async ({ ddb, isDryRun }) => { 196 | return utils.deleteItems(ddb, TABLE_NAME, USERS_DATA, isDryRun); 197 | }; 198 | 199 | module.exports = { 200 | transformUp, 201 | transformDown, 202 | transformationNumber: 1, 203 | }; 204 | ``` 205 | 206 | ### Add a new field to each record 207 | ```js 208 | // Adding a "randomNumber" field to each item 209 | const { utils } = require('dynamo-data-transform'); 210 | 211 | const TABLE_NAME = 'UsersExample'; 212 | 213 | const transformUp = async ({ ddb, isDryRun }) => { 214 | const addRandomNumberField = (item) => { 215 | const updatedItem = { ...item, randomNumber: Math.random() }; 216 | return updatedItem; 217 | }; 218 | return utils.transformItems(ddb, TABLE_NAME, addRandomNumberField, isDryRun); 219 | }; 220 | 221 | const transformDown = async ({ ddb, isDryRun }) => { 222 | const removeRandomNumberField = (item) => { 223 | const { randomNumber, ...oldItem } = item; 224 | return oldItem; 225 | }; 226 | return utils.transformItems(ddb, TABLE_NAME, removeRandomNumberField, isDryRun); 227 | }; 228 | 229 | module.exports = { 230 | transformUp, 231 | transformDown, 232 | transformationNumber: 2, 233 | }; 234 | ``` 235 | 236 | For more examples of data transformation code, see the [examples](https://github.com/jitsecurity/dynamo-data-transform/tree/main/examples/serverless-localstack/data-transformations/UsersExample) folder in the repository. 237 | 238 | 239 | Read how to automate data-transformation process here: 240 | https://www.infoq.com/articles/dynamoDB-data-transformation-safety/ 241 | -------------------------------------------------------------------------------- /cli/Components/App.js: -------------------------------------------------------------------------------- 1 | 'use-strict'; 2 | 3 | const React = require('react'); 4 | const importJsx = require('import-jsx'); 5 | const { render, Newline } = require('ink'); 6 | 7 | const Menu = importJsx('./Menu'); 8 | const CommandForm = importJsx('./CommandForm'); 9 | const Title = importJsx('./Title'); 10 | const BigTextErrorBoundary = importJsx('./BigTextErrorBoundary'); 11 | 12 | const App = () => { 13 | const [selection, setSelection] = React.useState(null); 14 | 15 | return ( 16 | <> 17 | 18 | 19 | </BigTextErrorBoundary> 20 | <Newline /> 21 | {selection ? ( 22 | <CommandForm selection={selection} setSelection={setSelection} /> 23 | ) : ( 24 | <Menu setSelection={setSelection} /> 25 | )} 26 | </> 27 | ); 28 | }; 29 | 30 | render(<App />); 31 | -------------------------------------------------------------------------------- /cli/Components/BigTextErrorBoundary.js: -------------------------------------------------------------------------------- 1 | 'use-strict'; 2 | 3 | const React = require('react'); 4 | const { Text } = require('ink'); 5 | const Gradient = require('ink-gradient'); 6 | 7 | 8 | class BigTextErrorBoundary extends React.Component { 9 | constructor(props) { 10 | super(props); 11 | this.state = {error: ""}; 12 | } 13 | 14 | componentDidCatch(error) { 15 | this.setState({error: `${error.name}: ${error.message}`}); 16 | } 17 | 18 | render() { 19 | const {error} = this.state; 20 | if (error) { 21 | return ( 22 | <Gradient name="rainbow"> 23 | <Text>{this.props.text || 'text'}</Text> 24 | </Gradient> 25 | ); 26 | } else { 27 | return <>{this.props.children}</>; 28 | } 29 | } 30 | } 31 | 32 | module.exports = BigTextErrorBoundary; -------------------------------------------------------------------------------- /cli/Components/CommandForm.js: -------------------------------------------------------------------------------- 1 | 'use-strict'; 2 | 3 | const React = require('react'); 4 | const { Text } = require('ink'); 5 | 6 | const { Form } = require('ink-form'); 7 | const scripts = require('../../src/command-handlers'); 8 | const { CLI_FORM, CLI_COMMANDS } = require('../cli-commands'); 9 | 10 | const getForm = (selection) => { 11 | return { 12 | form: { 13 | title: 'Please fill the following fields', 14 | sections: [CLI_FORM[selection]], 15 | }, 16 | }; 17 | }; 18 | 19 | const convertStringValueToArray = (values) => { 20 | const formattedValues = Object.entries(values).reduce((acc, [key, value]) => { 21 | if (typeof value === 'string') { 22 | return { 23 | ...acc, 24 | [key]: values[key] 25 | .replace(/ /g,'') // trim spaces 26 | .split(',') 27 | }; 28 | } 29 | return acc; 30 | }, values); 31 | return formattedValues; 32 | }; 33 | 34 | const buildScriptParameters = (selection, values) => { 35 | let parameters = values; 36 | if (selection === CLI_COMMANDS.init) { 37 | // convert tableNames to array of table names (table1,table2 -> ['table1', 'table2']) 38 | parameters = convertStringValueToArray(values); 39 | } 40 | return parameters; 41 | }; 42 | 43 | const CommandForm = ({ selection, setSelection }) => { 44 | const handleSubmit = (values) => { 45 | const params = buildScriptParameters(selection, values); 46 | const script = scripts[selection]; 47 | script(params).then(() => { 48 | setSelection(null); 49 | }).catch((error) => { 50 | console.error(error, `An error has occured while running transformation (${selection}).`); 51 | }); 52 | }; 53 | 54 | return ( 55 | <> 56 | <Text>{selection}</Text> 57 | <Form {...getForm(selection)} onSubmit={handleSubmit} /> 58 | </> 59 | ); 60 | }; 61 | 62 | module.exports = CommandForm; 63 | -------------------------------------------------------------------------------- /cli/Components/Menu.js: -------------------------------------------------------------------------------- 1 | 'use-strict'; 2 | 3 | const React = require('react'); 4 | const { Text } = require('ink'); 5 | const SelectInput = require('ink-select-input').default; 6 | 7 | const { CLI_COMMAND_OPTIONS } = require('../cli-commands'); 8 | 9 | const Menu = ({ setSelection }) => { 10 | const handleSelect = item => { 11 | setSelection(item.value); 12 | }; 13 | 14 | return ( 15 | <> 16 | <Text color={'yellow'}>Select a command:</Text> 17 | <SelectInput items={CLI_COMMAND_OPTIONS} onSelect={handleSelect} /> 18 | </> 19 | ); 20 | }; 21 | 22 | module.exports = Menu; -------------------------------------------------------------------------------- /cli/Components/Title.js: -------------------------------------------------------------------------------- 1 | 'use-strict'; 2 | 3 | const React = require('react'); 4 | const { Static } = require('ink'); 5 | const Gradient = require('ink-gradient'); 6 | const BigText = require('ink-big-text'); 7 | 8 | const Title = ({ text }) => { 9 | return ( 10 | 11 | <Static items={[text]} > 12 | {(item) => ( 13 | <Gradient key={item} name="rainbow"> 14 | <BigText font='tiny' text={item} /> 15 | </Gradient> 16 | )} 17 | </Static> 18 | ); 19 | }; 20 | 21 | module.exports = Title; -------------------------------------------------------------------------------- /cli/cli-commands.js: -------------------------------------------------------------------------------- 1 | const { COMMAND_DESCRIPTION } = require("../src/config/commands"); 2 | 3 | const COMMAND_OPTIONS = { 4 | dry: { 5 | type: 'boolean', name: 'dry', label: 'Specify if you want a dry run', initialValue: true, 6 | }, 7 | table: { 8 | type: 'string', name: 'table', label: 'Specify table name', initialValue: '', 9 | }, 10 | tNumber: { 11 | type: 'string', name: 'tNumber', label: 'Specify the version of current transformation number', 12 | }, 13 | tableNames: { 14 | type: 'string', name: 'tableNames', label: 'Specify table names e.g "table1, table2"', initialValue: '', 15 | }, 16 | }; 17 | 18 | const CLI_COMMANDS = { 19 | up: "up", 20 | down: "down", 21 | history: "history", 22 | prepare: "prepare", 23 | init: "init", 24 | }; 25 | 26 | const HELP_COMMANDS = { 27 | up: 'ddt up', 28 | down: 'ddt down --table <table>', 29 | history: 'ddt history --table <table>', 30 | prepare: 'ddt prepare --table <table> --tNumber <transformation_number>', 31 | init: 'ddt init --tableNames <table_names>', 32 | }; 33 | 34 | const CLI_FORM = { 35 | [CLI_COMMANDS.up]: { 36 | title: 'Up Parameters', 37 | fields: [ 38 | COMMAND_OPTIONS.dry, 39 | ], 40 | }, 41 | [CLI_COMMANDS.down]: { 42 | title: 'Down Parameters', 43 | fields: [ 44 | COMMAND_OPTIONS.table, 45 | COMMAND_OPTIONS.dry, 46 | ], 47 | }, 48 | [CLI_COMMANDS.prepare]: { 49 | title: 'Preparetion Parameters', 50 | fields: [ 51 | COMMAND_OPTIONS.table, 52 | COMMAND_OPTIONS.tNumber, 53 | COMMAND_OPTIONS.dry, 54 | ], 55 | }, 56 | [CLI_COMMANDS.history]: { 57 | title: 'History Parameters', 58 | fields: [ 59 | COMMAND_OPTIONS.table, 60 | ], 61 | }, 62 | [CLI_COMMANDS.init]: { 63 | title: 'Init Parameters', 64 | fields: [ 65 | COMMAND_OPTIONS.tableNames, 66 | ], 67 | }, 68 | }; 69 | 70 | const CLI_COMMAND_OPTIONS = Object.values(CLI_COMMANDS).map((command) => { 71 | return { 72 | label: `${command} - ${COMMAND_DESCRIPTION[command]}`, 73 | value: command, 74 | }; 75 | }); 76 | 77 | module.exports = { 78 | COMMAND_OPTIONS, 79 | CLI_FORM, 80 | CLI_COMMAND_OPTIONS, 81 | CLI_COMMANDS, 82 | HELP_COMMANDS, 83 | }; 84 | -------------------------------------------------------------------------------- /cli/index.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use-strict'; 3 | 4 | const scripts = require('../src/command-handlers'); 5 | const parseArgs = require('minimist'); 6 | const { COMMAND_DESCRIPTION } = require('../src/config/commands'); 7 | const { HELP_COMMANDS } = require('./cli-commands'); 8 | 9 | const commandAliases = { 10 | dry: 'd', 11 | table: 't', 12 | tNumber: 'tnum', 13 | tableNames: 'n', 14 | interactive: 'i', 15 | }; 16 | 17 | const parsedOptions = parseArgs(process.argv.slice(2),{ 18 | alias: commandAliases, 19 | boolean: ['dry', 'interactive', 'help'], 20 | string: ['table', 'tableNames'], 21 | number: ['tNumber'], 22 | }); 23 | 24 | const options = Object.entries(parsedOptions).reduce((acc, [key, value]) => { 25 | if (['tableNames','n'].includes(key)) { 26 | acc[key] = value.split(','); 27 | } else { 28 | acc[key] = value; 29 | } 30 | return acc; 31 | },{}); 32 | 33 | const showHelp = () => { 34 | console.info('Available commands:'); 35 | Object.entries(COMMAND_DESCRIPTION).forEach(([key, value]) => { 36 | console.info(` ${key} - ${value}\n`); 37 | }); 38 | }; 39 | 40 | (() => { 41 | if (options.interactive) { 42 | const importJsx = require('import-jsx'); 43 | importJsx('./Components/App'); 44 | return; 45 | } 46 | 47 | const [command] = options._; 48 | if(command === 'help' || !command) { 49 | showHelp(); 50 | process.exit(0); 51 | } 52 | 53 | if(options.help) { 54 | console.info(HELP_COMMANDS[command]); 55 | process.exit(0); 56 | } 57 | scripts[command](options).then(() => { 58 | console.info(`"${command}" command run successfully.`); 59 | process.exit(0); 60 | }).catch((error) => { 61 | console.error(error, `An error has occured while running command (${command}).`); 62 | process.exit(1); 63 | }); 64 | 65 | })(); 66 | -------------------------------------------------------------------------------- /docs/zero_downtime_data_transformation_process.md: -------------------------------------------------------------------------------- 1 | # Zero down time data transformation process 2 | The next section describes how the data transformation process looks like, and the order of each step. 3 | 4 | [Process Steps](#steps) 5 | [Troubleshooting](#troubleshooting) 6 | [Key Concepts](#key-concepts) 7 | ## Steps 8 | ### 1st Phase (Add New Resources) 9 | 1. Update the table resources if needed \ 10 | Reminder: we are not overriding existing data but creating new. 11 | 1. Your new code should be able to write to your old and new resources which ensures that we can roll back to the previous state and prevent possible data gaps. 12 | 1. Create a pull request and deploy it to every stage in your application 13 | 14 | ### 2nd Phase (data transformation) 15 | 16 | 1. For the first time use `sls dynamodt init` it will generate a folder per table inside the root folder of your service (The name of the folder is the exact name of the table). 17 | A template data transformation file (v1.js) will be created in each table folder. \ 18 | Implement these functions: 19 | 1. `transformUp` - transform all of the table items to the new shape (use preparationData if needed). 20 | 1. `transformDown` - transform all of the table items to the previous shape. 21 | 1. `prepare` - use this function whenever your data transformation relies on data from external resources. 22 | 23 | 1. Export these functions and export the version of the current data transformation (set the sequence variable value. It should be the same value as that of the file name). 24 | 25 | 1. Preparing data from external resources for the data transformation can be done by using `sls dynamodt prepare` 26 | 27 | Run `sls dynamodt prepare --tNumber <transformation_number> --table <table>`\ 28 | The data will be stored in a S3 bucket \ 29 | The data will be decrypted while running the data transformation script. 30 | 31 | 1. **Final Step** Create a pull request. \ 32 | Note that the data transformation runs after an sls deploy command it is integrated \ 33 | with lifecycle of serverless `after:deploy:deploy` hook. 34 | 35 | ### 3rd Phase (Use The New Resources/Data) 36 | 1. Adjust your code to work with the new data. \ 37 | For example, read from the new index instead of the old one. 38 | 1. Create a pull request with the updated lambdas. 39 | 40 | 41 | ### 4th Phase (Cleanup) 42 | 1. Clean the unused data (attributes/indexes/etc). 43 | 44 | 45 | ### Key Concepts 46 | - Don't override resources/data 47 | - Your code should be able to work with the old version of the data and keep it updated. 48 | - Prefer multiple data transformations over complex one. 49 | -------------------------------------------------------------------------------- /examples/serverless-localstack/.env.local: -------------------------------------------------------------------------------- 1 | LOCALSTACK_URL=http://localstack:4566 2 | LOCALSTACK_HOSTNAME=localstack 3 | DEPLOYMENT_STAGE=local 4 | AWS_REGION=us-east-1 5 | AWS_ACCESS_KEY_ID=dummy 6 | AWS_SECRET_ACCESS_KEY=dummy 7 | AWS_SESSION_TOKEN=dummy 8 | AWS_CUSTOM_ENDPOINT=http://localhost:4566 9 | LOG_LEVEL=DEBUG 10 | PREPARATION_DATA_BUCKET=transformations-preparation-data-local -------------------------------------------------------------------------------- /examples/serverless-localstack/EXERCISE-prepare-data/v4_using_preparation_data.js: -------------------------------------------------------------------------------- 1 | // Adding a new field "hasWikiPage" 2 | // "hasWikiPage" is a boolean field that is set to true if the item has a wiki page 3 | // It is calculated with a prepare function that fetches the wiki page status for each item 4 | 5 | const { utils } = require('dynamo-data-transform'); 6 | 7 | const userAgentHeader = { 8 | 'User-Agent': 'Chrome/81.0.4044.138', 9 | }; 10 | 11 | const fetch = (...args) => import('node-fetch').then(({ default: nodeFetch }) => nodeFetch( 12 | ...args, 13 | { 14 | headers: userAgentHeader, 15 | }, 16 | )); 17 | 18 | const TABLE_NAME = 'UsersExample'; 19 | 20 | const transformUp = async ({ ddb, preparationData, isDryRun }) => { 21 | const addHasWikiPage = (hasWikiDict) => (item) => { 22 | const valueFromPreparation = hasWikiDict[`${item.PK}-${item.SK}`]; 23 | const updatedItem = valueFromPreparation ? { 24 | ...item, 25 | hasWikiPage: valueFromPreparation, 26 | } : item; 27 | return updatedItem; 28 | }; 29 | 30 | return utils.transformItems( 31 | ddb, 32 | TABLE_NAME, 33 | addHasWikiPage(JSON.parse(preparationData)), 34 | isDryRun, 35 | ); 36 | }; 37 | 38 | const transformDown = async ({ ddb, isDryRun }) => { 39 | const removeHasWikiPage = (item) => { 40 | const { hasWikiPage, ...oldItem } = item; 41 | return oldItem; 42 | }; 43 | 44 | return utils.transformItems(ddb, TABLE_NAME, removeHasWikiPage, isDryRun); 45 | }; 46 | 47 | const prepare = async ({ ddb }) => { 48 | let lastEvalKey; 49 | let preparationData = {}; 50 | 51 | let scannedAllItems = false; 52 | 53 | while (!scannedAllItems) { 54 | const { Items, LastEvaluatedKey } = await utils.getItems(ddb, lastEvalKey, TABLE_NAME); 55 | lastEvalKey = LastEvaluatedKey; 56 | 57 | const currentPreparationData = await Promise.all(Items.map(async (item) => { 58 | const wikiItemUrl = `https://en.wikipedia.org/wiki/${item.name}`; 59 | const currWikiResponse = await fetch(wikiItemUrl); 60 | return { 61 | [`${item.PK}-${item.SK}`]: currWikiResponse.status === 200, 62 | }; 63 | })); 64 | 65 | preparationData = { 66 | ...preparationData, 67 | ...currentPreparationData.reduce((acc, item) => ({ ...acc, ...item }), {}), 68 | }; 69 | 70 | scannedAllItems = !lastEvalKey; 71 | } 72 | 73 | return preparationData; 74 | }; 75 | 76 | module.exports = { 77 | transformUp, 78 | transformDown, 79 | prepare, 80 | transformationNumber: 4, 81 | }; 82 | -------------------------------------------------------------------------------- /examples/serverless-localstack/README.md: -------------------------------------------------------------------------------- 1 | ## This repository shows the usage of the dynamo-data-transform package. 2 | 3 | ## Quick Start 4 | 5 | ### Initialize localstack container and dynamodb GUI 6 | *Please make sure you have installed and running docker on your machine. 7 | Run: 8 | ```bash 9 | docker compose up 10 | ``` 11 | 12 | ### Install dependencies 13 | ```bash 14 | npm install 15 | ``` 16 | 17 | ### Deploy service to localstack 18 | ```bash 19 | npm start 20 | ``` 21 | 22 | Note that after running the above command, all the transformation scripts in data-transformations folder will be executed. 23 | See the UsersExample table here: 24 | http://localhost:8001/ 25 | 26 | 27 | 28 | #### Serverless Plugin 29 | ```bash 30 | npx sls dynamodt --help 31 | ``` 32 | 33 | #### Interactive Cli 34 | ```bash 35 | npm install -g dynamo-data-transform 36 | 37 | ddt -i 38 | ``` 39 | 40 | #### Standalone npm package usage: 41 | - init - `ddt init -tableNames "Users"` 42 | - up - `ddt up` 43 | - down - `ddt down -t Users` 44 | - prepare - `ddt prepare -t Users --tNumber 3` 45 | - history - `ddt history -t Users` 46 | 47 | 48 | 49 | ## Exercises 50 | 1. For understanding how to prepare data for transformation, take a look at [v4_using_preparation_data.js](https://github.com/jitsecurity/dynamo-data-transform/blob/main/examples/serverless-localstack/EXERCISE-prepare-data/v4_using_preparation_data.js). 51 | Move the file `v4_using_preparation_data.js` to the data-transformations folder. 52 | ```bash 53 | mv EXERCISE-prepare-data/v4_using_preparation_data.js data-transformations/UsersExample 54 | ``` 55 | Let's check the `prepare` script results 56 | ```bash 57 | npx sls dynamodt prepare --table UsersExample --tNumber 4 --dry 58 | ``` 59 | The results in the console should be: 60 | ```js 61 | { 62 | 'USER#21-NAME#Bradley Wiggins: true, 63 | 'USER#34-NAME#Chaos': true, 64 | 'USER#32-NAME#Knuckles': true, 65 | 'USER#29-NAME#Plankton': true, 66 | ... 67 | } 68 | ``` 69 | Now lets prepare some data for the transformation. Run the same command as before but without `--dry`. 70 | ```bash 71 | npx sls dynamodt prepare --table UsersExample --tNumber 4 --stage local 72 | ``` 73 | Let's run the pending transformation script, currently it is "v4_using_preparation_data.js" 74 | ```bash 75 | npx sls dynamodt up --stage local 76 | ``` 77 | Now open the dynamodb GUI and check the data. 78 | http://localhost:8001/ 79 | 80 | 2. Rollback the last transformation 81 | ```bash 82 | npx sls dynamodt down --stage local --table UsersExample --dry 83 | ``` 84 | Now you will see in the console that `hasWikiPage` field was removed from each item. 85 | Let's rollback the last transformation for real. 86 | ```bash 87 | npx sls dynamodt down --stage local --table UsersExample 88 | ``` 89 | Now open the dynamodb GUI and check the data. 90 | -------------------------------------------------------------------------------- /examples/serverless-localstack/data-transformations/UsersExample/v1_insert_users.js: -------------------------------------------------------------------------------- 1 | // Seed users data transformation 2 | const { utils } = require('dynamo-data-transform'); 3 | 4 | const { USERS_DATA } = require('../../usersData'); 5 | 6 | const TABLE_NAME = 'UsersExample'; 7 | 8 | /** 9 | * @param {DynamoDBDocumentClient} ddb - dynamo db document client https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-dynamodb 10 | * @param {boolean} isDryRun - true if this is a dry run 11 | */ 12 | const transformUp = async ({ ddb, isDryRun }) => { 13 | return utils.insertItems(ddb, TABLE_NAME, USERS_DATA, isDryRun); 14 | }; 15 | 16 | const transformDown = async ({ ddb, isDryRun }) => { 17 | return utils.deleteItems(ddb, TABLE_NAME, USERS_DATA, isDryRun); 18 | }; 19 | 20 | module.exports = { 21 | transformUp, 22 | transformDown, 23 | transformationNumber: 1, 24 | }; 25 | -------------------------------------------------------------------------------- /examples/serverless-localstack/data-transformations/UsersExample/v2_add_random_number_field.js: -------------------------------------------------------------------------------- 1 | // Adding a "randomNumber" field to each item 2 | const { utils } = require('dynamo-data-transform'); 3 | 4 | const TABLE_NAME = 'UsersExample'; 5 | 6 | const transformUp = async ({ ddb, isDryRun }) => { 7 | const addRandomNumberField = (item) => { 8 | const updatedItem = { ...item, randomNumber: Math.random() }; 9 | return updatedItem; 10 | }; 11 | return utils.transformItems(ddb, TABLE_NAME, addRandomNumberField, isDryRun); 12 | }; 13 | 14 | const transformDown = async ({ ddb, isDryRun }) => { 15 | const removeRandomNumberField = (item) => { 16 | const { randomNumber, ...oldItem } = item; 17 | return oldItem; 18 | }; 19 | return utils.transformItems(ddb, TABLE_NAME, removeRandomNumberField, isDryRun); 20 | }; 21 | 22 | module.exports = { 23 | transformUp, 24 | transformDown, 25 | transformationNumber: 2, 26 | }; 27 | -------------------------------------------------------------------------------- /examples/serverless-localstack/data-transformations/UsersExample/v3_split_fullname.js: -------------------------------------------------------------------------------- 1 | // Split full name of every record into first and last name 2 | const { utils } = require('dynamo-data-transform'); 3 | 4 | const TABLE_NAME = 'UsersExample'; 5 | 6 | const transformUp = async ({ ddb, isDryRun }) => { 7 | const addFirstAndLastName = (item) => { 8 | // Just for the example: 9 | // Assume that the FullName has one space between first and last name 10 | const [firstName, ...lastName] = item.name.split(' '); 11 | return { 12 | ...item, 13 | firstName, 14 | lastName: lastName.join(' '), 15 | }; 16 | }; 17 | return utils.transformItems(ddb, TABLE_NAME, addFirstAndLastName, isDryRun); 18 | }; 19 | 20 | const transformDown = async ({ ddb, isDryRun }) => { 21 | const removeFirstAndLastName = (item) => { 22 | const { firstName, lastName, ...oldItem } = item; 23 | return oldItem; 24 | }; 25 | return utils.transformItems(ddb, TABLE_NAME, removeFirstAndLastName, isDryRun); 26 | }; 27 | 28 | module.exports = { 29 | transformUp, 30 | transformDown, 31 | transformationNumber: 3, 32 | }; 33 | -------------------------------------------------------------------------------- /examples/serverless-localstack/data-transformations/UsersExample/v4_using_preparation_data.js: -------------------------------------------------------------------------------- 1 | // Adding a new field "hasWikiPage" 2 | // "hasWikiPage" is a boolean field that is set to true if the item has a wiki page 3 | // It is calculated with a prepare function that fetches the wiki page status for each item 4 | 5 | const { utils } = require('dynamo-data-transform'); 6 | 7 | const userAgentHeader = { 8 | 'User-Agent': 'Chrome/81.0.4044.138', 9 | }; 10 | 11 | const fetch = (...args) => import('node-fetch').then(({ default: nodeFetch }) => nodeFetch( 12 | ...args, 13 | { 14 | headers: userAgentHeader, 15 | }, 16 | )); 17 | 18 | const TABLE_NAME = 'UsersExample'; 19 | 20 | const transformUp = async ({ ddb, preparationData, isDryRun }) => { 21 | const addHasWikiPage = (hasWikiDict) => (item) => { 22 | const valueFromPreparation = hasWikiDict[`${item.PK}-${item.SK}`]; 23 | const updatedItem = valueFromPreparation ? { 24 | ...item, 25 | hasWikiPage: valueFromPreparation, 26 | } : item; 27 | return updatedItem; 28 | }; 29 | 30 | return utils.transformItems( 31 | ddb, 32 | TABLE_NAME, 33 | addHasWikiPage(JSON.parse(preparationData)), 34 | isDryRun, 35 | ); 36 | }; 37 | 38 | const transformDown = async ({ ddb, isDryRun }) => { 39 | const removeHasWikiPage = (item) => { 40 | const { hasWikiPage, ...oldItem } = item; 41 | return oldItem; 42 | }; 43 | 44 | return utils.transformItems(ddb, TABLE_NAME, removeHasWikiPage, isDryRun); 45 | }; 46 | 47 | const prepare = async ({ ddb }) => { 48 | let lastEvalKey; 49 | let preparationData = {}; 50 | 51 | let scannedAllItems = false; 52 | 53 | while (!scannedAllItems) { 54 | const { Items, LastEvaluatedKey } = await utils.getItems(ddb, lastEvalKey, TABLE_NAME); 55 | lastEvalKey = LastEvaluatedKey; 56 | 57 | const currentPreparationData = await Promise.all(Items.map(async (item) => { 58 | const wikiItemUrl = `https://en.wikipedia.org/wiki/${item.name}`; 59 | const currWikiResponse = await fetch(wikiItemUrl); 60 | return { 61 | [`${item.PK}-${item.SK}`]: currWikiResponse.status === 200, 62 | }; 63 | })); 64 | 65 | preparationData = { 66 | ...preparationData, 67 | ...currentPreparationData.reduce((acc, item) => ({ ...acc, ...item }), {}), 68 | }; 69 | 70 | scannedAllItems = !lastEvalKey; 71 | } 72 | 73 | return preparationData; 74 | }; 75 | 76 | module.exports = { 77 | transformUp, 78 | transformDown, 79 | prepare, 80 | transformationNumber: 4, 81 | }; 82 | -------------------------------------------------------------------------------- /examples/serverless-localstack/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.9' 2 | 3 | services: 4 | dynamodb: 5 | image: aaronshaf/dynamodb-admin 6 | ports: 7 | - "8001:8001" 8 | environment: 9 | - DYNAMO_ENDPOINT=http://localstack:4566 10 | networks: 11 | - localstack-net 12 | 13 | localstack: 14 | image: localstack/localstack:0.14.2 15 | ports: 16 | - "4566-4583:4566-4583" 17 | - "${PORT_WEB_UI-4666}:${PORT_WEB_UI-8080}" 18 | - "8080:8080" 19 | - "4510:4510" 20 | environment: 21 | - SERVICES=${SERVICES-iam,s3,s3api,lambda,apigateway,apigatewaymanagementapi,cloudwatch,events,cloudformation,sts,cognito,secretsmanager,dynamodb,ecr,sqs,ssm,sns} 22 | - DATA_DIR=${DATA_DIR- } 23 | - LOCALSTACK_API_KEY=${LOCALSTACK_API_KEY- } 24 | - PORT_WEB_UI=8080 25 | - START_WEB=0 26 | - LAMBDA_REMOTE_DOCKER=true 27 | - LAMBDA_EXECUTOR=docker-reuse 28 | - LAMBDA_REMOVE_CONTAINERS=true 29 | - DOCKER_HOST=unix:///var/run/docker.sock 30 | - LAMBDA_DOCKER_NETWORK=localstack-net 31 | - KINESIS_ERROR_PROBABILITY=${KINESIS_ERROR_PROBABILITY- } 32 | - LS_LOG=info 33 | - DEBUG=0 34 | - TMPDIR=./tmp/localstack 35 | - HOST_TMP_FOLDER=${TMPDIR} 36 | volumes: 37 | - "/var/run/docker.sock:/var/run/docker.sock" 38 | - "${TMPDIR:-/tmp/localstack}" 39 | networks: 40 | - localstack-net 41 | 42 | networks: 43 | localstack-net: 44 | name: localstack-net 45 | -------------------------------------------------------------------------------- /examples/serverless-localstack/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynamo-db-data-transformation-example", 3 | "version": "1.0.0", 4 | "description": "", 5 | "scripts": { 6 | "start": "serverless deploy -s local", 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "dependencies": { 10 | "dynamo-data-transform": "^0.1.5", 11 | "serverless": "^3.17.0", 12 | "serverless-dotenv-plugin": "~3.9.0", 13 | "serverless-localstack": "~0.4.35", 14 | "serverless-manifest-plugin": "~1.0.7", 15 | "serverless-pseudo-parameters": "~2.5.0" 16 | }, 17 | "author": "Guy Braunstain <guy.braunstain@jit.io>", 18 | "license": "MIT" 19 | } 20 | -------------------------------------------------------------------------------- /examples/serverless-localstack/serverless.yml: -------------------------------------------------------------------------------- 1 | service: ddb-data-transformations 2 | 3 | provider: 4 | name: aws 5 | runtime: nodejs14.x 6 | timeout: 30 7 | stackName: ${self:service}-${self:provider.stage} 8 | stage: ${opt:stage, 'local'} 9 | region: us-east-1 10 | apiGateway: 11 | shouldStartNameWithService: true 12 | lambdaHashingVersion: 20201221 13 | environment: 14 | DEPLOYMENT_STAGE: ${self:provider.stage} 15 | SERVICE_NAME: ${self:service} 16 | AWS_REGION_NAME: us-east-1 17 | PREPARATION_DATA_BUCKET: transformations-preparation-data-${self:provider.stage} 18 | 19 | plugins: 20 | - serverless-localstack 21 | - serverless-manifest-plugin 22 | - serverless-dotenv-plugin 23 | - serverless-pseudo-parameters 24 | - dynamo-data-transform 25 | custom: 26 | localstack: 27 | hostname: 28 | local: http://localhost 29 | debug: true 30 | stages: 31 | - local 32 | host: ${self:custom.localstack.hostname.${self:provider.stage}, ''} 33 | endpoints: 34 | S3: ${self:custom.localstack.host}:4566 35 | DynamoDB: ${self:custom.localstack.host}:4566 36 | CloudFormation: ${self:custom.localstack.host}:4566 37 | Lambda: ${self:custom.localstack.host}:4566 38 | basePath: transformations 39 | dotenv: 40 | path: 41 | .env.${self:provider.stage, 'local'} 42 | 43 | 44 | resources: 45 | Resources: 46 | PreparationDataBucket: 47 | Type: AWS::S3::Bucket 48 | Properties: 49 | BucketName: transformations-preparation-data-${self:provider.stage} 50 | PublicAccessBlockConfiguration: 51 | BlockPublicAcls: true 52 | IgnorePublicAcls: true 53 | BlockPublicPolicy: true 54 | RestrictPublicBuckets: true 55 | BucketEncryption: 56 | ServerSideEncryptionConfiguration: 57 | - ServerSideEncryptionByDefault: 58 | SSEAlgorithm: AES256 59 | AccessControl: "Private" 60 | UsersExampleTable: 61 | Type: AWS::DynamoDB::Table 62 | Properties: 63 | TableName: UsersExample 64 | BillingMode: PAY_PER_REQUEST 65 | AttributeDefinitions: 66 | - AttributeName: PK 67 | AttributeType: S 68 | - AttributeName: SK 69 | AttributeType: S 70 | - AttributeName: GSI1PK 71 | AttributeType: S 72 | - AttributeName: GSI1SK 73 | AttributeType: S 74 | - AttributeName: LSI1SK 75 | AttributeType: N 76 | KeySchema: 77 | - AttributeName: PK 78 | KeyType: HASH 79 | - AttributeName: SK 80 | KeyType: RANGE 81 | GlobalSecondaryIndexes: 82 | - IndexName: GSI1 83 | KeySchema: 84 | - AttributeName: GSI1PK 85 | KeyType: HASH 86 | - AttributeName: GSI1SK 87 | KeyType: RANGE 88 | Projection: 89 | ProjectionType: 'ALL' 90 | LocalSecondaryIndexes: 91 | - IndexName: LSI1 92 | KeySchema: 93 | - AttributeName: PK 94 | KeyType: HASH 95 | - AttributeName: LSI1SK 96 | KeyType: RANGE 97 | Projection: 98 | ProjectionType: 'ALL' 99 | -------------------------------------------------------------------------------- /examples/serverless-localstack/usersData.js: -------------------------------------------------------------------------------- 1 | const USERS_DATA = [ 2 | { 3 | PK: 'USER#1', 4 | SK: 'NAME#Kobe Bryant', 5 | GSI1: 'CATEGORY#NBA', 6 | LSI1: 'ROLE#Player', 7 | id: '1', 8 | name: 'Kobe Bryant', 9 | category: 'NBA', 10 | role: 'Player', 11 | }, 12 | { 13 | PK: 'USER#2', 14 | SK: 'NAME#Lebron James', 15 | GSI1: 'CATEGORY#NBA', 16 | LSI1: 'ROLE#Player', 17 | id: '2', 18 | name: 'Lebron James', 19 | category: 'NBA', 20 | role: 'Player', 21 | }, 22 | { 23 | PK: 'USER#3', 24 | SK: 'NAME#Stephen Curry', 25 | GSI1: 'CATEGORY#NBA', 26 | LSI1: 'ROLE#Player', 27 | id: '3', 28 | name: 'Stephen Curry', 29 | category: 'NBA', 30 | role: 'Player', 31 | }, 32 | { 33 | PK: 'USER#4', 34 | SK: 'NAME#Kevin Durant', 35 | GSI1: 'CATEGORY#NBA', 36 | LSI1: 'ROLE#Player', 37 | id: '4', 38 | name: 'Kevin Durant', 39 | category: 'NBA', 40 | role: 'Player', 41 | }, 42 | { 43 | PK: 'USER#5', 44 | SK: 'NAME#James Harden', 45 | GSI1: 'CATEGORY#NBA', 46 | LSI1: 'ROLE#Player', 47 | id: '5', 48 | name: 'James Harden', 49 | category: 'NBA', 50 | role: 'Player', 51 | }, 52 | { 53 | PK: 'USER#6', 54 | SK: 'NAME#Draymond Green', 55 | GSI1: 'CATEGORY#NBA', 56 | LSI1: 'ROLE#Player', 57 | id: '6', 58 | name: 'Draymond Green', 59 | category: 'NBA', 60 | role: 'Player', 61 | }, 62 | { 63 | PK: 'USER#7', 64 | SK: 'NAME#Klay Thompson', 65 | GSI1: 'CATEGORY#NBA', 66 | LSI1: 'ROLE#Player', 67 | id: '7', 68 | name: 'Klay Thompson', 69 | category: 'NBA', 70 | role: 'Player', 71 | }, 72 | { 73 | PK: 'USER#8', 74 | SK: 'NAME#Anthony Davis', 75 | GSI1: 'CATEGORY#NBA', 76 | LSI1: 'ROLE#Player', 77 | id: '8', 78 | name: 'Anthony Davis', 79 | category: 'NBA', 80 | role: 'Player', 81 | }, 82 | { 83 | PK: 'USER#9', 84 | SK: 'NAME#Messi', 85 | GSI1: 'CATEGORY#Soccer', 86 | LSI1: 'ROLE#Player', 87 | id: '9', 88 | name: 'Messi', 89 | category: 'Soccer', 90 | role: 'Player', 91 | }, 92 | { 93 | PK: 'USER#10', 94 | SK: 'NAME#Ronaldo', 95 | GSI1: 'CATEGORY#Soccer', 96 | LSI1: 'ROLE#Player', 97 | id: '10', 98 | name: 'Ronaldo', 99 | category: 'Soccer', 100 | role: 'Player', 101 | }, 102 | { 103 | PK: 'USER#11', 104 | SK: 'NAME#Neymar', 105 | GSI1: 'CATEGORY#Soccer', 106 | LSI1: 'ROLE#Player', 107 | id: '11', 108 | name: 'Neymar', 109 | category: 'Soccer', 110 | role: 'Player', 111 | }, 112 | { 113 | PK: 'USER#12', 114 | SK: 'NAME#Cristiano', 115 | GSI1: 'CATEGORY#Soccer', 116 | LSI1: 'ROLE#Player', 117 | id: '12', 118 | name: 'Cristiano', 119 | category: 'Soccer', 120 | role: 'Player', 121 | }, 122 | { 123 | PK: 'USER#13', 124 | SK: 'NAME#Lukaku', 125 | GSI1: 'CATEGORY#Soccer', 126 | LSI1: 'ROLE#Player', 127 | id: '13', 128 | name: 'Lukaku', 129 | category: 'Soccer', 130 | role: 'Player', 131 | }, 132 | { 133 | PK: 'USER#14', 134 | SK: 'NAME#Kante', 135 | GSI1: 'CATEGORY#Soccer', 136 | LSI1: 'ROLE#Player', 137 | id: '14', 138 | name: 'Kante', 139 | category: 'Soccer', 140 | role: 'Player', 141 | }, 142 | { 143 | PK: 'USER#15', 144 | SK: 'NAME#Kroos', 145 | GSI1: 'CATEGORY#Soccer', 146 | LSI1: 'ROLE#Player', 147 | id: '15', 148 | name: 'Kroos', 149 | category: 'Soccer', 150 | role: 'Player', 151 | }, 152 | { 153 | PK: 'USER#16', 154 | SK: 'NAME#Modric', 155 | GSI1: 'CATEGORY#Soccer', 156 | LSI1: 'ROLE#Player', 157 | id: '16', 158 | name: 'Modric', 159 | category: 'Soccer', 160 | role: 'Player', 161 | }, 162 | { 163 | PK: 'USER#17', 164 | SK: 'NAME#Bale', 165 | GSI1: 'CATEGORY#Soccer', 166 | LSI1: 'ROLE#Player', 167 | id: '17', 168 | name: 'Bale', 169 | category: 'Soccer', 170 | role: 'Player', 171 | }, 172 | { 173 | PK: 'USER#18', 174 | SK: 'NAME#Benzema', 175 | GSI1: 'CATEGORY#Soccer', 176 | LSI1: 'ROLE#Player', 177 | id: '18', 178 | name: 'Benzema', 179 | category: 'Soccer', 180 | role: 'Player', 181 | }, 182 | { 183 | PK: 'USER#19', 184 | SK: 'NAME#Griezmann', 185 | GSI1: 'CATEGORY#Soccer', 186 | LSI1: 'ROLE#Player', 187 | id: '19', 188 | name: 'Griezmann', 189 | category: 'Soccer', 190 | role: 'Player', 191 | }, 192 | { 193 | PK: 'USER#20', 194 | SK: 'NAME#Puyol', 195 | GSI1: 'CATEGORY#Soccer', 196 | LSI1: 'ROLE#Player', 197 | id: '20', 198 | name: 'Puyol', 199 | category: 'Soccer', 200 | role: 'Player', 201 | }, 202 | { 203 | PK: 'USER#21', 204 | SK: 'NAME#Bradley Wiggins', 205 | GSI1: 'CATEGORY#Cycling', 206 | LSI1: 'ROLE#Rider', 207 | id: '21', 208 | name: 'Bradley Wiggins', 209 | category: 'Cycling', 210 | role: 'Rider', 211 | }, 212 | { 213 | PK: 'USER#22', 214 | SK: 'NAME#Jenson Button', 215 | GSI1: 'CATEGORY#Cycling', 216 | LSI1: 'ROLE#Rider', 217 | id: '22', 218 | name: 'Jenson Button', 219 | category: 'Cycling', 220 | role: 'Rider', 221 | }, 222 | { 223 | PK: 'USER#22', 224 | SK: 'NAME#Spongebob', 225 | GSI1: 'CATEGORY#TV', 226 | LSI1: 'ROLE#Character', 227 | id: '22', 228 | name: 'Spongebob', 229 | category: 'TV', 230 | role: 'Character', 231 | }, 232 | { 233 | PK: 'USER#23', 234 | SK: 'NAME#Patrick', 235 | GSI1: 'CATEGORY#TV', 236 | LSI1: 'ROLE#Character', 237 | id: '23', 238 | name: 'Patrick', 239 | category: 'TV', 240 | role: 'Character', 241 | }, 242 | { 243 | PK: 'USER#24', 244 | SK: 'NAME#Squidward', 245 | GSI1: 'CATEGORY#TV', 246 | LSI1: 'ROLE#Character', 247 | id: '24', 248 | name: 'Squidward', 249 | category: 'TV', 250 | role: 'Character', 251 | }, 252 | { 253 | PK: 'USER#25', 254 | SK: 'NAME#Mr. Krabs', 255 | GSI1: 'CATEGORY#TV', 256 | LSI1: 'ROLE#Character', 257 | id: '25', 258 | name: 'Mr. Krabs', 259 | category: 'TV', 260 | role: 'Character', 261 | }, 262 | { 263 | PK: 'USER#26', 264 | SK: 'NAME#Gary', 265 | GSI1: 'CATEGORY#TV', 266 | LSI1: 'ROLE#Character', 267 | id: '26', 268 | name: 'Gary', 269 | category: 'TV', 270 | role: 'Character', 271 | }, 272 | { 273 | PK: 'USER#27', 274 | SK: 'NAME#Larry', 275 | GSI1: 'CATEGORY#TV', 276 | LSI1: 'ROLE#Character', 277 | id: '27', 278 | name: 'Larry', 279 | category: 'TV', 280 | role: 'Character', 281 | }, 282 | { 283 | PK: 'USER#28', 284 | SK: 'NAME#Sandy', 285 | GSI1: 'CATEGORY#TV', 286 | LSI1: 'ROLE#Character', 287 | id: '28', 288 | name: 'Sandy', 289 | category: 'TV', 290 | role: 'Character', 291 | }, 292 | { 293 | PK: 'USER#29', 294 | SK: 'NAME#Plankton', 295 | GSI1: 'CATEGORY#TV', 296 | LSI1: 'ROLE#Character', 297 | id: '29', 298 | name: 'Plankton', 299 | category: 'TV', 300 | role: 'Character', 301 | }, 302 | { 303 | PK: 'USER#30', 304 | SK: 'NAME#Sonic', 305 | GSI1: 'CATEGORY#TV', 306 | LSI1: 'ROLE#Character', 307 | id: '30', 308 | name: 'Sonic', 309 | category: 'TV', 310 | role: 'Character', 311 | }, 312 | { 313 | PK: 'USER#31', 314 | SK: 'NAME#Tails', 315 | GSI1: 'CATEGORY#TV', 316 | LSI1: 'ROLE#Character', 317 | id: '31', 318 | name: 'Tails', 319 | category: 'TV', 320 | role: 'Character', 321 | }, 322 | { 323 | PK: 'USER#32', 324 | SK: 'NAME#Knuckles', 325 | GSI1: 'CATEGORY#TV', 326 | LSI1: 'ROLE#Character', 327 | id: '32', 328 | name: 'Knuckles', 329 | category: 'TV', 330 | role: 'Character', 331 | }, 332 | { 333 | PK: 'USER#33', 334 | SK: 'NAME#Rouge', 335 | GSI1: 'CATEGORY#TV', 336 | LSI1: 'ROLE#Character', 337 | id: '33', 338 | name: 'Rouge', 339 | category: 'TV', 340 | role: 'Character', 341 | }, 342 | { 343 | PK: 'USER#34', 344 | SK: 'NAME#Chaos', 345 | GSI1: 'CATEGORY#TV', 346 | LSI1: 'ROLE#Character', 347 | id: '34', 348 | name: 'Chaos', 349 | category: 'TV', 350 | role: 'Character', 351 | }, 352 | ]; 353 | 354 | module.exports = { 355 | USERS_DATA, 356 | }; 357 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const SlsDynamoDataTransformationsPlugin = require('./serverless-plugin/sls-plugin'); 2 | const utils = require('./src/utils'); 3 | 4 | module.exports = SlsDynamoDataTransformationsPlugin; 5 | module.exports.utils = utils; 6 | -------------------------------------------------------------------------------- /license: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Jit 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynamo-data-transform", 3 | "version": "0.1.11", 4 | "description": "DynamoDB Data Transformation Tool", 5 | "main": "./index.js", 6 | "repository": "https://github.com/jitsecurity/dynamo-data-transform", 7 | "author": "Guy Braunstain <guy.braunstain@jit.io>", 8 | "contributors": [ 9 | "Guy Braunstain (https://github.com/Guy7B)" 10 | ], 11 | "license": "MIT", 12 | "scripts": { 13 | "format": "eslint . --fix", 14 | "lint": "eslint ." 15 | }, 16 | "bugs": { 17 | "url": "https://github.com/jitsecurity/dynamo-data-transform/issues" 18 | }, 19 | "homepage": "https://github.com/jitsecurity/dynamo-data-transform#readme", 20 | "publishConfig": { 21 | "registry": "https://registry.npmjs.org" 22 | }, 23 | "keywords": [ 24 | "dynamodb", 25 | "dynamodb data", 26 | "dynamo", 27 | "dynamo data", 28 | "dynamodt", 29 | "transform", 30 | "transformation", 31 | "dynamo transformations", 32 | "dynamo transformer", 33 | "dynamo migration", 34 | "serverless", 35 | "serverless plugin", 36 | "serverless dynamo", 37 | "sls dyanmo" 38 | ], 39 | "dependencies": { 40 | "@aws-sdk/client-dynamodb": "^3.82.0", 41 | "@aws-sdk/client-s3": "^3.94.0", 42 | "@aws-sdk/lib-dynamodb": "^3.82.0", 43 | "import-jsx": "^4.0.1", 44 | "ink": "^3.2.0", 45 | "ink-big-text": "^1.2.0", 46 | "ink-form": "^1.0.2", 47 | "ink-gradient": "^2.0.0", 48 | "ink-select-input": "^4.2.1", 49 | "minimist": "^1.2.6", 50 | "react": "^17.0.2" 51 | }, 52 | "devDependencies": { 53 | "eslint": "^8.15.0", 54 | "eslint-config-airbnb-base": "^15.0.0" 55 | }, 56 | "bin": { 57 | "dynamodt": "./cli/index.js", 58 | "ddt": "./cli/index.js" 59 | } 60 | } -------------------------------------------------------------------------------- /serverless-plugin/commands.js: -------------------------------------------------------------------------------- 1 | const { COMMAND_DESCRIPTION } = require('../src/config/commands'); 2 | 3 | const COMMAND_OPTIONS = { 4 | dry: { 5 | usage: '--dry', 6 | shortcut: 'd', 7 | required: false, 8 | type: 'boolean', 9 | }, 10 | table: { 11 | usage: 'Specify the name of the table (e.g. "--table TABLE_NAME")', 12 | shortcut: 't', 13 | required: true, 14 | type: 'string', 15 | }, 16 | tNumber: { 17 | usage: 'Specify the version of current transformation (e.g. "--tNumber 1")', 18 | required: true, 19 | type: 'number', 20 | }, 21 | tableNames: { 22 | type: 'string', name: 'tableNames', label: 'Specify table name', initialValue: '', 23 | }, 24 | }; 25 | 26 | module.exports = { 27 | dynamodt: { 28 | usage: 'Run data transformations', 29 | commands: { 30 | init: { 31 | usage: `sls dynamodt init --stage <stage> - ${COMMAND_DESCRIPTION.init}`, 32 | lifecycleEvents: ['init'], 33 | }, 34 | up: { 35 | usage: `sls dynamodt up --stage <stage> - ${COMMAND_DESCRIPTION.up}`, 36 | lifecycleEvents: ['transform'], 37 | options: { 38 | dry: COMMAND_OPTIONS.dry, 39 | }, 40 | }, 41 | prepare: { 42 | usage: `"sls dynamodt prepare --table <table> --tNumber <transformation_number> --stage <stage>" - ${COMMAND_DESCRIPTION.prepare}`, 43 | lifecycleEvents: ['prepare'], 44 | options: { 45 | tNumber: COMMAND_OPTIONS.tNumber, 46 | dry: COMMAND_OPTIONS.dry, 47 | table: COMMAND_OPTIONS.table, 48 | }, 49 | }, 50 | down: { 51 | usage: `sls dynamodt down --table <table> --stage <stage> - ${COMMAND_DESCRIPTION.down}`, 52 | lifecycleEvents: ['rollback'], 53 | options: { 54 | table: COMMAND_OPTIONS.table, 55 | dry: COMMAND_OPTIONS.dry, 56 | }, 57 | }, 58 | history: { 59 | usage: `sls dynamodt history --table <table> --stage <stage> - ${COMMAND_DESCRIPTION.history}`, 60 | lifecycleEvents: ['history'], 61 | options: { 62 | table: COMMAND_OPTIONS.table, 63 | }, 64 | }, 65 | }, 66 | }, 67 | }; 68 | -------------------------------------------------------------------------------- /serverless-plugin/sls-plugin.js: -------------------------------------------------------------------------------- 1 | const { 2 | init, up, down, prepare, history: getHistory, 3 | } = require('../src/command-handlers'); 4 | const commands = require('./commands'); 5 | const { getTableNames } = require('./sls-resources-parser'); 6 | 7 | class SlsDynamoDataTransformationsPlugin { 8 | constructor(serverless, options) { 9 | this.serverless = serverless; 10 | this.provider = serverless.getProvider('aws'); 11 | this.log = (message) => serverless.cli.log.bind(serverless.cli)(`Transformations - ${message}`); 12 | this.options = options; 13 | this.commands = commands; 14 | 15 | // A region environment variable is required for aws sdk 16 | const region = this.serverless.configurationInput.provider.region || process.env.AWS_REGION || 'us-east-1'; 17 | process.env.AWS_REGION = region; 18 | this.hooks = { 19 | 'after:deploy:deploy': this.up.bind(this), 20 | 'dynamodt:init:init': this.init.bind(this), 21 | 'dynamodt:prepare:prepare': this.prepare.bind(this), 22 | 'dynamodt:up:transform': this.up.bind(this), 23 | 'dynamodt:down:rollback': this.rollback.bind(this), 24 | 'dynamodt:history:history': this.getHistory.bind(this), 25 | }; 26 | } 27 | 28 | async init() { 29 | const resources = this.provider.serverless.service.resources.Resources; 30 | const tableNames = getTableNames(resources); 31 | 32 | return init({ tableNames }).then(() => { 33 | console.info('"init" command ran successfully.'); 34 | }).catch((error) => { 35 | console.error('An error has occured while running dynamodt (init).', error.message); 36 | }); 37 | } 38 | 39 | async prepare() { 40 | return prepare(this.options).then(() => { 41 | console.info('"prepare" command ran successfully.'); 42 | }).catch((error) => { 43 | console.error('An error has occured while running dynamodt (prepare).', error.message); 44 | }); 45 | } 46 | 47 | async up() { 48 | return up(this.options).then(() => { 49 | console.info('"up" command ran successfully.'); 50 | }).catch((error) => { 51 | console.error('An error has occured while running dynamodt (up).', error.message); 52 | }); 53 | } 54 | 55 | async rollback() { 56 | return down(this.options).then(() => { 57 | console.info('"down" command run successfully.'); 58 | }).catch((error) => { 59 | console.error('An error has occured while running dynamodt (down).', error.message); 60 | }); 61 | } 62 | 63 | async getHistory() { 64 | return getHistory(this.options).then(() => { 65 | console.info('"history" command run successfully.'); 66 | }).catch((error) => { 67 | console.error('An error has occured while running dynamodt (history).', error.message); 68 | }); 69 | } 70 | } 71 | 72 | module.exports = SlsDynamoDataTransformationsPlugin; 73 | -------------------------------------------------------------------------------- /serverless-plugin/sls-resources-parser.js: -------------------------------------------------------------------------------- 1 | const getTableNames = (resources) => { 2 | return Object.values(resources) 3 | .filter((rValue) => rValue.Type === 'AWS::DynamoDB::Table') 4 | .map((rValue) => rValue.Properties.TableName); 5 | }; 6 | 7 | module.exports = { 8 | getTableNames, 9 | }; 10 | -------------------------------------------------------------------------------- /src/clients/dynamodb.js: -------------------------------------------------------------------------------- 1 | const { DynamoDBClient } = require('@aws-sdk/client-dynamodb'); 2 | const { DynamoDBDocumentClient } = require('@aws-sdk/lib-dynamodb'); 3 | 4 | const getDynamoDBClient = () => { 5 | let ddbClient; 6 | if (process.env.AWS_CUSTOM_ENDPOINT) { 7 | ddbClient = new DynamoDBClient({ 8 | endpoint: process.env.AWS_CUSTOM_ENDPOINT, 9 | }); 10 | } else { 11 | ddbClient = new DynamoDBClient(); 12 | } 13 | return DynamoDBDocumentClient.from(ddbClient); 14 | }; 15 | 16 | module.exports = { 17 | getDynamoDBClient, 18 | }; 19 | -------------------------------------------------------------------------------- /src/clients/index.js: -------------------------------------------------------------------------------- 1 | const { getDynamoDBClient } = require('./dynamodb'); 2 | 3 | module.exports = { 4 | getDynamoDBClient, 5 | }; 6 | -------------------------------------------------------------------------------- /src/command-handlers/down.js: -------------------------------------------------------------------------------- 1 | const { getDynamoDBClient } = require('../clients'); 2 | const { getDataTransformationScriptFullPath } = require('../data-transformation-script-explorer'); 3 | const { getlatestDataTransformationNumber, rollbackTransformation } = require('../services/dynamodb'); // rename folder 4 | const { ddbErrorsWrapper } = require('../services/dynamodb'); 5 | const { getDataFromS3Bucket } = require('../services/s3'); 6 | 7 | const down = async ({ table, dry: isDryRun }) => { 8 | const ddb = getDynamoDBClient(); 9 | 10 | const latestDataTransformationNumber = await getlatestDataTransformationNumber(ddb, table); 11 | if (!latestDataTransformationNumber) { 12 | console.info('No transformation has been executed, there is no need to rollback.'); 13 | return; 14 | } 15 | 16 | const dataTransformationFilePath = await getDataTransformationScriptFullPath( 17 | latestDataTransformationNumber, 18 | table, 19 | ); 20 | 21 | const { transformDown, prepare } = require(dataTransformationFilePath); 22 | 23 | let preparationData = {}; 24 | const shouldUsePreparationData = Boolean(prepare); 25 | if (shouldUsePreparationData) { 26 | const preparationFilePath = `${table}/v${latestDataTransformationNumber}`; 27 | preparationData = await getDataFromS3Bucket(preparationFilePath); 28 | console.info('Running data transformation script using preparation data'); 29 | } 30 | 31 | await transformDown({ ddb, preparationData, isDryRun }); 32 | 33 | if (!isDryRun) { 34 | await rollbackTransformation(ddb, latestDataTransformationNumber, table); 35 | } else { 36 | console.info("It's a dry run"); 37 | } 38 | }; 39 | 40 | module.exports = ddbErrorsWrapper(down); 41 | -------------------------------------------------------------------------------- /src/command-handlers/history.js: -------------------------------------------------------------------------------- 1 | const { getTransformationsRunHistory } = require('../services/dynamodb/transformations-executions-manager'); 2 | const { getDynamoDBClient } = require('../clients'); 3 | const { ddbErrorsWrapper } = require('../services/dynamodb'); 4 | 5 | const getHistory = async ({ table }) => { 6 | const ddb = getDynamoDBClient(); 7 | 8 | const history = await getTransformationsRunHistory(ddb, table); 9 | 10 | console.info(`History for table ${table}`); 11 | const sortedHistory = Object.keys(history) 12 | .sort((a, b) => new Date(a) - new Date(b)) 13 | .map((key) => ({ 14 | Date: new Date(key).toLocaleString(), 15 | Command: history[key].executedCommand, 16 | 'Transformation Number': history[key].transformationNumber, 17 | })); 18 | 19 | console.table(sortedHistory); 20 | }; 21 | 22 | module.exports = ddbErrorsWrapper(getHistory); 23 | -------------------------------------------------------------------------------- /src/command-handlers/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | history: require('./history'), 3 | up: require('./up'), 4 | down: require('./down'), 5 | prepare: require('./prepare'), 6 | init: require('./init'), 7 | }; 8 | -------------------------------------------------------------------------------- /src/command-handlers/init.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | const { DATA_TRANSFORMATIONS_FOLDER_NAME } = require('../config/constants'); 4 | 5 | const createFolderIfNotExist = (folderPath) => { 6 | if (!fs.existsSync(folderPath)) { 7 | console.info(`${folderPath} folder does not exist, creating...`); 8 | fs.mkdirSync(folderPath); 9 | return false; 10 | } 11 | console.info(`${folderPath} already exists`); 12 | return true; 13 | }; 14 | 15 | const initHandler = async ({ tableNames }) => { 16 | const baseTransformationsFolderPath = path.join(process.cwd(), DATA_TRANSFORMATIONS_FOLDER_NAME); 17 | 18 | createFolderIfNotExist(baseTransformationsFolderPath); 19 | 20 | tableNames?.forEach((tableName) => { 21 | const isExist = createFolderIfNotExist(`${baseTransformationsFolderPath}/${tableName}`); 22 | if (!isExist) { 23 | const origin = path.join(__dirname, '../config/transformation-template-file.js'); 24 | const destination = path.join(baseTransformationsFolderPath, tableName, 'v1_script-name.js'); 25 | const originFile = fs.readFileSync(origin, 'utf8'); 26 | const destinationFile = originFile.replace(/{{YOUR_TABLE_NAME}}/g, tableName); 27 | 28 | fs.writeFileSync(destination, destinationFile); 29 | console.info(`Transformation template v1.js file has created for ${tableName}`); 30 | } 31 | }); 32 | }; 33 | 34 | module.exports = initHandler; 35 | -------------------------------------------------------------------------------- /src/command-handlers/prepare.js: -------------------------------------------------------------------------------- 1 | const { uploadDataToS3Bucket } = require('../services/s3'); 2 | const { getDynamoDBClient } = require('../clients'); 3 | const { ddbErrorsWrapper } = require('../services/dynamodb'); 4 | const { getDataTransformationScriptFullPath } = require('../data-transformation-script-explorer'); 5 | const { TRANSFORMATION_NUMBER_PREFIX } = require('../config/constants'); 6 | 7 | const prepareHandler = async ({ table, tNumber, dry: isDryRun }) => { 8 | const preparationPath = await getDataTransformationScriptFullPath( 9 | Number(tNumber), 10 | table, 11 | ); 12 | 13 | const { prepare } = require(preparationPath); 14 | 15 | const ddb = getDynamoDBClient(); 16 | const prepatationDataPath = `${table}/${TRANSFORMATION_NUMBER_PREFIX}${tNumber}`; 17 | const preparationData = await prepare({ ddb, isDryRun }); 18 | if (isDryRun) { 19 | console.info(preparationData, 'preparationData'); 20 | console.info("It's a dry run"); 21 | } else { 22 | await uploadDataToS3Bucket( 23 | prepatationDataPath, 24 | JSON.stringify(preparationData), 25 | ); 26 | } 27 | }; 28 | 29 | module.exports = ddbErrorsWrapper(prepareHandler); 30 | -------------------------------------------------------------------------------- /src/command-handlers/up.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const fs = require('fs').promises; 3 | const { getlatestDataTransformationNumber, hasTransformationRun, syncTransformationRecord } = require('../services/dynamodb/transformations-executions-manager'); 4 | const { getDynamoDBClient } = require('../clients'); 5 | const { getDataFromS3Bucket } = require('../services/s3'); 6 | const { BASE_TRANSFORMATIONS_FOLDER_PATH, TRANSFORMATION_SCRIPT_EXTENSION } = require('../config/constants'); 7 | const { ddbErrorsWrapper } = require('../services/dynamodb'); 8 | const { parseTransformationFileNumber, getTableDataTransformationFiles } = require('../data-transformation-script-explorer'); 9 | 10 | const executeDataTransformation = async (ddb, transformation, table, isDryRun) => { 11 | const { transformationNumber, transformUp, prepare } = transformation; 12 | const isTransformationRun = await hasTransformationRun(ddb, transformationNumber, table); 13 | 14 | if (!isTransformationRun) { 15 | let preparationData = {}; 16 | const shouldUsePreparationData = Boolean(prepare); 17 | if (shouldUsePreparationData) { 18 | const preparationFilePath = `${table}/v${transformationNumber}`; 19 | preparationData = await getDataFromS3Bucket(preparationFilePath); 20 | console.info('Running data transformation script using preparation data'); 21 | } 22 | 23 | const transformationResponse = await transformUp({ ddb, preparationData, isDryRun }); 24 | if (!isDryRun) { 25 | if (transformationResponse) { 26 | await syncTransformationRecord(ddb, transformationNumber, table, transformationResponse?.transformed); 27 | } else { 28 | console.error(`"return" statement is missing or invalid in your transformUp function. 29 | Please provide a transformation response "return { transformed: <number> }". 30 | See documentation for more details.`); 31 | } 32 | } else { 33 | console.info("It's a dry run"); 34 | } 35 | } else { 36 | console.info(`Data Transformation script ${transformationNumber} has already been executed for table ${table}`); 37 | } 38 | }; 39 | 40 | const isGreaterThanLatestTransformationNumber = (fileName, latestDataTransformationNumber) => { 41 | const transformationFileNumber = parseTransformationFileNumber(fileName); 42 | return transformationFileNumber > latestDataTransformationNumber; 43 | }; 44 | 45 | const getScriptsToExecuteForTable = async (table, latestDataTransformationNumber) => { 46 | try { 47 | const transformationFiles = await getTableDataTransformationFiles(table); 48 | 49 | const currentTransformationFiles = transformationFiles.filter((fileName) => { 50 | const isJsFile = path.extname(fileName) === TRANSFORMATION_SCRIPT_EXTENSION; 51 | return isJsFile && isGreaterThanLatestTransformationNumber(fileName, latestDataTransformationNumber); 52 | }); 53 | 54 | const sortedTransformationFiles = currentTransformationFiles.sort((a, b) => { 55 | const aNumber = parseTransformationFileNumber(a); 56 | const bNumber = parseTransformationFileNumber(b); 57 | return aNumber - bNumber; 58 | }); 59 | 60 | const scriptsToExecute = sortedTransformationFiles 61 | .map((fileName) => require( 62 | path.join(BASE_TRANSFORMATIONS_FOLDER_PATH, table, fileName), 63 | )); 64 | 65 | console.info(`Number of data transformation scripts to execute - ${scriptsToExecute.length} for table ${table}.`); 66 | return scriptsToExecute; 67 | } catch (error) { 68 | console.error(`Could not get data transformations scripts for current table - ${table}, latest data transformation number: ${latestDataTransformationNumber}`); 69 | throw error; 70 | } 71 | }; 72 | 73 | const up = async ({ dry: isDryRun }) => { 74 | const ddb = getDynamoDBClient(); 75 | 76 | const tables = await fs.readdir(BASE_TRANSFORMATIONS_FOLDER_PATH); 77 | console.info(`Available tables for transformation: ${tables || 'No tables found'}.`); 78 | 79 | return Promise.all(tables.map(async (table) => { 80 | const latestDataTransformationNumber = await getlatestDataTransformationNumber(ddb, table); 81 | const transformationsToExecute = await getScriptsToExecuteForTable( 82 | table, 83 | latestDataTransformationNumber, 84 | ); 85 | 86 | for (const transformation of transformationsToExecute) { 87 | console.info('Started data transformation ', transformation.transformationNumber, table); 88 | await executeDataTransformation(ddb, transformation, table, isDryRun); 89 | } 90 | })); 91 | }; 92 | 93 | module.exports = ddbErrorsWrapper(up); 94 | -------------------------------------------------------------------------------- /src/config/commands.js: -------------------------------------------------------------------------------- 1 | const COMMAND_DESCRIPTION = { 2 | up: 'Run the pending data transformation scripts', 3 | down: 'Rollback the last data transformation', 4 | history: 'Show the history of data transformations', 5 | prepare: 'Prepare data for the data transformation script', 6 | init: 'Initialize the data-transformations folder', 7 | }; 8 | 9 | module.exports = { 10 | COMMAND_DESCRIPTION, 11 | }; 12 | -------------------------------------------------------------------------------- /src/config/constants.js: -------------------------------------------------------------------------------- 1 | const DATA_TRANSFORMATIONS_FOLDER_NAME = 'data-transformations'; 2 | const BASE_TRANSFORMATIONS_FOLDER_PATH = `${process.cwd()}/${DATA_TRANSFORMATIONS_FOLDER_NAME}`; 3 | const TRANSFORMATION_SCRIPT_EXTENSION = '.js'; 4 | const TRANSFORMATION_NUMBER_PREFIX = 'v'; 5 | const TRANSFORMATION_NAME_SEPARATOR = '_'; 6 | 7 | const DATA_TRANSFORMATIONS_KEY_ID = 'DataTransformations'; 8 | 9 | module.exports = { 10 | DATA_TRANSFORMATIONS_FOLDER_NAME, 11 | BASE_TRANSFORMATIONS_FOLDER_PATH, 12 | TRANSFORMATION_SCRIPT_EXTENSION, 13 | TRANSFORMATION_NUMBER_PREFIX, 14 | TRANSFORMATION_NAME_SEPARATOR, 15 | DATA_TRANSFORMATIONS_KEY_ID, 16 | }; 17 | -------------------------------------------------------------------------------- /src/config/transformation-template-file.js: -------------------------------------------------------------------------------- 1 | const { utils } = require('dynamo-data-transform'); 2 | 3 | const TABLE_NAME = '{{YOUR_TABLE_NAME}}'; 4 | 5 | /** 6 | * @param {DynamoDBDocumentClient} ddb - dynamo db client of @aws-sdk https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-dynamodb 7 | * @param {boolean} isDryRun 8 | * @returns the number of transformed items { transformed: number } 9 | * 10 | */ 11 | const transformUp = async ({ ddb, isDryRun }) => { 12 | // Replace this with your own logic 13 | const addNewFieldToItem = (item) => { 14 | const updatedItem = { ...item, newField: 'value' }; 15 | return updatedItem; 16 | }; 17 | return utils.transformItems(ddb, TABLE_NAME, addNewFieldToItem, isDryRun); 18 | }; 19 | 20 | const transformDown = async ({ ddb, isDryRun }) => { 21 | // Replace this function with your own logic 22 | const removeFieldFromItem = (item) => { 23 | const { newField, ...oldItem } = item; 24 | return oldItem; 25 | }; 26 | return utils.transformItems(ddb, TABLE_NAME, removeFieldFromItem, isDryRun); 27 | }; 28 | 29 | module.exports = { 30 | transformUp, 31 | transformDown, 32 | // prepare, // export this function only if you need preparation data for the transformation 33 | transformationNumber: 1, 34 | }; 35 | 36 | /** 37 | * For more data transformation scripts examples, see: 38 | * https://github.com/jitsecurity/dynamo-data-transform/tree/main/examples/serverless-localstack 39 | * 40 | */ 41 | -------------------------------------------------------------------------------- /src/data-transformation-script-explorer.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const fs = require('fs').promises; 3 | 4 | const { 5 | BASE_TRANSFORMATIONS_FOLDER_PATH, TRANSFORMATION_SCRIPT_EXTENSION, 6 | TRANSFORMATION_NAME_SEPARATOR, TRANSFORMATION_NUMBER_PREFIX, 7 | } = require('./config/constants'); 8 | 9 | const parseTransformationFileNumber = (fileName) => { 10 | const filetransformationNumber = Number( 11 | path.basename(fileName, TRANSFORMATION_SCRIPT_EXTENSION) 12 | .split(TRANSFORMATION_NAME_SEPARATOR)[0] 13 | .replace(TRANSFORMATION_NUMBER_PREFIX, ''), 14 | ); 15 | 16 | return filetransformationNumber; 17 | }; 18 | 19 | const getTableDataTransformationFiles = async (table) => { 20 | const transformationFiles = await fs.readdir(path.join(BASE_TRANSFORMATIONS_FOLDER_PATH, table)); 21 | return transformationFiles; 22 | }; 23 | 24 | const getDataTransformationScriptFileName = async (transformationNumber, table) => { 25 | const transformationFiles = await getTableDataTransformationFiles(table); 26 | 27 | const fileName = transformationFiles.find((currFileName) => { 28 | const currFiletransformationNumber = parseTransformationFileNumber(currFileName); 29 | return currFiletransformationNumber === transformationNumber; 30 | }); 31 | 32 | if (!fileName) { 33 | throw new Error(`Could not find data transformation script for transformation number ${transformationNumber}`); 34 | } 35 | 36 | return fileName; 37 | }; 38 | 39 | const getDataTransformationScriptFullPath = async (transformationNumber, table) => { 40 | const fileName = await getDataTransformationScriptFileName(transformationNumber, table); 41 | return path.join(BASE_TRANSFORMATIONS_FOLDER_PATH, table, fileName); 42 | }; 43 | 44 | module.exports = { 45 | parseTransformationFileNumber, 46 | getDataTransformationScriptFileName, 47 | getDataTransformationScriptFullPath, 48 | getTableDataTransformationFiles, 49 | }; 50 | -------------------------------------------------------------------------------- /src/services/dynamodb/errorHandlers.js: -------------------------------------------------------------------------------- 1 | const { ResourceNotFoundException } = require('@aws-sdk/client-dynamodb'); 2 | 3 | const ddbErrorsWrapper = (func) => { 4 | return async (...args) => { 5 | try { 6 | return await func(...args); 7 | } catch (error) { 8 | if (error instanceof ResourceNotFoundException) { 9 | throw new Error('Table does not exist. Please verify the table name.'); 10 | } 11 | throw error; 12 | } 13 | }; 14 | }; 15 | 16 | module.exports = { 17 | ddbErrorsWrapper, 18 | }; 19 | -------------------------------------------------------------------------------- /src/services/dynamodb/index.js: -------------------------------------------------------------------------------- 1 | const transformationsExecutionManager = require('./transformations-executions-manager'); 2 | const errorHandlers = require('./errorHandlers'); 3 | 4 | module.exports = { 5 | ...transformationsExecutionManager, 6 | ...errorHandlers, 7 | }; 8 | -------------------------------------------------------------------------------- /src/services/dynamodb/transformations-executions-manager.js: -------------------------------------------------------------------------------- 1 | const { GetCommand, PutCommand } = require('@aws-sdk/lib-dynamodb'); 2 | const { DATA_TRANSFORMATIONS_KEY_ID } = require('../../config/constants'); 3 | const getTableKeySchema = require('../../utils/getTableKeySchema'); 4 | 5 | const getDataTransformationKey = async (table) => { 6 | const keySchema = await getTableKeySchema(table); 7 | const dataTransformationKey = keySchema.reduce((acc, val) => ({ 8 | ...acc, 9 | [val.AttributeName]: DATA_TRANSFORMATIONS_KEY_ID, 10 | }), {}); 11 | return dataTransformationKey; 12 | }; 13 | 14 | const getDataTransformationRecord = async (ddb, table) => { 15 | const dataTransformationKey = await getDataTransformationKey(table); 16 | const getCommand = new GetCommand({ 17 | TableName: table, 18 | Key: dataTransformationKey, 19 | }); 20 | 21 | try { 22 | const { Item } = await ddb.send(getCommand); 23 | 24 | if (!Item) console.info(`No data transformations record for table ${table}`); 25 | return Item || dataTransformationKey; 26 | } catch (error) { 27 | console.error(`Could not get data transformations record for table ${table}`); 28 | throw error; 29 | } 30 | }; 31 | 32 | const hasTransformationRun = async (ddb, transformationNumber, table) => { 33 | const transformationRecord = await getDataTransformationRecord(ddb, table); 34 | 35 | const hasTransformationExecuted = transformationRecord 36 | .executionState?.includes(transformationNumber); 37 | 38 | return !!hasTransformationExecuted; 39 | }; 40 | 41 | const syncTransformationRecord = async (ddb, transformationNumber, table, transformed) => { 42 | const transformationRecord = await getDataTransformationRecord(ddb, table); 43 | const executionState = transformationRecord?.executionState || []; 44 | const transformationsRunHistory = transformationRecord ? transformationRecord.TransformationsRunHistory : {}; 45 | const updatedItem = { 46 | ...transformationRecord, 47 | executionState: [...executionState, transformationNumber], 48 | TransformationsRunHistory: { 49 | ...transformationsRunHistory, 50 | [new Date().toISOString()]: { 51 | transformationNumber, 52 | executedCommand: 'up', 53 | transformed, 54 | }, 55 | }, 56 | }; 57 | 58 | const putCommand = new PutCommand({ 59 | TableName: table, 60 | Item: updatedItem, 61 | }); 62 | 63 | await ddb.send(putCommand); 64 | }; 65 | 66 | const getTransformationsRunHistory = async (ddb, table) => { 67 | const transformationRecord = await getDataTransformationRecord(ddb, table); 68 | return transformationRecord?.TransformationsRunHistory || {}; 69 | }; 70 | 71 | const rollbackTransformation = async (ddb, transformationNumber, table) => { 72 | const transformationRecord = await getDataTransformationRecord(ddb, table); 73 | if (!transformationRecord) { 74 | return; 75 | } 76 | const executionState = transformationRecord?.executionState || []; 77 | const updatedExecutionState = executionState.filter( 78 | (executedNumber) => executedNumber !== transformationNumber, 79 | ); 80 | const transformationsRunHistory = transformationRecord ? transformationRecord.TransformationsRunHistory : {}; 81 | 82 | const updatedItem = { 83 | ...transformationRecord, 84 | executionState: updatedExecutionState, 85 | TransformationsRunHistory: { 86 | ...transformationsRunHistory, 87 | [new Date().toISOString()]: { 88 | transformationNumber, 89 | executedCommand: 'down', 90 | }, 91 | }, 92 | }; 93 | 94 | const putCommand = new PutCommand({ 95 | TableName: table, 96 | Item: updatedItem, 97 | }); 98 | 99 | return ddb.send(putCommand); 100 | }; 101 | 102 | const getlatestDataTransformationNumber = async (ddb, table) => { 103 | console.info(`Getting latest data transformation number for table ${table}.`); 104 | 105 | try { 106 | const transformationRecord = await getDataTransformationRecord(ddb, table); 107 | const latestDataTransformationNumber = transformationRecord.executionState 108 | ?.sort((a, b) => b - a)[0]; 109 | 110 | if (!latestDataTransformationNumber) { 111 | console.info('No data transformations have been run'); 112 | return 0; 113 | } 114 | return latestDataTransformationNumber; 115 | } catch (error) { 116 | console.error( 117 | `An error has occured while trying to get latest data transformation number for table ${table}`, 118 | ); 119 | throw error; 120 | } 121 | }; 122 | 123 | module.exports = { 124 | getlatestDataTransformationNumber, 125 | getDataTransformationRecord, 126 | rollbackTransformation, 127 | syncTransformationRecord, 128 | hasTransformationRun, 129 | getTransformationsRunHistory, 130 | }; 131 | -------------------------------------------------------------------------------- /src/services/s3.js: -------------------------------------------------------------------------------- 1 | const { 2 | S3Client, PutObjectCommand, GetObjectCommand, 3 | } = require('@aws-sdk/client-s3'); 4 | 5 | const getS3Client = () => { 6 | let s3Client; 7 | if (process.env.AWS_CUSTOM_ENDPOINT) { 8 | s3Client = new S3Client({ 9 | endpoint: process.env.AWS_CUSTOM_ENDPOINT, 10 | forcePathStyle: true, 11 | }); 12 | } else { 13 | s3Client = new S3Client(); 14 | } 15 | return s3Client; 16 | }; 17 | 18 | const uploadDataToS3Bucket = async (filePath, body) => { 19 | try { 20 | if (!body) { 21 | throw new Error(`Empty body for preparation data is not allowed. 22 | Please fix "prepare" function and make sure you return a valid object.`); 23 | } 24 | const s3Client = getS3Client(); 25 | const command = new PutObjectCommand({ 26 | Bucket: process.env.PREPARATION_DATA_BUCKET || 'transformations-preparation-data', 27 | Key: filePath, 28 | Body: body, 29 | ServerSideEncryption: 'AES256', 30 | ACL: 'private', 31 | ContentType: 'application/json', 32 | }); 33 | 34 | const dataUpload = await s3Client.send(command); 35 | return dataUpload; 36 | } catch (error) { 37 | console.error('Error uploading data to S3 bucket', error.message); 38 | throw error; 39 | } 40 | }; 41 | 42 | const getS3ObjectPromisified = (Bucket, Key) => { 43 | const s3Client = getS3Client(); 44 | 45 | return new Promise((resolve, reject) => { 46 | const getObjectCommand = new GetObjectCommand({ Bucket, Key }); 47 | 48 | s3Client.send(getObjectCommand).then((data) => { 49 | const responseDataChunks = []; 50 | const body = data.Body; 51 | body.once('error', (err) => reject(err)); 52 | 53 | body.on('data', (chunk) => responseDataChunks.push(chunk)); 54 | 55 | body.once('end', () => resolve(responseDataChunks.join(''))); 56 | }).catch((error) => { 57 | reject(error); 58 | }); 59 | }); 60 | }; 61 | 62 | const getDataFromS3Bucket = async (filePath) => { 63 | try { 64 | const content = await getS3ObjectPromisified( 65 | process.env.PREPARATION_DATA_BUCKET || 'transformations-preparation-data', 66 | filePath, 67 | ); 68 | if (!content) { 69 | throw new Error(`Received empty body for preparation data. 70 | Please rerun prepare script and make sure you return a valid object.`); 71 | } 72 | return content; 73 | } catch (error) { 74 | console.error(`Error getting data for path: 75 | '${filePath}' from S3 bucket: 76 | ${process.env.PREPARATION_DATA_BUCKET} \n`, error.message); 77 | throw error; 78 | } 79 | }; 80 | 81 | module.exports = { 82 | uploadDataToS3Bucket, 83 | getDataFromS3Bucket, 84 | }; 85 | -------------------------------------------------------------------------------- /src/utils/batchDeleteItems.js: -------------------------------------------------------------------------------- 1 | const { BatchWriteCommand } = require('@aws-sdk/lib-dynamodb'); 2 | 3 | const getKeySchemaOfItem = (item, keySchema) => { 4 | const keySchemaOfItem = Object.keys(item).reduce((acc, key) => { 5 | if (keySchema.find(({ AttributeName }) => AttributeName === key)) { 6 | acc[key] = item[key]; 7 | } 8 | return acc; 9 | }, {}); 10 | 11 | return keySchemaOfItem; 12 | }; 13 | 14 | const batchDeleteItems = async (ddb, tableName, items, keySchema) => { 15 | const params = { 16 | RequestItems: { 17 | [tableName]: items.map((item) => ({ 18 | DeleteRequest: { 19 | Key: getKeySchemaOfItem(item, keySchema), 20 | }, 21 | })), 22 | }, 23 | ReturnConsumedCapacity: 'TOTAL', 24 | ReturnItemCollectionMetrics: 'SIZE', 25 | }; 26 | const batchWriteCommand = new BatchWriteCommand(params); 27 | return ddb.send(batchWriteCommand); 28 | }; 29 | 30 | module.exports = batchDeleteItems; 31 | -------------------------------------------------------------------------------- /src/utils/batchWriteItems.js: -------------------------------------------------------------------------------- 1 | const { BatchWriteCommand } = require('@aws-sdk/lib-dynamodb'); 2 | 3 | const batchWriteItems = async (ddb, tableName, items) => { 4 | const params = { 5 | RequestItems: { 6 | [tableName]: items.map((item) => ({ 7 | PutRequest: { 8 | Item: item, 9 | }, 10 | })), 11 | }, 12 | ReturnConsumedCapacity: 'TOTAL', 13 | ReturnItemCollectionMetrics: 'SIZE', 14 | }; 15 | const batchWriteCommand = new BatchWriteCommand(params); 16 | return ddb.send(batchWriteCommand); 17 | }; 18 | 19 | module.exports = batchWriteItems; 20 | -------------------------------------------------------------------------------- /src/utils/deleteItems.js: -------------------------------------------------------------------------------- 1 | const batchDeleteItems = require('./batchDeleteItems'); 2 | const getTableKeySchema = require('./getTableKeySchema'); 3 | const { getUnprocessedItems } = require('./responseUtils'); 4 | 5 | const MAX_ITEMS_PER_BATCH = 25; 6 | 7 | const deleteItems = async (ddb, tableName, items, isDryRun) => { 8 | if (isDryRun) { 9 | console.info(`Dry run: would have deleted ${items.length} items from ${tableName}`, items); 10 | return; 11 | } 12 | 13 | const batches = []; 14 | for (let i = 0; i < items.length; i += MAX_ITEMS_PER_BATCH) { 15 | batches.push(items.slice(i, i + MAX_ITEMS_PER_BATCH)); 16 | } 17 | 18 | const keySchema = await getTableKeySchema(tableName); 19 | 20 | try { 21 | const response = await Promise.all( 22 | batches.map((batch) => batchDeleteItems(ddb, tableName, batch, keySchema)), 23 | ); 24 | const unprocessedItems = getUnprocessedItems(response); 25 | 26 | if (unprocessedItems.length > 0) { 27 | console.error(`Failed to delete ${unprocessedItems.length} items to ${tableName}`, unprocessedItems); 28 | } 29 | 30 | return { transformed: items.length }; 31 | } catch (error) { 32 | console.error(`An error has occurred in delete items for ${tableName}`, error); 33 | throw error; 34 | } 35 | }; 36 | 37 | module.exports = deleteItems; 38 | -------------------------------------------------------------------------------- /src/utils/getItems.js: -------------------------------------------------------------------------------- 1 | const { ScanCommand } = require('@aws-sdk/lib-dynamodb'); 2 | const { DATA_TRANSFORMATIONS_KEY_ID } = require('../config/constants'); 3 | 4 | const filterTransformationRecord = (items) => { 5 | return items.filter((item) => !Object.values(item).includes(DATA_TRANSFORMATIONS_KEY_ID)); 6 | }; 7 | 8 | const getItems = async (ddb, lastEvalKey, tableName) => { 9 | const params = { 10 | TableName: tableName, 11 | ExclusiveStartKey: lastEvalKey, 12 | Limit: 25, 13 | }; 14 | 15 | const scanCommand = new ScanCommand(params); 16 | const scanResponse = await ddb.send(scanCommand); 17 | scanResponse.Items = filterTransformationRecord(scanResponse.Items); 18 | 19 | return scanResponse; 20 | }; 21 | 22 | module.exports = getItems; 23 | -------------------------------------------------------------------------------- /src/utils/getTableKeySchema.js: -------------------------------------------------------------------------------- 1 | const { DescribeTableCommand } = require('@aws-sdk/client-dynamodb'); 2 | const { getDynamoDBClient } = require('../clients'); 3 | 4 | const getTableKeySchema = async (table) => { 5 | try { 6 | const ddbClient = getDynamoDBClient(); 7 | const { Table } = await ddbClient.send(new DescribeTableCommand({ TableName: table })); 8 | return Table.KeySchema; 9 | } catch (error) { 10 | console.error(`Could not get key schema of table ${table}`); 11 | throw error; 12 | } 13 | }; 14 | 15 | module.exports = getTableKeySchema; 16 | -------------------------------------------------------------------------------- /src/utils/index.js: -------------------------------------------------------------------------------- 1 | const getItems = require('./getItems'); 2 | const transformItems = require('./transformItems'); 3 | const transactWriteItems = require('./transactWriteItems'); 4 | const batchWriteItems = require('./batchWriteItems'); 5 | const insertItems = require('./insertItems'); 6 | const deleteItems = require('./deleteItems'); 7 | 8 | module.exports = { 9 | getItems, 10 | transactWriteItems, 11 | transformItems, 12 | batchWriteItems, 13 | insertItems, 14 | deleteItems, 15 | }; 16 | -------------------------------------------------------------------------------- /src/utils/insertItems.js: -------------------------------------------------------------------------------- 1 | const batchWriteItems = require('./batchWriteItems'); 2 | const { getUnprocessedItems } = require('./responseUtils'); 3 | 4 | const MAX_ITEMS_PER_BATCH = 25; 5 | 6 | const insertItems = async (ddb, tableName, items, isDryRun) => { 7 | if (isDryRun) { 8 | console.info(`Dry run: would have seeded ${items.length} items to ${tableName}`, items); 9 | return; 10 | } 11 | 12 | const batches = []; 13 | for (let i = 0; i < items.length; i += MAX_ITEMS_PER_BATCH) { 14 | batches.push(items.slice(i, i + MAX_ITEMS_PER_BATCH)); 15 | } 16 | 17 | try { 18 | const response = await Promise.all( 19 | batches.map((batch) => batchWriteItems(ddb, tableName, batch)), 20 | ); 21 | const unprocessedItems = getUnprocessedItems(response); 22 | 23 | if (unprocessedItems.length > 0) { 24 | console.error(`Failed to seed ${unprocessedItems.length} items to ${tableName}`, JSON.stringify(unprocessedItems, null, 2)); 25 | } 26 | 27 | return { transformed: items.length }; 28 | } catch (error) { 29 | console.error(`An error has occured in seed items for ${tableName}`); 30 | throw error; 31 | } 32 | }; 33 | 34 | module.exports = insertItems; 35 | -------------------------------------------------------------------------------- /src/utils/responseUtils.js: -------------------------------------------------------------------------------- 1 | const getUnprocessedItems = (response) => { 2 | const unprocessedItems = response.reduce((acc, { UnprocessedItems }) => { 3 | if (Object.keys(UnprocessedItems)?.length) { 4 | acc.push(UnprocessedItems); 5 | } 6 | return acc; 7 | }, []); 8 | 9 | return unprocessedItems; 10 | }; 11 | 12 | module.exports = { 13 | getUnprocessedItems, 14 | }; 15 | -------------------------------------------------------------------------------- /src/utils/transactWriteItems.js: -------------------------------------------------------------------------------- 1 | const { TransactWriteCommand } = require('@aws-sdk/lib-dynamodb'); 2 | 3 | async function transactWriteItems(ddb, tableName, records) { 4 | try { 5 | const transactItems = records.map((record) => { 6 | const { action = 'put', key, ...item } = record; 7 | switch (action) { 8 | case 'put': 9 | return { 10 | Put: { 11 | TableName: tableName, 12 | Item: item, 13 | }, 14 | }; 15 | case 'delete': 16 | return { 17 | Delete: { 18 | TableName: tableName, 19 | Key: key, 20 | }, 21 | }; 22 | default: 23 | throw new Error(`Unknown action: ${action}`); 24 | } 25 | }); 26 | 27 | const transactWriteCommand = new TransactWriteCommand({ 28 | TransactItems: transactItems, 29 | ReturnConsumedCapacity: 'TOTAL', 30 | ReturnItemCollectionMetrics: 'SIZE', 31 | }); 32 | 33 | return await ddb.send(transactWriteCommand); 34 | } catch (e) { 35 | console.error(`Error while writing to table ${tableName}`, e); 36 | throw e; 37 | } 38 | } 39 | 40 | module.exports = transactWriteItems; 41 | -------------------------------------------------------------------------------- /src/utils/transformItems.js: -------------------------------------------------------------------------------- 1 | const getItems = require('./getItems'); 2 | const batchWriteItems = require('./batchWriteItems'); 3 | 4 | const transformItems = async (ddb, tableName, transformer, isDryRun) => { 5 | let lastEvalKey; 6 | let transformedItemsKeys = []; 7 | 8 | let scannedAllItems = false; 9 | 10 | while (!scannedAllItems) { 11 | const { Items, LastEvaluatedKey } = await getItems(ddb, lastEvalKey, tableName); 12 | lastEvalKey = LastEvaluatedKey; 13 | 14 | const updatedItems = Items.map(transformer).filter(i => i !== undefined); 15 | 16 | if (!isDryRun && updatedItems.length > 0) { 17 | if (updatedItems?.length) await batchWriteItems(ddb, tableName, updatedItems); 18 | transformedItemsKeys = transformedItemsKeys.concat(updatedItems.map((item) => `${item.PK}-${item.SK}`)); 19 | } else { 20 | console.info(updatedItems, 'updatedItems'); 21 | } 22 | 23 | scannedAllItems = !lastEvalKey; 24 | } 25 | 26 | return { transformed: transformedItemsKeys.length }; 27 | }; 28 | 29 | module.exports = transformItems; 30 | --------------------------------------------------------------------------------