├── .eslintignore ├── .eslintrc.json ├── .gitallowed ├── .gitignore ├── .lintstagedrc ├── .npmignore ├── .prettierignore ├── .prettierrc ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── bin └── dms-cdk.ts ├── cdk.json ├── conf └── task_settings.json ├── docs ├── dms-architecture.png ├── dms-architecture.xml └── dms-cdk-classes.png ├── example └── demo-dms-stack.ts ├── jest.config.js ├── lib ├── dms_props.ts ├── dms_replication.ts ├── dms_stack.ts └── rules_props.ts ├── package-lock.json ├── package.json ├── test ├── dms-replication.test.ts └── dms-stack.test.ts └── tsconfig.json /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .idea 3 | cdk.out 4 | # ignore compiled files 5 | *.d.ts 6 | *.js -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es2021": true 5 | }, 6 | "extends": [ 7 | "airbnb-base", 8 | "eslint-config-prettier", 9 | "plugin:jest/recommended", 10 | "plugin:import/typescript" 11 | ], 12 | "parser": "@typescript-eslint/parser", 13 | "parserOptions": { 14 | "ecmaVersion": 12, 15 | "sourceType": "module" 16 | }, 17 | "plugins": [ 18 | "@typescript-eslint", 19 | "eslint-plugin-prettier" 20 | ], 21 | "rules": { 22 | "prettier/prettier": "error", 23 | "no-new": "off", 24 | "import/extensions": [ 25 | "error", 26 | "ignorePackages", 27 | { 28 | "js": "never", 29 | "jsx": "never", 30 | "ts": "never", 31 | "tsx": "never" 32 | } 33 | ] 34 | } 35 | } -------------------------------------------------------------------------------- /.gitallowed: -------------------------------------------------------------------------------- 1 | "111111111111" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !jest.config.js 3 | *.d.ts 4 | node_modules 5 | 6 | # CDK asset staging directory 7 | .cdk.staging 8 | cdk.out 9 | .cdk.json 10 | .npmrc 11 | 12 | # IDE files 13 | .idea 14 | .vscode 15 | *.log 16 | *~ 17 | /.eslintcache 18 | -------------------------------------------------------------------------------- /.lintstagedrc: -------------------------------------------------------------------------------- 1 | { 2 | "**/*.+(js|jsx|css|less|scss|ts|tsx|md)": [ 3 | "prettier --write", 4 | "git add" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .idea 3 | cdk.out 4 | # ignore compiled files 5 | *.d.ts 6 | *.js 7 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 120, 3 | "proseWrap": "preserve", 4 | "semi": true, 5 | "singleQuote": true, 6 | "useTabs": false, 7 | "tabWidth": 2, 8 | "arrowParens": "avoid", 9 | "trailingComma": "es5" 10 | } 11 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | This CDK application contains CDK constructs to provision AWS DMS (data migration service) related resources. The solution provided is primarily based on RDS MySQL database as the source and target database used for migration. The 4 | 5 | 1. A DMS Subnet group : Subnet group allows DMS service to choose the subnet and IP address within the subnet. DMS service uses the IP addresses from this subnet group when provisioning resources like DMS replication instance. 6 | 1. A DMS Replication Instance : This is an managed instances where DMS replication tasks are run 7 | 1. DMS database endpoints : Database connections to source and target database 8 | 1. DMS task(s) : Task(s) for performing data replication with configurable settings 9 | 10 | # Pre-requisite 11 | 12 | 1. Connectivity: The DMS service and acts as "hub" between the source and target database. So, a connectivity is needed from DMS service to both the source and database . 13 | 14 | ___Note: If the source database in located on-premise ensure that the port (default 3306 for mysql) is opened by your corporate firewall. For configuring target endpoint, ensure that your target RDS security group allows this connectivity. The DMS service automatically validates the endpoint for connectivity during installation.___ 15 | 16 | 1. Secrets Manager to store database configurations : The solution assumes all database configurations are stored in the AWS Secrets manager. Configure the secrets manager for both the source and target databases with necessary parameters like hostname, port, database, passwords needed for establishing the connectivity. Use command below to check the secrets in your account. 17 | 18 | ``` 19 | aws secretsmanager list-secrets 20 | 21 | ``` 22 | 23 | 3. VPC endpoints for DMS, Secrets Manager. This ensures traffic is routed via AWS backbone and provides additional security. To check the VPC endpoint run the command below and check for json output for service name like 'com.amazonaws..dms' and 'com.amazonaws..secretsmanager' 24 | 25 | ``` 26 | aws ec2 describe-vpc-endpoints | grep dms 27 | aws ec2 describe-vpc-endpoints | grep secretsmanager 28 | 29 | ``` 30 | 4. CDK v2.30.x+ 31 | 32 | 5. Nodejs v 17.x+ 33 | 34 | 6. Source and target databases (MySQL v8, Oracle and Aurora PostGresSQL) : Check the DMS endpoint in the **AWS Console > Services > DMS** and make sure the test connectiity is working. 35 | 36 | # Solution Architecture 37 | 38 | High level architecture for DMS is shown below. The main components involved are 39 | 40 | 1. A dedicated network (VPN or Direct connect) with a connectivity to the customers VPC (Virtual Private Cloud) 41 | 1. Source database preferably with a dedicated slave. This helps to offload the database load coming from read operations during migration. 42 | 1. A target RDS database instance where data is migrated. 43 | 1. Source and target DMS endpoints with database credentials stored in AWS Secrets Manager 44 | 1. DMS replication instance and a number of replication tasks for replicating the data 45 | 1. DMS subnet group is where DMS service provisions resources into your subnet for connecting to databases. Remember that the DMS service itself is hosted inside AWS. 46 | 47 | 48 | ![DMS Architecture](./docs/dms-architecture.png) 49 | 50 | # Installation 51 | 52 | Ensure that all pre-requisites are in place. The source code is repository is located at https://github.com/aws-samples/dms-cdk/ 53 | 54 | 1. Install Nodejs and npm. See the link https://nodejs.org/en/download/package-manager/#macos. To check the version use the command below 55 | ``` 56 | node --version 57 | ``` 58 | 59 | 1. Install CDK for typescript (https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html) or use the command below to install and check the version of CDK 60 | ``` 61 | npm install -g aws-cdk 62 | 63 | npm audit fix --force 64 | 65 | cdk --version 66 | 67 | ``` 68 | 1. Create a service role "dms-vpc-role" : A service role named "dms-vpc-role" is needed if your are deploying the DMS for the first time via CLI or cloudformation in your account. The following steps are necessary to deploy the "dms-vpc-role" IAM role. For more details see the link [dms-vpc-role](https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Security.html#CHAP_Security.APIRole) 69 | - Create json policy file - dmsAssumeRolePolicyDocument.json - with the following content: 70 | ``` 71 | { 72 | "Version": "2012-10-17", 73 | "Statement": [ 74 | { 75 | "Effect": "Allow", 76 | "Principal": { 77 | "Service": "dms.amazonaws.com" 78 | }, 79 | "Action": "sts:AssumeRole" 80 | } 81 | ] 82 | } 83 | ``` 84 | - Create the required IAM role (dms-vpc-role) 85 | ``` 86 | aws iam create-role --role-name dms-vpc-role --assume-role-policy-document file://dmsAssumeRolePolicyDocument.json 87 | ``` 88 | 89 | - Attach the existing policy to the new role 90 | ``` 91 | aws iam attach-role-policy --role-name dms-vpc-role --policy-arn arn:aws:iam::aws:policy/service-role/AmazonDMSVPCManagementRole 92 | 93 | ``` 94 | 95 | 1. Create an aws profile named 'dms' (if you do not have one) 96 | ``` 97 | aws configure --profile dms 98 | ``` 99 | 100 | 1. Configure cdk.json file in the project root folder. The cdk.json file is where you configure the target account to deploy your solution in addition to settings like vpc, subnet id, database schema for migrating the data. 101 | ___Example of cdk.json___ 102 | ``` 103 | "context": { 104 | "environment": "dev", 105 | "account": "111111111111", 106 | 107 | "dev": { 108 | "region": "eu-central-1", 109 | "vpcId": "vpc-xxxxxxxxxxxxx", 110 | "subnetIds": [ 111 | "subnet-xxxxxxxxxxxxxxxxa", 112 | "subnet-xxxxxxxxxxxxxxxxb" 113 | ], 114 | "vpcSecurityGroupIds": [ 115 | "sg-xxxxxxxxxxxxxxxx" 116 | ], 117 | "schemas": [ 118 | { 119 | "name": "demo-src-db", 120 | "sourceSecretsManagerSecretId": "arn:aws:secretsmanager:eu-central-1:111111111111:secret:dev/mysql/aaa-xxxxpp", 121 | "targetSecretsManagerSecretId": "arn:aws:secretsmanager:eu-central-1:111111111111:secret:dev/mysql/bbb-xxxxpp" 122 | } 123 | ], 124 | "replicationInstanceClass": "dms.r5.4xlarge", 125 | "replicationInstanceIdentifier": "dms-dev-eu", 126 | "replicationSubnetGroupIdentifier": "dms-dev-subnet-eu", 127 | "replicationTaskSettings": { 128 | }, 129 | "migrationType": "full-load" 130 | } 131 | } 132 | 133 | ``` 134 | 1. Go to your project root directory. Compile and test the solution. 135 | 136 | ``` 137 | cd dms-cdk 138 | 139 | npm install 140 | 141 | npm run build && test 142 | ``` 143 | 144 | 1. Deploy the solution to your account based on cdk.json configuration. Use the 'dms' profile created previously in the cdk deploy command 145 | ``` 146 | npx cdk deploy --profile dms 147 | ``` 148 | 149 | 1. Post validation : If the deployment is sucessful you will see the cloudformation stack under **AWS Services > Cloudformation** in AWS console. Following resources are created by this deployment. 150 | - AWS::DMS::ReplicationSubnetGroup 151 | - AWS::DMS::ReplicationInstance 152 | - AWS::IAM::Role 153 | - AWS::IAM::Policy 154 | - AWS::DMS::Endpoint 155 | - AWS::DMS::ReplicationTask 156 | 157 | # CDK construct Overview 158 | 159 | The solution described in this post relies mainly on 2 classes - DMSReplication and DMStack and uses out of the box CDK construct library '@aws-cdk/aws-dms' 160 | The solution is primarily designed for RDS MySQL database. However, it can easily be adopted or extended for use with other databases like PostgreSQL or Oracle. 161 | 162 | - DMSReplication class is a construct responsible for creating resources such as replication instance, task settings, subnet-group and IAM role for accessing AWS Secrets Manager. Note that all database credentials are stored in AWS secrets manager. 163 | 164 | - DMSStack class is a stack that leverages DMSReplication construct to provision the DMS resource(s) based on parameters defined in cdk.json 165 | 166 | - ContextProps is an interface that helps to map input parameters from cdk.json file in a type safe manner. The cdk.json file includes settings related to DMS resources like replication instance, task settings, logging etc. ContextProps also defines default values which can be overridden by you by defining it in the cdk.json file 167 | 168 | ![dms-cdk-classes](./docs/dms-cdk-classes.png) 169 | 170 | 171 | ``` 172 | import * as cdk from 'aws-cdk-lib'; 173 | import DmsStack from '../lib/dms-stack'; 174 | 175 | const app = new cdk.App(); 176 | const dmsStack = new DmsStack(app, 'DmsOraclePostGresStack', { 177 | vpcId: 'vpc-id', 178 | subnetIds: ['subnet-1a', 'subnet-1b'], 179 | replicationInstanceClass: 'dms.r5.4xlarge', 180 | replicationInstanceIdentifier: 'test-repl-01', 181 | replicationSubnetGroupIdentifier: 'subnet-group', 182 | vpcSecurityGroupIds: ['vpc-sg'], 183 | engineVersion: '3.4.6', 184 | tasks: [ 185 | { 186 | name: 'demo_stack', 187 | sourceSecretsManagerSecretId: 'sourceSecretsManagerSecretId', 188 | targetSecretsManagerSecretId: 'targetSecretsManagerSecretId', 189 | migrationType: 'cdc', 190 | engineName: 'oracle', 191 | targetEngineName: 'aurora-postgresql', 192 | tableMappings: { 193 | rules: [ 194 | { 195 | 'rule-type': 'selection', 196 | 'rule-id': '1', 197 | 'rule-name': '1', 198 | 'object-locator': { 199 | 'schema-name': 'demo_test', 200 | 'table-name': '%', 201 | }, 202 | 'rule-action': 'include', 203 | }, 204 | ], 205 | }, 206 | }, 207 | ], 208 | publiclyAccessible: true, 209 | allocatedStorage: 50, 210 | env: { 211 | account: '11111111111', 212 | region: 'eu-central-1', 213 | }, 214 | }); 215 | ``` 216 | 217 | ## Configuration Options 218 | 219 | In the cdk.json file you define the DMS related settings. 220 | 221 | | Params | Description | Default | Required | 222 | | ----------------------------------- | :----------------------------------------------------------------------------- | :------------ | -------- | 223 | | environment | Name of your environment identifier to deploy the CDK application | dev | y | 224 | | subnetIds | Subnet ids of your VPC used in creation subnet group for DMS service. | | y | 225 | | replicationInstanceClass | The type of DMS instance class | dms.t3.medium | n | 226 | | replicationSubnetGroupIdentifier | The identifier of the subnet group where replication instance would be located | | n | 227 | | replicationInstanceIdentifier | The unique identifier of the replication instance | | y | 228 | | replicationTaskSettings | DMS task settings that overrides default values in task-settings.ts | | n | 229 | | migrationType | Is full load or change data capture (full-load or cdc) | full-load | y | 230 | | sourceSecretsManagerSecretId | ARN of secrets manager for source database credentials | | y | 231 | | targetSecretsManagerSecretId | ARN of secrets manager for target database | | y | 232 | | allocatedStorage | Storage space for the replication instance (should be based on log size) | 50g | n | 233 | | engineName | Supported source databases oracle, postgres, sqlserver, mysql | mysql | y | 234 | | targetEngineName | Supported target databases oracle, postgres, sqlserver, aurora-postgresql | mysql | y | 235 | | engineVersion | DMS engine version | 3.4.6, 3.4.7 | n | 236 | | targetEngineVersion | The name of target database engine (mysql, oracle and aurora-postgresql) | mysql | n | 237 | | databaseName | Name of the database to be migrated. Relavant for oracle, postgres... | false | n | 238 | | publiclyAccessible | DMS endpoint is publicly accessible or not | false | n | 239 | 240 | 241 | ## Security 242 | 243 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 244 | 245 | ## License 246 | 247 | This library is licensed under the MIT-0 License. See the LICENSE file. 248 | -------------------------------------------------------------------------------- /bin/dms-cdk.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import * as cdk from 'aws-cdk-lib'; 3 | import DmsStack from '../lib/dms-stack'; 4 | 5 | const app = new cdk.App(); 6 | 7 | // get context from conf 8 | const environment = app.node.tryGetContext('environment'); 9 | const ctx = app.node.tryGetContext(environment); 10 | 11 | const dmsProps = { 12 | vpcId: ctx.vpcId, 13 | subnetIds: ctx.subnetIds, 14 | replicationSubnetGroupIdentifier: ctx.replicationSubnetGroupIdentifier, 15 | replicationInstanceClass: ctx.replicationInstanceClass, 16 | replicationInstanceIdentifier: ctx.replicationInstanceIdentifier, 17 | tasks: ctx.tasks, 18 | publiclyAccessible: ctx.publiclyAccessible, 19 | stackName: `dms-cdk-stack`, 20 | env: { 21 | region: ctx.region, 22 | account: ctx.account, 23 | }, 24 | }; 25 | 26 | new DmsStack(app, 'DmsStack', dmsProps); -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node --prefer-ts-exts bin/dms-cdk.ts", 3 | "watch": { 4 | "include": [ 5 | "**" 6 | ], 7 | "exclude": [ 8 | "README.md", 9 | "cdk*.json", 10 | "**/*.d.ts", 11 | "**/*.js", 12 | "tsconfig.json", 13 | "package*.json", 14 | "yarn.lock", 15 | "node_modules", 16 | "test" 17 | ] 18 | }, 19 | "context": { 20 | "dev": { 21 | "account": "99999999999", 22 | "region": "eu-central-1", 23 | "vpcId": "vpc-xxxxxxxx", 24 | "subnetIds": [ 25 | "subnet-a", 26 | "subnet-b" 27 | ], 28 | "tasks": [ 29 | { 30 | "name": "DMS_SAMPLE", 31 | "migrationType": "full-load", 32 | "engineName": "oracle", 33 | "targetEngineName": "aurora-postgresql", 34 | "databaseName": "oradev", 35 | "sourceSecretsManagerSecretId": "arn:aws:secretsmanager:eu-central-1:99999999999:secret:kmstan_dms_source_db-APs9MM", 36 | "targetSecretsManagerSecretId": "arn:aws:secretsmanager:eu-central-1:99999999999:secret:kmstan-dms-target-db-T0cQeQ", 37 | "tableMappings": { 38 | "rules": [ 39 | { 40 | "rule-type": "transformation", 41 | "rule-id": "1", 42 | "rule-name": "Default Lowercase Table Rule", 43 | "rule-target": "table", 44 | "object-locator": { 45 | "schema-name": "DMS_SAMPLE", 46 | "table-name": "%" 47 | }, 48 | "rule-action": "convert-lowercase", 49 | "value": null, 50 | "old-value": null 51 | }, 52 | { 53 | "rule-type": "transformation", 54 | "rule-id": "2", 55 | "rule-name": "Default Lowercase Schema Rule", 56 | "rule-action": "convert-lowercase", 57 | "rule-target": "schema", 58 | "object-locator": { 59 | "schema-name": "DMS_SAMPLE" 60 | } 61 | }, 62 | { 63 | "rule-type": "transformation", 64 | "rule-id": "3", 65 | "rule-name": "Default Lowercase Column Rule", 66 | "rule-action": "convert-lowercase", 67 | "rule-target": "column", 68 | "object-locator": { 69 | "schema-name": "DMS_SAMPLE", 70 | "table-name": "%", 71 | "column-name": "%" 72 | } 73 | }, 74 | { 75 | "rule-type": "transformation", 76 | "rule-id": "10", 77 | "rule-name": "Rename Schema Rule", 78 | "rule-target": "schema", 79 | "object-locator": { 80 | "schema-name": "DMS_SAMPLE" 81 | }, 82 | "rule-action": "rename", 83 | "value": "dms_sample", 84 | "old-value": null 85 | }, 86 | { 87 | "rule-type": "selection", 88 | "rule-id": "11", 89 | "rule-name": "Selection Rule DMS_SAMPLE", 90 | "object-locator": { 91 | "schema-name": "DMS_SAMPLE", 92 | "table-name": "%" 93 | }, 94 | "rule-action": "include", 95 | "filters": [] 96 | }, 97 | { 98 | "rule-action": "change-data-type", 99 | "object-locator": { 100 | "schema-name": "DMS_SAMPLE", 101 | "column-name": "COL1_NUMBER_INT", 102 | "table-name": "SAMPLE_NUMBER_DATA_TYPE" 103 | }, 104 | "rule-target": "column", 105 | "rule-type": "transformation", 106 | "rule-id": "12", 107 | "data-type": { 108 | "type": "int8" 109 | }, 110 | "rule-name": "30" 111 | } 112 | ] 113 | } 114 | } 115 | ], 116 | "replicationInstanceClass": "dms.r5.large", 117 | "replicationInstanceIdentifier": "dms-dev-eu", 118 | "replicationSubnetGroupIdentifier": "dms-dev-subnet-eu", 119 | "replicationTaskSettings": {}, 120 | "publiclyAccessible": false 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /conf/task_settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "TaskSettings": { 3 | "TargetMetadata": { 4 | "TargetSchema": "", 5 | "SupportLobs": true, 6 | "FullLobMode": true, 7 | "LobChunkSize": 256, 8 | "LimitedSizeLobMode": false, 9 | "LobMaxSize": 0, 10 | "InlineLobMaxSize": 0, 11 | "LoadMaxFileSize": 0, 12 | "ParallelLoadThreads": 0, 13 | "ParallelLoadBufferSize": 0, 14 | "BatchApplyEnabled": false, 15 | "TaskRecoveryTableEnabled": false, 16 | "ParallelLoadQueuesPerThread": 0, 17 | "ParallelApplyThreads": 0, 18 | "ParallelApplyBufferSize": 0, 19 | "ParallelApplyQueuesPerThread": 0 20 | }, 21 | "FullLoadSettings": { 22 | "CreatePkAfterFullLoad": false, 23 | "StopTaskCachedChangesApplied": false, 24 | "StopTaskCachedChangesNotApplied": false, 25 | "MaxFullLoadSubTasks": 8, 26 | "TransactionConsistencyTimeout": 900, 27 | "CommitRate": 10000 28 | }, 29 | "Logging": { 30 | "EnableLogging": true, 31 | "LogComponents": [ 32 | { 33 | "Id": "SOURCE_UNLOAD", 34 | "Severity": "LOGGER_SEVERITY_DEFAULT" 35 | }, 36 | { 37 | "Id": "SOURCE_CAPTURE", 38 | "Severity": "LOGGER_SEVERITY_DEFAULT" 39 | }, 40 | { 41 | "Id": "TARGET_LOAD", 42 | "Severity": "LOGGER_SEVERITY_DEFAULT" 43 | }, 44 | { 45 | "Id": "TARGET_APPLY", 46 | "Severity": "LOGGER_SEVERITY_DEFAULT" 47 | }, 48 | { 49 | "Id": "TASK_MANAGER", 50 | "Severity": "LOGGER_SEVERITY_DEFAULT" 51 | } 52 | ] 53 | }, 54 | "ControlTablesSettings": { 55 | "ControlSchema": "", 56 | "HistoryTimeslotInMinutes": 5, 57 | "HistoryTableEnabled": false, 58 | "SuspendedTablesTableEnabled": false, 59 | "StatusTableEnabled": false 60 | }, 61 | "StreamBufferSettings": { 62 | "StreamBufferCount": 4, 63 | "StreamBufferSizeInMB": 16, 64 | "CtrlStreamBufferSizeInMB": 5 65 | }, 66 | "ChangeProcessingDdlHandlingPolicy": { 67 | "HandleSourceTableDropped": true, 68 | "HandleSourceTableTruncated": true, 69 | "HandleSourceTableAltered": true 70 | }, 71 | "ErrorBehavior": { 72 | "DataErrorPolicy": "LOG_ERROR", 73 | "DataTruncationErrorPolicy": "LOG_ERROR", 74 | "DataErrorEscalationPolicy": "SUSPEND_TABLE", 75 | "DataErrorEscalationCount": 0, 76 | "TableErrorPolicy": "SUSPEND_TABLE", 77 | "TableErrorEscalationPolicy": "STOP_TASK", 78 | "TableErrorEscalationCount": 0, 79 | "RecoverableErrorCount": -1, 80 | "RecoverableErrorInterval": 5, 81 | "RecoverableErrorThrottling": true, 82 | "RecoverableErrorThrottlingMax": 1800, 83 | "RecoverableErrorStopRetryAfterThrottlingMax": false, 84 | "ApplyErrorDeletePolicy": "IGNORE_RECORD", 85 | "ApplyErrorInsertPolicy": "LOG_ERROR", 86 | "ApplyErrorUpdatePolicy": "LOG_ERROR", 87 | "ApplyErrorEscalationPolicy": "LOG_ERROR", 88 | "ApplyErrorEscalationCount": 0, 89 | "ApplyErrorFailOnTruncationDdl": false, 90 | "FullLoadIgnoreConflicts": true, 91 | "FailOnTransactionConsistencyBreached": false, 92 | "FailOnNoTablesCaptured": false 93 | }, 94 | "ChangeProcessingTuning": { 95 | "BatchApplyPreserveTransaction": true, 96 | "BatchApplyTimeoutMin": 1, 97 | "BatchApplyTimeoutMax": 30, 98 | "BatchApplyMemoryLimit": 500, 99 | "BatchSplitSize": 0, 100 | "MinTransactionSize": 1000, 101 | "CommitTimeout": 1, 102 | "MemoryLimitTotal": 1024, 103 | "MemoryKeepTime": 60, 104 | "StatementCacheSize": 50 105 | }, 106 | "ValidationSettings": { 107 | "EnableValidation": false, 108 | "ValidationMode": "ROW_LEVEL", 109 | "ThreadCount": 5, 110 | "FailureMaxCount": 10000, 111 | "TableFailureMaxCount": 1000, 112 | "HandleCollationDiff": false, 113 | "ValidationOnly": false, 114 | "RecordFailureDelayLimitInMinutes": 0, 115 | "SkipLobColumns": false, 116 | "ValidationPartialLobSize": 0, 117 | "ValidationQueryCdcDelaySeconds": 0, 118 | "PartitionSize": 10000 119 | }, 120 | "PostProcessingRules": null, 121 | "CharacterSetSettings": null, 122 | "LoopbackPreventionSettings": null, 123 | "BeforeImageSettings": null 124 | } 125 | } -------------------------------------------------------------------------------- /docs/dms-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/dms-cdk/d14102fe728da93e2384cdaa716f3953c3028c6d/docs/dms-architecture.png -------------------------------------------------------------------------------- /docs/dms-architecture.xml: -------------------------------------------------------------------------------- 1 | 7Z1bc6M6EoB/Tap2H+QSd3iMnWROqk7OpiazM7v74hIgHCYYvICTeH79aXFxuBnjsUnAkZNKTANC6NYf3S1xIc2Wr19Csnq8C2zqXYjYfr2Qri5E+Bgy/GOSTSrRFCkVLELXTkXCm+DB/UUzIc6ka9emUenAOAi82F2VhVbg+9SKSzIShsFL+TAn8MpXXZEFrQkeLOLVpT9cO35MpbqC3+R/UHfxmF9ZwNmeJckPzgTRI7GDl4JIur6QZmEQxOm35euMeqzw8nLJzvPpa8z23NrfibfOsiUJ2xS+0ihYhxa9opEVuqs4COGkMBOmR88vpMs8FzR0ief+IrEb+OiZhhH8T496zg4hWZGFDQln13ygS+LHrnVFYjIL/Ji4Pg27pJ6eHYeuv/jTjWlIvLTuYurHpbtehcGKhnHWah7jmNX35YV4A79weOAFi80kotY6dOPNhCzJr8Cf2PQZdjvB2reTHMCG7ZJFSJbo2Y3W25yBnIiiohiKiizJdpBsiRYyZd2ATaw5qk5My9CSnNykeb79elsr1oNyBa3YXfjI9aMVtFNWljdWsFwFPtx5BBu6THRsKg5SVFlEMhE0ZCi6gqjp2KaqOLJJrdMWTbSJYrpES9ZnoT5AgmVFMhRDQ7KoS0h2VAXp2MCIUFVzDFuihmwUCwW+NLeEfG9D28x3Zc29uelve11YbheHNHhhJA1e4A3+0zX4wPzJlJWIPWKCwkzu60JUPWgSU9uFjKsL9jUXmbng8scDnPT9fpbvgXTN6tFJIyikkfQCaG+W633brLKqJC8Rsrxgbaf78443mwrx5gf543+/po7uad++ObfGf5G4bcnbnhrFm1xFrgKX1Sh8V6bwC81+hi8U2DNjWxNRqQiq21pZINS3WBplQXVbKwuEavJC5fpCNYMFQW2rlDyuXB8XMgi/0jRYx9C66GwLJBiEj/GSVbMAX18eYRx4WJGkU70ANYHMgYaUsY8g5ttZCbPTgR2Silu+LhhmTaD2RDxZhMGanb0Ib61k1GnYPYfv87SiIZk4DJ4gZx5rlpB/SRdNSVXZBV3PK8htheq2DHIY0aCtE+/Sg74M++KAXZBkWx51WBOL4GbY+JZsXUkswzaJHqmd5T5tYiwp+rpzfM9Eoe38ewV3YdOiKsja3hcaLGkcbuC4LCFVzcbwbHAQ8u2XN1gT9Uz2WAA1WakM/ott2m/dvaOmaek0I9A/rbnnWmkEWqn3QpEdRTAMwUGOZKlQKDIUiokxwoKkqJiYhujIuwplm4eXl5fJizQJQnaLkJzB7pTdhCgi6PUo2kCzfkU+XFCKmZ465R2WSjwZJBEMhmsrXoeU5cOUKSWWjrBMFSRTqHBdNaAt2orqmIasmZrZ6w229tCC1j8kCyJmz6I3GEbQm+T61iOMInDxFDq6Xj3Un/7zRf25vn26/+bJy/jnSke37zM6YMWyFEmwkC3KApINS4KOYLNqkTRNsxRZVJ0+8IxlPyW0FlprxKoFhcu5FkpVcwe0EuRWtirqZR/6Y5MSVy7Vma4Wla6wU3NXwYJtNyVV4JU+tbdR1t6y1KC9pQbtLch9a29BHrX6FnaOx1x/c/3N9fe76m+uQX/X3jFbRzEM7OFHGD3aFTM3eozC6DEnSeuoMZOtsQGwbvhwHKpa1ggMHxLeb/jYHvO+ho9xkxMHJw5OHJyGAU5V9c+tHyNiN5YHk0QUPaxNn8ZHw9sqdJ9JTFGUJbef4BROcEcSHOMRl76ZiDIT1MeBXd6d5m9HPyRslyddBT1B1q6nl3XQu1ZvQOcdAXoVy9n2Ou8GgGLFdqY02M7UD7GdKaMGQIUDIAdADoCDAMAaQ3ACHBEBxiRc0Bh9vXo4Gv5CO0LLTfR/rwv3FRVnnfsaQaeizeHnhiWwC4AqMAEfeWrU8CM7uEQEOW78yYryPojcOFVbZhDDo85eHrEgL6w+iwi2D7dIMrbBhuO+snxME/Kl4fUzTQFY2MlkUOzzpNjnMELGxGcjUJ9QIwvaRCljjbGVFMCmiWvU3l2Co8aazxpmKtimBqMoQaJsQYZEW0O6TjGSDFFyZMvSdcc5O6wZtdIvqg2u70ek7+8zC82HGXxUbvDhBp/zNfiUyUhsCnVWPsTgo46ajNRPSkbc4MMNPkNjvypCcP4bEf/9y0erkC7dCOobswzBv1lqsDiWBWHwhxEfZRUVdYFBjcPgaOO3An+et6QGxFOTT0PwlsJ+Tod46ecdEQ9tlz3IhhhJqDOepCgNjGf0zXjaqBlP44zHGY8z3iAYrw0TOO8NhvcaOSyf32Y9EiCFbt64dqsctRc0p4YgjB+DReAT7/pNOg1ZUW7VbwFKSvPdopiE8SVb8Ye5yTwSRa6Vi29cLz/nJ43jTUYtZB0HjH22V/0zYKCQJVeBDmhs2HG2+p9le5/2L3bv5rJJxz2cWrxbC1E5DChC6kEjey5nsT/P2LgNQMK5WoD2qAlVERRNFTHSZBO0s4YpjDyGjUzDdBTVVC1T7ldNTD3iP/0FpHB7dUBrM05bW/VysQXHwqpKkaSrDJooRrqEZaRa1JGBDTDJzKxH3YZw4kCzhvoVNJPCSIUsB7hPJlhARFNspAvUloB6REUSOY3uoVHODh/JDu1GHM4OndhBHjQ7jNuwIJyrZYGzw5DZ4cSzFDk7cHYYCjt0m1kGZ925cHPpHeEHGj670O6O9TPZWcD5fJknPo/ypPfjitFKK40up0a3U5PrqdH9VHdBlQ5LnEINV6gKm2RaXSjUD8v9SHVhk6zJaVY9W2g4W6icvdtl1TWOG/bJVxrsLOy7ckNIKOUCn4Fd50DvZOEA9hlitPeOyO6sx6ZOuClsNrrjWvpDr54wTSsHO22XMvjoIHBj1LR6YvwZDKzugwfTUrBBRQEJ2LSQrIg60hUZtJ5gGRaVJCAdfWjw8LmdRG06njuJBgNrJzP0tM/ZOsLQ091o08kGtNu2U7EoscPyN0aIJzb95FEh+20/6qBNPyOfUXWuU6rO0/STd5qx235O7Kzkph9u+hkGTXy//wsEiQZJH8Uv3l7r1Awa8NzZaV5Y4eUxTXSRJFOCBmjMUGTm9qG6bLq5anpErz7JL13bTsgkucMpsZ4WCaM02Qx2Pkhnt5/l5OLt7UgFamhXTodGoOKJqMpa6dkbZZzXGQ6y1O9ZoRUOCRwnymL6bwptZJuJ44Fi+6qtcQLFp333FQ/I7PtZm+voQ3X0Xm3ELRADYIaab+hu88AWicEPHnne4Q5qd/ykM4062SzaqWKv7b9mqS8ASLNfoHF5mRqJNDpGql4Ay4XG9jqp3na/q7tUJrfo2atA99n09b7tEOPGhlM/2I4GG8aoczhLjZ6lWrtoRf9wUBgYKLTGldyx92x9pSsPVG96SypZMsXrm9GqHEqyN7xk/+Vu8yXVjg1aoZbYiVfEVl7h4SmDCk+50fRrLB8WnnKFlZmgfaLwFNby+41DkcrMKmv1OJSm+di5rD9mzZ2vI2VW8ZMyq+NIhq1ZIpJUHSNZogSx9Qnhj6IQg0gG1qSzY9ZRE12NCt70Nqe70dHdNxI9RccvR8OiQtareQypdUIvqRW9QAfaSdxJ7Q1SBc3aaPypAADGmnGl5k6ngkavuamOVfk1tf0L+vKkVC69KmZdVCuKub5KcO7HKinm3lcJlsatmE+sfUajmLkxiRuThokemcrisDEY2Ghkgt+Jem2fMPSppzfnyyN0mN98GEC8c4zruCeMCOc6Y+RMY1x7X1PkneY3930bPMaVx7gO21CRr4CM0wVTj/YRgdqH+0OQ5ksQPqHkDUAO6TavWWh/k2YPb1RSLiU8VXZZPQbnxOj+RqWsHuZZPczf6qHf0JvK2yKb5tN+jB9j3C+LPLXCHQwfnaO+4uaSz2AuqSpObjcZEftc+3YWhyLifyT1OWNVSq2QJrI74pMFDSeTyT+PR6LsWl0ISGx34HACOoCA8mLvF3jk8gIi2wXyiyvpNwQbS337h8Rx+4dE7h/iwMOBZxjAc5C25Bw0GA46lf9IbLfLfGr/kZgNPB38R/gwwnhfB5I4bgOJeK4GkvN0IIm9Lyz7Tg6k3hd74R4k7kH6CHooTU9qBInOq6K0v5jnHFdFybXRgcYEPMGiXo43HdGiKON+OU+e/bMDCP5s/dHP1lwlH6qSa8qHGxQGgAQ7F0G5I9HOly83osNvrIKSPjHspAi+CkptiTVBq8RiDGUdlPzhb/CkMKJhgOsY7pc4R3bqtAhKrn8+gBTex7x1qoeJnmCBFWEQxMXnStaS7ljPBeHf7Z1bd5s4F4Z/jS+Vxdlw2SR129WmX1adtJ3vpksgYdNg8ICcxP31s8XBRwx2YseCqjOrNUIIISQ97xbSVk+/mjx/SPB0fBMTGvY0hTz39OueBn/6KvzDQ+Z5SN/U84BREpA8SF0GDIM/tAhUitBZQGi6FpHFcciC6XqgF0cR9dhaGE6S+Gk9mh+H63ed4hHdChh6ONwO/REQNs5Dba2/DP9Ig9G4vLNqOfmZCS4jF0+SjjGJn1aC9Pc9/SqJY5b/mjxf0ZAXXlkuyvX987++E3y0ra/64MNn7Z/b/39FRWoRfWY8/ifyHYezIrO6vkj3G03jWeLRa5p6STBlcQIXJUVgHvtXT39Xe4+UJgEOgz+YBXGEHmmSwr/5tY9FFFwUb1JxuyInQzrBEQu8a8zwVRwxHEQ02Sf1/GqWBNHoS8BogsP8PTMasT1KaJrEU5qwot6NGeM15l1PG8D/kEgcxqP5RUq9WRKw+QWe4D9xdEHoI5z241lEsnzBAQnwKMET9Biks0V+IRxrmmk6poU8nfjI8DQPuYbtwKHS9y0bu56TV5JB/iSfvn3aegUH5QraQTCKUBClU6jpvIQHXjyZxhGURwoHtoFtxTV9ZFqGhgys9pFj2iairk9cy/QNl3rHLZp0njI6QRPe6uEtQYhimLpjOn1kaLaODN8yka04CsLU6vsO0aljOKuFAj+q60d5tqIel6eKBnNI41m05mSfOvT6hqS2uiGpsiH9dQ0pdn9zjGpKiF1AefZcRSWG6uIF4d18WpT5iEL6gYcWZZfHK1vgz8fpkH7+9ydBz/+zHubvg++f1XukLSrkosmmbF7S1g/C8ApKLckO9feDgTLQe/plypL4gZZnIrgbBBKcjim/mQIHUO3h0XH4LoRXC2EsnkKoD8UwLJJXiuOV9OG/AfzQL8dswp9VhZ/5Q/Dk6POGBGjqNIqICfHvp1DTCV2VGsUzf6DxhLJkDvGK5JFVdAFFLdKN4vhpqTcsswgbr2iNhUYq+47RIu1lvdiz+6p5WS3ovmpzL7uvFnRfJy8UwzdVx1F95OueBYViQKG4ioIUVTctBbuO5hu7CmWRh6enp4sn/SJO+CNCcg5/Uv4Q0Iqh1aN0DtX6GUVwQ53xfvKYT7hW4iNubSHoFmcemyWU58M1KMWejRSDmsig8MJty4G6SEzLdx2j7/bdkz5gbQvF99bl3Sjtm4Nfd/4PYv/4FtoPb9M+FdPzTF31ENEMFRmOp0NVJLxg9H6/75mGZvmnICnPfg7TnWC9vhl+o9MQuJXntRKzZJImm5HqGbu0AqsYm/BCy9DEcfc0hhc2nOKsSj5BrVqn4QaQfd/XPG8LyHCGWNC1WOfgp26u81PVt/m5CFvlp3FqfOqtxqcu8SnxKfEpBD6rQLFvPjRFUXg+oBcdZJnwxtCVQA5yEEmCv4bgTabxGEdRUcgN0DZqoU3JiJaGbJywcTyKIxy+X4ZeLrHODd1lnC8xt4QzmP+mjM2LIXY8Y/E66uFNJ/Of/PoLszz8Z/Xc9XOReH40L47yh+MZfBnuV1t0ddGYeUSGkxHdB1n7CoiEhlDFHtfzfTI1YLRaDezs3VuuBhqwYJmq2bc0BfUNF2jcVyj0Og5BruP6JkhezzVOi4XLEEcPX0EZfLo+oK6Zx31b2+VCVN9TLIsi3ba4SKIKsnXFQJZHfQO0gIILVf6qxziyBK14vWrfpdB3Ic8HmWdgRUW4bxJkq5ToIHI0U9ek+GwQn1I2vMTwHzLsPew2+dPl6XrdYP7Vxr4mjLFvthrvRwaGMHjvYn8rjf3OG/srcJBmvgC8Pp6Zb4lj5q8a+Ss2/9nMfHtfM98U2cy3Wq0DrI7qgG6a+XY3zPyTj1ZIM1+a+e2WDf1zy4bFZDv1LbXA0RFfXHobB3y6YjmqoKnW2qiCrm8gOtckxVXLqvQuSfB8JdqUR0j3v49mr/Weh8aHH3kOltVzUSavFzL9VguZcn2GFDJSyLxAyEgFIBXAMRQAT5s+s1sogXSHGvDyKNNllHopYNdKgdcM+BOT2sSoGvC3NVe3zjLgrwo74G+3mo9H7nGF4WMX+1054N/tAf9tSMhBfxHYff05L6i8JvGh6THm/xDqwy3hVSn+LPL4g/DfYDFD7IRiBm9eub4ZrtToXewfA69ogkL6SEPkkQe0uBvi90LFndDiPojFKL8HIpMUbaRfLxycWuHAa+C6OqhcdLcqGYogXCy586A+8RextRZvEhCSjUJUyZH1kYk31xeGs7F6QNvWF6XZvaYvrFPrC6fV+sKR+kLqC6kvxNAXRweZ1Cci65NZmr3TzUUjihfilJ/oaRaecPbmf3PhwDYqQREhctPpgspb6iWMn+rEC88GKlcv5oApctCsVdQlfKRYWYgVU9kYDKlwFbDwN7QqVja/ZxxdrJSuCFqqVsrsS7ki5YqUK6LKlWaubYNMqhVB1MoN5oNbShBNZyxHJAYEZd16LkNJdpYXJnm4+J3u9IcAh5DULAR5mqIsObSS2B7SYpWxUlqU0sJcHwaxzS1loVY5ITr5Zxa1LT7U6rMvlYVUFlJZnFlZvJRBUkAIICB2yIEXTKZUtVoF8NazKbdmWyjZnzeZZ6kWS6ybF12cdEbmoRMp7epxkF3zKOuj9046jVItK25L9YvWUf3SzYmUZXsWZyaloBNC5ZIQOSG0LSrmk50Mtegx8l3N/XX1w3TJfeghrX4i6KEq5ghLRDdUDU7YO74Dw3KEJAsbBLy4sgsqRFKlmNqtkPIz5c4Mb7M2ZX/NVO4SIYZoQurGGI95EhlUV1tbIIPqsy9lkJRBbyiDtCMvY5I6SOogMXTQcoDNCvn3FhcqgjViuyebrI/INQzw1HvATsc4S5XE3mySvaemhTJuLpW+uGXA1reeNFcxyoVW5UjL86DTq9AxuqU7OhFh1wngTm/zg49T8b2n/E50uu897faafWzyCCMUuthvy+893f7eI7/iCMf9OaTTK2cVlx/iUsoYFOS+8055Gtnyl+V1e2iCegfbr5/2EVKftWzSB9r8MGJvzydVjQoVoJ181ke7vWWrO3t1qQKkCpAq4E1VQANypDQQRBrkLwhHBP7emAO8rzLgogASWF+Nsp88qPejLcas0NXtLM+yV+VilKDczbti/UnlVpUn1wvtdr+tSv/bUi9IvSCGXqjnkJQLgsiFO5w+DNdU3JYcYBDloDGCeu/cXfOthYx181+39jT/T7/oo91etMvsS5xLnEucnxnn25yQ+BYC33z23A1lGJ4CN2CcR51URG3Aeb3X7K7j3KhwZXUmnLfbl7TaVWfSEucS5+3D+S5uSKwLgPXBLAy/xJg0WOY+RAsh2kHW+V/m+VoT1jpvy1z++uxLnEucS5yfGefVvJAoFwDlX+LRCMqsgeRhHusgkNd7ou4ayLfs8n1n2Z0e5O32MV1mX4JcglyC/Mwgr6SF5LgAHOdpJ3F4h92QNsDcy6MyHvUQomv17pq7RvRN01yckfYyIy0lepl9SXRJdEn0c/th3skNiXUBsA5FRvHkcub7NGnAeppFdbOoB2G93lVy57EujKFePnlbsS6dIEusS6yLgfXd3JBYFwDrV2McjehtEns0TbMWvOkg55qEH3FEePO+jcPAm+8y5rOUpouUECHhuLhwunJhgwbQ/ioNsOkrWJyv7vmLaK8G0KQGkBpAagAhNMAmZHYgRQoCAQTB+ySJk0s6xo9B5sigkvWUR3LXIzVwvd5HXte5LtCQfbsd3pXZl1yXXJdcPzPXq1ghIS4AxJut+rtZlIXvZ8qzldgNnK/3e9d5zoszht9ul3aadGknOS85LwbnN3GyCg/JewF4/x3ymee/4dP84yLiQR/m673VdQ3qhrCOarR2+53TpN85CXUJdTGgvosZEugCAH0Hvl+w+7BW72TurXcfzrN8yv3yyk2smvfL0w9TB3tvl3cc1LfbJ12Z/c6hvps72J1+57e32cHu5BvxyQ3s5AZ25xAE1zfDW3j6XXY9maTT5ekGRfB3+alTN6x5Va+w5vWzWPPt9lN3bGYIg/gudrnSmu+2Nb/OB2nDvzGyefnFMVuJ+oFXoBveYCHwPw== -------------------------------------------------------------------------------- /docs/dms-cdk-classes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/dms-cdk/d14102fe728da93e2384cdaa716f3953c3028c6d/docs/dms-cdk-classes.png -------------------------------------------------------------------------------- /example/demo-dms-stack.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-unused-vars */ 2 | import * as cdk from 'aws-cdk-lib'; 3 | import DmsStack from '../lib/dms-stack'; 4 | 5 | const app = new cdk.App(); 6 | const dmsStack = new DmsStack(app, 'DmsOraclePostGresStack', { 7 | vpcId: 'vpc-id', 8 | subnetIds: ['subnet-1a', 'subnet-1b'], 9 | replicationInstanceClass: 'dms.r5.4xlarge', 10 | replicationInstanceIdentifier: 'test-repl-01', 11 | replicationSubnetGroupIdentifier: 'subnet-group', 12 | vpcSecurityGroupIds: ['vpc-sg'], 13 | engineVersion: '3.4.6', 14 | tasks: [ 15 | { 16 | name: 'demo_stack', 17 | sourceSecretsManagerSecretId: 'sourceSecretsManagerSecretId', 18 | targetSecretsManagerSecretId: 'targetSecretsManagerSecretId', 19 | migrationType: 'cdc', 20 | engineName: 'oracle', 21 | targetEngineName: 'aurora-postgresql', 22 | tableMappings: { 23 | rules: [ 24 | { 25 | 'rule-type': 'selection', 26 | 'rule-id': '1', 27 | 'rule-name': '1', 28 | 'object-locator': { 29 | 'schema-name': 'demo_test', 30 | 'table-name': '%', 31 | }, 32 | 'rule-action': 'include', 33 | }, 34 | ], 35 | }, 36 | }, 37 | ], 38 | publiclyAccessible: true, 39 | allocatedStorage: 50, 40 | env: { 41 | account: '11111111111', 42 | region: 'eu-central-1', 43 | }, 44 | }); 45 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | roots: ['/test'], 3 | testMatch: ['**/*.test.ts'], 4 | transform: { 5 | '^.+\\.tsx?$': 'ts-jest' 6 | } 7 | }; 8 | -------------------------------------------------------------------------------- /lib/dms_props.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from 'aws-cdk-lib'; 2 | import { Rules } from './rules-props'; 3 | 4 | export interface TaskConfig { 5 | name: string; 6 | /** 7 | * Secrets manager arn for the target database 8 | */ 9 | targetSecretsManagerSecretId: string; 10 | /** 11 | * Role to access target secret 12 | */ 13 | targetSecretsManagerRoleArn?: string; 14 | /** 15 | * Secrets manager arn for the target database 16 | */ 17 | sourceSecretsManagerSecretId: string; 18 | /** 19 | * Role to access source secretTaskSettings; 20 | */ 21 | sourceSecretsManagerRoleArn?: string; 22 | 23 | /** 24 | * Type of migration full-load (default) or fullload-with-cdc 25 | */ 26 | migrationType: 'cdc' | 'full-load' | 'full-load-and-cdc'; 27 | 28 | /** Source database Engine names */ 29 | engineName: 'mysql' | 'oracle' | 'aurora-postgresql' | 'sqlserver' | 'postgres'; 30 | 31 | /** Target database Engine names */ 32 | targetEngineName: 'mysql' | 'oracle' | 'aurora-postgresql' | 'sqlserver' | 'postgres'; 33 | 34 | /** Database name for oracle and postgres */ 35 | databaseName?: string; 36 | 37 | /** Mapping or Transformation rules */ 38 | tableMappings: Rules; 39 | } 40 | 41 | export interface DmsProps extends cdk.StackProps { 42 | /** 43 | * Target VPC where CDK is provisioning the RDS instances and Security Groups 44 | */ 45 | vpcId: string; 46 | /** 47 | * Target Subnets where CDK is deploying the RDS instances 48 | */ 49 | subnetIds: string[]; 50 | /** 51 | * Security group Ids of target RDS for DMS task access 52 | */ 53 | vpcSecurityGroupIds?: string[]; 54 | /** 55 | * DMS tasks for migrating databases / schemas which runs in the DMS replication instance specified 56 | */ 57 | tasks: TaskConfig[]; 58 | /** 59 | * Type of replication instance dms.t3.medium (default) 60 | */ 61 | replicationInstanceClass: string; 62 | /** 63 | * Identifier for the replication instance 64 | */ 65 | replicationInstanceIdentifier: string; 66 | /** 67 | * Subnet Group identfier for DMS tasks to run. This should be private subnet in a vpc 68 | */ 69 | replicationSubnetGroupIdentifier: string; 70 | 71 | /** DMS is publicly accessible */ 72 | publiclyAccessible?: boolean; 73 | 74 | /** DMS Engine Version */ 75 | engineVersion?: '3.4.6' | '3.4.7'; 76 | 77 | allocatedStorage?: number; 78 | } 79 | -------------------------------------------------------------------------------- /lib/dms_replication.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/prefer-default-export */ 2 | /* eslint-disable camelcase */ 3 | import * as dms from 'aws-cdk-lib/aws-dms'; 4 | import * as cdk from 'aws-cdk-lib'; 5 | import { Role, PolicyStatement, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; 6 | import { 7 | CfnReplicationSubnetGroup, 8 | CfnReplicationInstance, 9 | CfnReplicationTask, 10 | CfnEndpoint, 11 | } from 'aws-cdk-lib/aws-dms'; 12 | import { Construct } from 'constructs'; 13 | import { DmsProps } from './dms-props'; 14 | import { TaskSettings } from '../conf/task_settings.json'; 15 | import { Rules } from './rules-props'; 16 | 17 | class DmsReplication extends Construct { 18 | private instance: dms.CfnReplicationInstance; 19 | 20 | private region: string; 21 | 22 | private secretsManagerAccessRole: Role; 23 | 24 | constructor(scope: Construct, id: string, props: DmsProps) { 25 | super(scope, id); 26 | 27 | const subnetGrp = this.createSubnetGroup(props); 28 | this.region = cdk.Stack.of(this).region; 29 | this.secretsManagerAccessRole = this.createRoleForSecretsManagerAccess(); 30 | const replicationInstance = this.createReplicationInstance(props); 31 | replicationInstance.addDependsOn(subnetGrp); 32 | 33 | this.instance = replicationInstance; 34 | } 35 | 36 | /** 37 | * Creates subnet group where DMS runs its task 38 | * 39 | * @param replicationSubnetGroupIdentifier 40 | * @param subnetIds 41 | * @returns 42 | */ 43 | public createSubnetGroup(props: DmsProps): CfnReplicationSubnetGroup { 44 | const subnet = new CfnReplicationSubnetGroup(this, 'dms-subnet-group', { 45 | replicationSubnetGroupIdentifier: props.replicationSubnetGroupIdentifier, 46 | replicationSubnetGroupDescription: 'Private subnets that have access to my data source and target', 47 | subnetIds: props.subnetIds, 48 | }); 49 | return subnet; 50 | } 51 | 52 | /** 53 | * Creates IAM role to access secrets manager for provisioning DMS endpoints. 54 | * 55 | * @returns 56 | */ 57 | public createRoleForSecretsManagerAccess(): Role { 58 | const role = new Role(this, 'dms-secretsmgr-access-role', { 59 | assumedBy: new ServicePrincipal(`dms.${this.region}.amazonaws.com`), 60 | }); 61 | 62 | role.addToPolicy( 63 | new PolicyStatement({ 64 | resources: ['*'], 65 | actions: [ 66 | 'secretsmanager:GetSecretValue', 67 | 'secretsmanager:DescribeSecret', 68 | 'secretsmanager:ListSecretVersionIds', 69 | 'secretsmanager:ListSecrets', 70 | ], 71 | }) 72 | ); 73 | 74 | return role; 75 | } 76 | 77 | /** 78 | * Creates replication instance where DMS multiple DMS replication tasks runs. 79 | * 80 | * @param props 81 | * @returns 82 | */ 83 | private createReplicationInstance(props: DmsProps): CfnReplicationInstance { 84 | const instance = new CfnReplicationInstance(this, 'dms-replication-instance', { 85 | replicationInstanceClass: props.replicationInstanceClass || 'dms.t3.medium', 86 | replicationSubnetGroupIdentifier: props.replicationSubnetGroupIdentifier, 87 | replicationInstanceIdentifier: props.replicationInstanceIdentifier, 88 | vpcSecurityGroupIds: props.vpcSecurityGroupIds, 89 | allocatedStorage: props.allocatedStorage, 90 | publiclyAccessible: props.publiclyAccessible, 91 | engineVersion: props.engineVersion, 92 | }); 93 | 94 | return instance; 95 | } 96 | 97 | /** 98 | * Creates endpoints for DMS tasks targeted for MySQL DB. Endpoint represents database source and target destionation. 99 | * 100 | * @param endpointIdentifier 101 | * @param endpointType 102 | * @param secretId - Secret ARN for the database which contains database, user, password... 103 | * @param secretAccessRoleArn Role to be assumed by DMS to access the secrets store 104 | * @returns CfnEndpoint 105 | */ 106 | public createMySQLEndpoint( 107 | endpointIdentifier: string, 108 | endpointType: 'source' | 'target', 109 | secretId: string 110 | ): CfnEndpoint { 111 | const target_extra_conn_attr = 'parallelLoadThreads=1 maxFileSize=512'; 112 | const endpoint = new CfnEndpoint(this, `mysql-${endpointType}-${endpointIdentifier}`, { 113 | endpointIdentifier, 114 | endpointType, 115 | engineName: 'mysql', 116 | mySqlSettings: { 117 | secretsManagerAccessRoleArn: this.secretsManagerAccessRole.roleArn, 118 | secretsManagerSecretId: secretId, 119 | }, 120 | extraConnectionAttributes: endpointType === 'source' ? 'parallelLoadThreads=1' : target_extra_conn_attr, 121 | }); 122 | 123 | return endpoint; 124 | } 125 | 126 | /** 127 | * Creates endpoint for oracle database 128 | * 129 | * @param endpointIdentifier 130 | * @param endpointType 131 | * @param secretId 132 | * @param databaseName 133 | * @returns 134 | */ 135 | public createOracleEndpoint( 136 | endpointIdentifier: string, 137 | endpointType: 'source' | 'target', 138 | secretId: string, 139 | databaseName: string 140 | ): CfnEndpoint { 141 | const target_extra_conn_attr = 142 | 'useLogMinerReader=N;useBfile=Y;failTasksOnLobTruncation=true;numberDataTypeScale=-2'; 143 | const endpoint = new CfnEndpoint(this, `oracle-${endpointType}-${endpointIdentifier}`, { 144 | endpointIdentifier, 145 | endpointType, 146 | engineName: 'oracle', 147 | databaseName, 148 | oracleSettings: { 149 | secretsManagerAccessRoleArn: this.secretsManagerAccessRole.roleArn, 150 | secretsManagerSecretId: secretId, 151 | }, 152 | extraConnectionAttributes: endpointType === 'source' ? 'addSupplementalLogging=true' : target_extra_conn_attr, 153 | }); 154 | 155 | return endpoint; 156 | } 157 | 158 | /** 159 | * Creates endpoint for Aurora PostgresSql 160 | * 161 | * @param endpointIdentifier 162 | * @param endpointType 163 | * @param secretId 164 | * @param databaseName 165 | * @returns 166 | */ 167 | public createAuroraPostgresEndpoint( 168 | endpointIdentifier: string, 169 | endpointType: 'source' | 'target', 170 | secretId: string, 171 | databaseName: string 172 | ): CfnEndpoint { 173 | const target_extra_conn_attr = 'executeTimeout=180'; 174 | const endpoint = new CfnEndpoint(this, `aurora-postgresql-${endpointType}-${endpointIdentifier}`, { 175 | endpointIdentifier, 176 | endpointType, 177 | engineName: 'aurora-postgresql', 178 | databaseName, 179 | postgreSqlSettings: { 180 | secretsManagerAccessRoleArn: this.secretsManagerAccessRole.roleArn, 181 | secretsManagerSecretId: secretId, 182 | }, 183 | extraConnectionAttributes: endpointType === 'source' ? 'heartbeatFrequency=5' : target_extra_conn_attr, 184 | }); 185 | 186 | return endpoint; 187 | } 188 | 189 | /** 190 | * Creates endpoint for PostgresSql 191 | * 192 | * @param endpointIdentifier 193 | * @param endpointType 194 | * @param secretId 195 | * @param databaseName 196 | * @returns 197 | */ 198 | public createPostgresEndpoint( 199 | endpointIdentifier: string, 200 | endpointType: 'source' | 'target', 201 | secretId: string, 202 | databaseName: string 203 | ): CfnEndpoint { 204 | const target_extra_conn_attr = 'executeTimeout=180'; 205 | const endpoint = new CfnEndpoint(this, `postgresql-${endpointType}-${endpointIdentifier}`, { 206 | endpointIdentifier, 207 | endpointType, 208 | engineName: 'postgres', 209 | databaseName, 210 | postgreSqlSettings: { 211 | secretsManagerAccessRoleArn: this.secretsManagerAccessRole.roleArn, 212 | secretsManagerSecretId: secretId, 213 | }, 214 | extraConnectionAttributes: endpointType === 'source' ? 'heartbeatFrequency=5' : target_extra_conn_attr, 215 | }); 216 | 217 | return endpoint; 218 | } 219 | 220 | /** 221 | * Creates endpoint for Microsoft SqlServer 222 | * 223 | * @param endpointIdentifier 224 | * @param endpointType 225 | * @param secretId 226 | * @param databaseName 227 | * @returns 228 | */ 229 | public createSqlServerEndpoint( 230 | endpointIdentifier: string, 231 | endpointType: 'source' | 'target', 232 | secretId: string, 233 | databaseName: string 234 | ): CfnEndpoint { 235 | const endpoint = new CfnEndpoint(this, `sqlserver-${endpointType}-${endpointIdentifier}`, { 236 | endpointIdentifier, 237 | endpointType, 238 | engineName: 'sqlserver', 239 | databaseName, 240 | microsoftSqlServerSettings: { 241 | secretsManagerAccessRoleArn: this.secretsManagerAccessRole.roleArn, 242 | secretsManagerSecretId: secretId, 243 | }, 244 | }); 245 | 246 | return endpoint; 247 | } 248 | 249 | /** 250 | * Creates replication tasks for per schema. Replication tasks runs in replication instance. 251 | * 252 | * @param props DMS properties 253 | * @returns CfnReplicationTask 254 | */ 255 | public createReplicationTask( 256 | replicationTaskIdentifier: string, 257 | source: CfnEndpoint, 258 | target: CfnEndpoint, 259 | migrationType: 'cdc' | 'full-load' | 'full-load-and-cdc', 260 | rules: Rules 261 | ): CfnReplicationTask { 262 | const replicationTask = new CfnReplicationTask(this, replicationTaskIdentifier, { 263 | replicationInstanceArn: this.instance.ref, 264 | replicationTaskIdentifier, 265 | migrationType: migrationType || 'full-load', 266 | sourceEndpointArn: source.ref, 267 | targetEndpointArn: target.ref, 268 | replicationTaskSettings: JSON.stringify(TaskSettings), 269 | tableMappings: JSON.stringify(rules), 270 | }); 271 | return replicationTask; 272 | } 273 | } 274 | 275 | export { DmsReplication }; 276 | -------------------------------------------------------------------------------- /lib/dms_stack.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from 'aws-cdk-lib'; 2 | import { Construct } from 'constructs'; 3 | import { DmsReplication } from './dms-replication'; 4 | import { DmsProps } from './dms-props'; 5 | 6 | class DmsStack extends cdk.Stack { 7 | public dmsReplication; 8 | 9 | constructor(scope: Construct, id: string, props: DmsProps) { 10 | super(scope, id, props); 11 | 12 | this.dmsReplication = new DmsReplication(this, 'Replication', props); 13 | const suffix = props.replicationInstanceIdentifier; 14 | 15 | // Creates DMS Task for each schema 16 | props.tasks.forEach(task => { 17 | let source; 18 | let target; 19 | 20 | // endpoints cannot have underscore 21 | const schemaName = task.name.includes('_') ? task.name.replace('_', '-') : task.name; 22 | 23 | // source 24 | switch (task.engineName) { 25 | case 'mysql': 26 | source = this.dmsReplication.createMySQLEndpoint( 27 | `source-${schemaName}-${suffix}`, 28 | 'source', 29 | task.sourceSecretsManagerSecretId 30 | ); 31 | break; 32 | case 'oracle': 33 | source = this.dmsReplication.createOracleEndpoint( 34 | `source-${schemaName}-${suffix}`, 35 | 'source', 36 | task.sourceSecretsManagerSecretId, 37 | task.databaseName! 38 | ); 39 | break; 40 | case 'sqlserver': 41 | source = this.dmsReplication.createSqlServerEndpoint( 42 | `source-${schemaName}-${suffix}`, 43 | 'source', 44 | task.sourceSecretsManagerSecretId, 45 | task.databaseName! 46 | ); 47 | break; 48 | case 'postgres': 49 | source = this.dmsReplication.createPostgresEndpoint( 50 | `source-${schemaName}-${suffix}`, 51 | 'source', 52 | task.sourceSecretsManagerSecretId, 53 | task.databaseName! 54 | ); 55 | break; 56 | default: 57 | source = this.dmsReplication.createMySQLEndpoint( 58 | `source-${schemaName}-${suffix}`, 59 | 'source', 60 | task.sourceSecretsManagerSecretId 61 | ); 62 | break; 63 | } 64 | 65 | // target 66 | switch (task.targetEngineName) { 67 | case 'mysql': 68 | target = this.dmsReplication.createMySQLEndpoint( 69 | `target-${schemaName}-${suffix}`, 70 | 'target', 71 | task.targetSecretsManagerSecretId 72 | ); 73 | break; 74 | case 'oracle': 75 | target = this.dmsReplication.createOracleEndpoint( 76 | `target-${schemaName}-${suffix}`, 77 | 'target', 78 | task.targetSecretsManagerSecretId, 79 | task.databaseName! 80 | ); 81 | break; 82 | case 'aurora-postgresql': 83 | target = this.dmsReplication.createAuroraPostgresEndpoint( 84 | `target-${schemaName}-${suffix}`, 85 | 'target', 86 | task.targetSecretsManagerSecretId, 87 | task.databaseName! 88 | ); 89 | break; 90 | case 'sqlserver': 91 | target = this.dmsReplication.createSqlServerEndpoint( 92 | `target-${schemaName}-${suffix}`, 93 | 'target', 94 | task.targetSecretsManagerSecretId, 95 | task.databaseName! 96 | ); 97 | break; 98 | case 'postgres': 99 | target = this.dmsReplication.createPostgresEndpoint( 100 | `target-${schemaName}-${suffix}`, 101 | 'target', 102 | task.targetSecretsManagerSecretId, 103 | task.databaseName! 104 | ); 105 | break; 106 | default: 107 | target = this.dmsReplication.createMySQLEndpoint( 108 | `target-${schemaName}-${suffix}`, 109 | 'target', 110 | task.targetSecretsManagerSecretId 111 | ); 112 | break; 113 | } 114 | 115 | this.dmsReplication.createReplicationTask( 116 | `${schemaName}-replication-${suffix}`, 117 | source, 118 | target, 119 | task.migrationType, 120 | task.tableMappings 121 | ); 122 | }); 123 | } 124 | } 125 | export default DmsStack; 126 | -------------------------------------------------------------------------------- /lib/rules_props.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable prettier/prettier */ 2 | 3 | export interface DataType { 4 | 'type': 'date' | 'time' | 'datetime' | 'int1' | 'int2' | 'int4' | 'int8' | 'numeric' | 'real4' | 'real8' | 'string' | 'uint1' | 'uint2' | 'uint4' | 'uint8' | 'wstring' | 'blob' | 'nclob' | 'clob' | 'boolean' | 'set' | 'list' | 'map' | 'tuple'; 5 | 'length'?: number; 6 | 'precision'?: number; 7 | 'scale'?: number; 8 | }; 9 | 10 | export interface RuleObjectLocator { 11 | 'schema-name': string; 12 | 'table-name'?: string; 13 | 'table-tablespace-name'?: string; 14 | 'index-tablespace-name'?: string; 15 | 'table-type'?: 'table' | 'view' | 'all'; 16 | 'column-name'?: string; 17 | 'data-type'?: DataType; 18 | }; 19 | 20 | export type FilterCondition = { 21 | 'filter-operator'?: 'lte' | 'ste' | 'gte' | 'eq' | 'noteq' | 'between' | 'notbetween' | 'null' | 'notnull'; 22 | 'value'?: string; 23 | 'start-value'?: string; 24 | 'end-value'?: string; 25 | }; 26 | 27 | export type Filter = { 28 | 'filter-type': string; 29 | 'column-name'?: string; 30 | 'filter-conditions'?: FilterCondition[]; 31 | }; 32 | 33 | export interface BaseRule {}; 34 | 35 | export interface BeforeImageDef { 36 | 'column-prefix'?: string; 37 | 'column-suffix'?: string; 38 | 'column-filter'?: 'pk-only' | 'non-blob' | 'all'; 39 | } 40 | 41 | export interface ParallelLoad { 42 | 'type'?: 'partitions-auto' | 'subpartitions-auto' | 'partitions-list' | 'ranges' | 'none'; 43 | 'partitions'?: string; 44 | 'subpartitions'?: string; 45 | 'columns'?: string; 46 | 'boundaries'?: string; 47 | 'number-of-partitions'?: number; 48 | 'collection-count-from-metadata'?: 'true' | 'false'; 49 | 'max-records-skip-per-page'?: number; 50 | 'batch-size'?: number; 51 | 'lob-settings'?: string; 52 | 'mode'?: 'limited' | 'unlimited' | 'none'; 53 | 'bulk-max-size'?: string 54 | } 55 | 56 | export interface SelectionRule extends BaseRule { 57 | 'rule-type': 'selection'; 58 | 'rule-id': number; 59 | 'rule-name': string; 60 | 'rule-action'?: 'include' | 'exclude' | 'explicit'; 61 | 'object-locator'?: RuleObjectLocator; 62 | 'load-order'?: number; 63 | 'filters'?: Filter[] 64 | }; 65 | 66 | 67 | export interface TransformationRule extends BaseRule { 68 | 'rule-type': 'transformation'; 69 | 'rule-id': number; 70 | 'rule-name': string; 71 | 'object-locator'?: RuleObjectLocator; 72 | 'rule-action'?: 'add-column' | 'include-column' | 'remove-column' | 'rename' | 'convert-lowercase' | 'convert-uppercase' | 'add-prefix' | 'remove-prefix' | 'replace-prefix' | 'add-suffix' | 'remove-suffix' | 'replace-suffix' | 'define-primary-key' | 'change-data-type' | 'add-before-image-columns'; 73 | 'rule-target'?: 'schema' | 'table' | 'column' | 'table-tablespace' | 'index-tablespace'; 74 | 'value'?: string | null; 75 | 'old-value'?: string | null; 76 | 'data-type'?: DataType; 77 | 'expression'?: string; 78 | 'primary-key-def'?: string; 79 | 'before-image-def'?: BeforeImageDef; 80 | 'filters'?: Filter[] 81 | }; 82 | export interface TableSettingsRule extends BaseRule { 83 | 'rule-type': 'table-settings'; 84 | 'rule-id': number; 85 | 'rule-name': string; 86 | 'object-locator'?: RuleObjectLocator; 87 | 'parallel-load'?: ParallelLoad; 88 | }; 89 | 90 | export type Rules = { 91 | 'rules': BaseRule[]; 92 | }; 93 | 94 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.1.0", 3 | "main": "index.js", 4 | "bin": { 5 | "dms-cdk": "bin/dms-cdk.js" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "test": "jest", 11 | "cdk": "cdk", 12 | "lint": "eslint . --ext .js,.jsx,.ts,.tsx", 13 | "lint:fix": "npm run lint -- --fix", 14 | "lint:watch": "esw . --ext .js,.jsx,.ts,.tsx -w --changed --clear", 15 | "prepare": "husky install" 16 | }, 17 | "devDependencies": { 18 | "@commitlint/cli": "17.0.3", 19 | "@commitlint/config-angular": "17.0.3", 20 | "@commitlint/config-conventional": "17.0.3", 21 | "@semantic-release/changelog": "6.0.1", 22 | "@semantic-release/git": "10.0.1", 23 | "aws-cdk": "2.30.0", 24 | "@typescript-eslint/parser": "5.26.0", 25 | "@typescript-eslint/eslint-plugin": "5.26.0", 26 | "@types/node": "17.0.35", 27 | "@types/jest": "27.5.2", 28 | "eslint": "8.18.0", 29 | "eslint-config-airbnb-base": "15.0.0", 30 | "eslint-config-prettier": "8.5.0", 31 | "eslint-plugin-import": "2.26.0", 32 | "eslint-plugin-jest": "26.5.3", 33 | "eslint-plugin-prettier": "4.2.1", 34 | "husky": "^8.0.1", 35 | "jest": "28.1.0", 36 | "lint-staged": "12.4.1", 37 | "prettier": "2.6.2", 38 | "ts-jest": "28.0.2", 39 | "ts-node": "10.7.0", 40 | "typescript": "4.6.4" 41 | }, 42 | "dependencies": { 43 | "aws-cdk-lib": "2.80.0", 44 | "cdk-assets": "2.31.0", 45 | "constructs": "10.1.1", 46 | "npm": "8.11.0", 47 | "promptly": "3.2.0", 48 | "source-map-support": "0.5.21" 49 | }, 50 | "husky": { 51 | "hooks": { 52 | "pre-commit": "lint-staged" 53 | } 54 | }, 55 | "lint-staged": { 56 | "*.{js,jsx,ts,tsx}": [ 57 | "prettier --write", 58 | "eslint --cache --fix" 59 | ] 60 | } 61 | } 62 | 63 | -------------------------------------------------------------------------------- /test/dms-replication.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable prettier/prettier */ 2 | /* eslint-disable jest/expect-expect */ 3 | import * as cdk from 'aws-cdk-lib'; 4 | import { Template } from 'aws-cdk-lib/assertions'; 5 | import DmsStack from '../lib/dms-stack'; 6 | 7 | let template: Template; 8 | let stack: DmsStack; 9 | 10 | test('init stack', () => { 11 | const app = new cdk.App(); 12 | 13 | stack = new DmsStack(app, 'DmsTestStack', { 14 | vpcId: 'vpc-id', 15 | subnetIds: ['subnet-1', 'subnet-2'], 16 | replicationInstanceClass: 'dms-mysql-uk', 17 | replicationInstanceIdentifier: 'test-repl-01', 18 | replicationSubnetGroupIdentifier: 'subnet-group', 19 | vpcSecurityGroupIds: ['vpc-security'], 20 | engineVersion: '3.4.7', 21 | tasks: [ 22 | { 23 | name: 'demo_test', 24 | sourceSecretsManagerSecretId: 'sourceSecretsManagerSecretId', 25 | targetSecretsManagerSecretId: 'targetSecretMgrId', 26 | migrationType: 'full-load', 27 | engineName: 'mysql', 28 | targetEngineName: 'oracle', 29 | tableMappings: { 30 | rules: [ 31 | { 32 | 'rule-type': 'selection', 33 | 'rule-id': '1', 34 | 'rule-name': '1', 35 | 'object-locator': { 36 | 'schema-name': 'demo_schema', 37 | 'table-name': '%', 38 | }, 39 | 'rule-action': 'include', 40 | }, 41 | ], 42 | }, 43 | }, 44 | ], 45 | publiclyAccessible: false, 46 | allocatedStorage: 50, 47 | env: { 48 | account: '11111111111', 49 | region: 'eu-central-1', 50 | }, 51 | }); 52 | 53 | template = Template.fromStack(stack); 54 | }); 55 | 56 | test('should have AWS::DMS::ReplicationInstance', () => { 57 | template.hasResourceProperties('AWS::DMS::ReplicationInstance', { 58 | ReplicationInstanceClass: 'dms-mysql-uk', 59 | ReplicationInstanceIdentifier: 'test-repl-01', 60 | AllocatedStorage: 50, 61 | VpcSecurityGroupIds: ['vpc-security'], 62 | EngineVersion: '3.4.7', 63 | }); 64 | }); 65 | 66 | test('should have AWS::DMS::ReplicationSubnetGroup', () => { 67 | template.hasResourceProperties('AWS::DMS::ReplicationSubnetGroup', { 68 | ReplicationSubnetGroupIdentifier: 'subnet-group', 69 | }); 70 | }); 71 | 72 | test('should allocated storage AWS::DMS::ReplicationInstance', () => { 73 | template.hasResourceProperties('AWS::DMS::ReplicationInstance', { 74 | ReplicationInstanceClass: 'dms-mysql-uk', 75 | AllocatedStorage: 50, 76 | }); 77 | }); 78 | 79 | test('target endpoint created with correct attributes', () => { 80 | template.hasResourceProperties('AWS::DMS::Endpoint', { 81 | EndpointType: 'source', 82 | EngineName: 'mysql', 83 | ExtraConnectionAttributes: 'parallelLoadThreads=1', 84 | MySqlSettings: { 85 | SecretsManagerSecretId: 'sourceSecretsManagerSecretId', 86 | }, 87 | }); 88 | 89 | template.hasResourceProperties('AWS::DMS::Endpoint', { 90 | EndpointType: 'target', 91 | EngineName: 'oracle', 92 | }); 93 | }); 94 | 95 | test('create replication task', () => { 96 | template.hasResourceProperties('AWS::DMS::ReplicationTask', { 97 | MigrationType: 'full-load', 98 | ReplicationTaskIdentifier: 'demo-test-replication-test-repl-01', 99 | TableMappings: 100 | '{"rules":[{"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"demo_schema","table-name":"%"},"rule-action":"include"}]}', 101 | }); 102 | }); 103 | 104 | // Oracle transformation rules test 105 | test('oracle transformation rules and attributes', () => { 106 | const app = new cdk.App(); 107 | const oracleStack = new DmsStack(app, 'DmsOraclePostGresStack', { 108 | vpcId: 'vpc-id', 109 | subnetIds: ['subnet-1a', 'subnet-1b'], 110 | replicationInstanceClass: 'dms.r5.4xlarge', 111 | replicationInstanceIdentifier: 'test-repl-01', 112 | replicationSubnetGroupIdentifier: 'subnet-group', 113 | vpcSecurityGroupIds: ['vpc-sg'], 114 | engineVersion: '3.4.6', 115 | tasks: [ 116 | { 117 | name: 'demo_stack', 118 | sourceSecretsManagerSecretId: 'sourceSecretsManagerSecretId', 119 | targetSecretsManagerSecretId: 'targetSecretsManagerSecretId', 120 | migrationType: 'cdc', 121 | engineName: 'oracle', 122 | targetEngineName: 'aurora-postgresql', 123 | tableMappings: { 124 | rules: [ 125 | { 126 | 'rule-type': 'transformation', 127 | 'rule-id': '1', 128 | 'rule-name': 'Default Lowercase Table Rule', 129 | 'rule-target': 'table', 130 | 'object-locator': { 131 | 'schema-name': 'DMS_SAMPLE', 132 | 'table-name': '%', 133 | }, 134 | 'rule-action': 'convert-lowercase', 135 | 'value': null, 136 | 'old-value': null 137 | }, 138 | { 139 | 'rule-type': 'transformation', 140 | 'rule-id': '2', 141 | 'rule-name': 'Default Lowercase Schema Rule', 142 | 'rule-action': 'convert-lowercase', 143 | 'rule-target': 'schema', 144 | 'object-locator': { 145 | 'schema-name': 'DMS_SAMPLE', 146 | }, 147 | }, 148 | { 149 | 'rule-type': 'transformation', 150 | 'rule-id': '3', 151 | 'rule-name': 'Default Lowercase Column Rule', 152 | 'rule-action': 'convert-lowercase', 153 | 'rule-target': 'column', 154 | 'object-locator': { 155 | 'schema-name': 'DMS_SAMPLE', 156 | 'table-name': '%', 157 | 'column-name': '%', 158 | }, 159 | }, 160 | { 161 | 'rule-type': 'transformation', 162 | 'rule-id': '10', 163 | 'rule-name': 'Rename Schema Rule', 164 | 'rule-target': 'schema', 165 | 'object-locator': { 166 | 'schema-name': 'DMS_SAMPLE', 167 | }, 168 | 'rule-action': 'rename', 169 | 'value': 'dms_sample', 170 | 'old-value': null 171 | }, 172 | { 173 | 'rule-type': 'selection', 174 | 'rule-id': '11', 175 | 'rule-name': 'Selection Rule DMS_SAMPLE', 176 | 'object-locator': { 177 | 'schema-name': 'DMS_SAMPLE', 178 | 'table-name': '%', 179 | }, 180 | 'rule-action': 'include', 181 | 'filters': [] 182 | }, 183 | { 184 | 'rule-action': 'change-data-type', 185 | 'object-locator': { 186 | 'schema-name': 'DMS_SAMPLE', 187 | 'column-name': 'COL1_NUMBER_INT', 188 | 'table-name': 'SAMPLE_NUMBER_DATA_TYPE', 189 | }, 190 | 'rule-target': 'column', 191 | 'rule-type': 'transformation', 192 | 'rule-id': '1', 193 | 'data-type': { 194 | 'type': 'numeric', 195 | }, 196 | 'rule-name': '30', 197 | }, 198 | ], 199 | }, 200 | }, 201 | ], 202 | publiclyAccessible: true, 203 | allocatedStorage: 50, 204 | env: { 205 | account: '11111111111', 206 | region: 'eu-central-1', 207 | }, 208 | }); 209 | 210 | const oracleTemplate = Template.fromStack(oracleStack); 211 | 212 | oracleTemplate.hasResourceProperties('AWS::DMS::Endpoint', { 213 | EndpointType: 'source', 214 | EngineName: 'oracle', 215 | }); 216 | }); 217 | -------------------------------------------------------------------------------- /test/dms-stack.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable jest/expect-expect */ 2 | import * as cdk from 'aws-cdk-lib'; 3 | import { Template } from 'aws-cdk-lib/assertions'; 4 | import DmsStack from '../lib/dms-stack'; 5 | 6 | let stackOracleToPostgres: DmsStack; 7 | let stackMultiSchemaMySql: DmsStack; 8 | let template: Template; 9 | let multiSchemaTemplate: Template; 10 | 11 | test('init stack', () => { 12 | const app = new cdk.App(); 13 | 14 | stackOracleToPostgres = new DmsStack(app, 'DmsOraclePostGresStack', { 15 | vpcId: 'vpc-id', 16 | subnetIds: ['subnet-1a', 'subnet-1b'], 17 | replicationInstanceClass: 'dms.r5.4xlarge', 18 | replicationInstanceIdentifier: 'test-repl-01', 19 | replicationSubnetGroupIdentifier: 'subnet-group', 20 | vpcSecurityGroupIds: ['vpc-sg'], 21 | engineVersion: '3.4.6', 22 | tasks: [ 23 | { 24 | name: 'demo_stack', 25 | sourceSecretsManagerSecretId: 'sourceSecretsManagerSecretId', 26 | targetSecretsManagerSecretId: 'targetSecretsManagerSecretId', 27 | migrationType: 'cdc', 28 | engineName: 'oracle', 29 | targetEngineName: 'aurora-postgresql', 30 | tableMappings: { 31 | rules: [ 32 | { 33 | 'rule-type': 'selection', 34 | 'rule-id': '1', 35 | 'rule-name': '1', 36 | 'object-locator': { 37 | 'schema-name': 'demo_test', 38 | 'table-name': '%', 39 | }, 40 | 'rule-action': 'include', 41 | }, 42 | ], 43 | }, 44 | }, 45 | ], 46 | publiclyAccessible: true, 47 | allocatedStorage: 50, 48 | env: { 49 | account: '11111111111', 50 | region: 'eu-central-1', 51 | }, 52 | }); 53 | 54 | stackMultiSchemaMySql = new DmsStack(app, 'DmsMultiSchemaStack', { 55 | vpcId: 'vpc-id', 56 | subnetIds: ['subnet-1a', 'subnet-1b'], 57 | replicationInstanceClass: 'dms.r5.4xlarge', 58 | replicationInstanceIdentifier: 'test-repl-01', 59 | replicationSubnetGroupIdentifier: 'subnet-group', 60 | vpcSecurityGroupIds: ['vpc-sg'], 61 | engineVersion: '3.4.6', 62 | tasks: [ 63 | { 64 | name: 'task_1', 65 | sourceSecretsManagerSecretId: 'sourceSecretsManagerSecretId', 66 | targetSecretsManagerSecretId: 'targetSecretsManagerSecretId', 67 | migrationType: 'full-load', 68 | engineName: 'mysql', 69 | targetEngineName: 'mysql', 70 | tableMappings: { 71 | rules: [ 72 | { 73 | 'rule-type': 'selection', 74 | 'rule-id': '1', 75 | 'rule-name': '1', 76 | 'object-locator': { 77 | 'schema-name': 'Database_1', 78 | 'table-name': '%', 79 | }, 80 | 'rule-action': 'include', 81 | }, 82 | ], 83 | }, 84 | }, 85 | { 86 | name: 'task_2', 87 | sourceSecretsManagerSecretId: 'sourceSecretsManagerSecretId', 88 | targetSecretsManagerSecretId: 'targetSecretsManagerSecretId', 89 | migrationType: 'cdc', 90 | engineName: 'mysql', 91 | targetEngineName: 'sqlserver', 92 | tableMappings: { 93 | rules: [ 94 | { 95 | 'rule-type': 'selection', 96 | 'rule-id': '1', 97 | 'rule-name': '1', 98 | 'object-locator': { 99 | 'schema-name': 'Database_2', 100 | 'table-name': '%', 101 | }, 102 | 'rule-action': 'include', 103 | }, 104 | ], 105 | }, 106 | }, 107 | { 108 | name: 'task_3', 109 | sourceSecretsManagerSecretId: 'sourceSecretsManagerSecretId', 110 | targetSecretsManagerSecretId: 'targetSecretsManagerSecretId', 111 | migrationType: 'full-load', 112 | engineName: 'mysql', 113 | targetEngineName: 'aurora-postgresql', 114 | tableMappings: { 115 | rules: [ 116 | { 117 | 'rule-type': 'selection', 118 | 'rule-id': '1', 119 | 'rule-name': '1', 120 | 'object-locator': { 121 | 'schema-name': 'Database_3', 122 | 'table-name': '%', 123 | }, 124 | 'rule-action': 'include', 125 | }, 126 | ], 127 | }, 128 | }, 129 | ], 130 | publiclyAccessible: false, 131 | allocatedStorage: 50, 132 | env: { 133 | account: '11111111111', 134 | region: 'eu-central-1', 135 | }, 136 | }); 137 | 138 | template = Template.fromStack(stackOracleToPostgres); 139 | multiSchemaTemplate = Template.fromStack(stackMultiSchemaMySql); 140 | }); 141 | 142 | test('subnet group', () => { 143 | template.hasResourceProperties('AWS::DMS::ReplicationSubnetGroup', { 144 | SubnetIds: ['subnet-1a', 'subnet-1b'], 145 | }); 146 | }); 147 | 148 | test('replication instance class and engine version', () => { 149 | template.hasResourceProperties('AWS::DMS::ReplicationInstance', { 150 | ReplicationInstanceClass: 'dms.r5.4xlarge', 151 | VpcSecurityGroupIds: ['vpc-sg'], 152 | EngineVersion: '3.4.6', 153 | }); 154 | }); 155 | 156 | test('DMS endpoints', () => { 157 | template.hasResourceProperties('AWS::DMS::Endpoint', { 158 | EndpointType: 'source', 159 | EndpointIdentifier: 'source-demo-stack-test-repl-01', 160 | EngineName: 'oracle', 161 | ExtraConnectionAttributes: 'addSupplementalLogging=true', 162 | OracleSettings: { 163 | SecretsManagerSecretId: 'sourceSecretsManagerSecretId', 164 | }, 165 | }); 166 | 167 | template.hasResourceProperties('AWS::DMS::Endpoint', { 168 | EndpointType: 'target', 169 | EndpointIdentifier: 'target-demo-stack-test-repl-01', 170 | EngineName: 'aurora-postgresql', 171 | ExtraConnectionAttributes: 'executeTimeout=180', 172 | PostgreSqlSettings: { 173 | SecretsManagerSecretId: 'targetSecretsManagerSecretId', 174 | }, 175 | }); 176 | 177 | template.resourceCountIs('AWS::DMS::Endpoint', 2); 178 | }); 179 | 180 | test('Publicly accessible is true', () => { 181 | template.hasResourceProperties('AWS::DMS::ReplicationInstance', { 182 | ReplicationInstanceClass: 'dms.r5.4xlarge', 183 | PubliclyAccessible: true, 184 | }); 185 | }); 186 | 187 | // Multiple schemas test 188 | test('Multiple schemas test', () => { 189 | multiSchemaTemplate.hasResourceProperties('AWS::DMS::ReplicationTask', { 190 | MigrationType: 'full-load', 191 | TableMappings: 192 | '{"rules":[{"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"Database_1","table-name":"%"},"rule-action":"include"}]}', 193 | }); 194 | 195 | multiSchemaTemplate.hasResourceProperties('AWS::DMS::ReplicationTask', { 196 | MigrationType: 'cdc', 197 | TableMappings: 198 | '{"rules":[{"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"Database_2","table-name":"%"},"rule-action":"include"}]}', 199 | }); 200 | 201 | multiSchemaTemplate.hasResourceProperties('AWS::DMS::ReplicationTask', { 202 | MigrationType: 'full-load', 203 | TableMappings: 204 | '{"rules":[{"rule-type":"selection","rule-id":"1","rule-name":"1","object-locator":{"schema-name":"Database_3","table-name":"%"},"rule-action":"include"}]}', 205 | }); 206 | 207 | multiSchemaTemplate.resourceCountIs('AWS::DMS::ReplicationTask', 3); 208 | multiSchemaTemplate.resourceCountIs('AWS::DMS::ReplicationInstance', 1); 209 | }); 210 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2018", 4 | "module": "commonjs", 5 | "lib": [ 6 | "es2018" 7 | ], 8 | "declaration": true, 9 | "strict": true, 10 | "noImplicitAny": true, 11 | "strictNullChecks": true, 12 | "noImplicitThis": true, 13 | "alwaysStrict": true, 14 | "noUnusedLocals": false, 15 | "noUnusedParameters": false, 16 | "noImplicitReturns": true, 17 | "noFallthroughCasesInSwitch": false, 18 | "inlineSourceMap": true, 19 | "inlineSources": true, 20 | "experimentalDecorators": true, 21 | "strictPropertyInitialization": false, 22 | "typeRoots": [ 23 | "./node_modules/@types" 24 | ], 25 | "resolveJsonModule": true, 26 | "esModuleInterop": true 27 | }, 28 | "exclude": [ 29 | "node_modules", 30 | "cdk.out" 31 | ] 32 | } --------------------------------------------------------------------------------