├── .DS_Store ├── BigQuery-Redshift-Migration └── BQtoS3Glue.py ├── CDKstaging.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Config ├── InputCIDR.png ├── InputRegion.png ├── LICENSE ├── README.md ├── Redshift-Loader ├── .DS_Store ├── CODE-OF-CONDUCT ├── CONTRIBUTING ├── CloudFormation │ └── .DS_Store ├── Images │ ├── .gitkeep │ ├── cfn_lambda_role.png │ └── redshift_loader.png ├── LICENSE ├── README.md ├── Sample_file_autoloader.zip ├── Test_Data_Files │ ├── .gitkeep │ ├── Test_Table_ddls.sql │ ├── Testing_Script.sql │ ├── customer.txt_0000_part_00.gz │ ├── customer.txt_0000_part_01.gz │ ├── customer.txt_0000_part_02.gz │ └── customer.txt_0000_part_03.gz └── loader_ui │ ├── LoaderUI.yaml │ ├── README.md │ ├── images │ ├── AWS_logo_RGB.png │ ├── aws_logo_favicon.ico │ └── loading.gif │ ├── index.html │ ├── js │ ├── amazon-cognito-identity.min.js │ ├── amazon-cognito-identity.min.js.map.txt │ ├── aws-cognito-sdk.min.js │ └── aws-cognito-sdk.min.js.map.txt │ ├── loader_ui_architecture_diagram.png │ ├── script.js │ └── styles.css ├── app.py ├── cdk.json ├── images ├── AccessKeys.png ├── InputCIDR.png ├── InputConfig.png ├── InputDBPassword.png ├── InputRedshiftPassword.png ├── InputRegion.png ├── InputStack.png ├── NewUsers.png ├── Policies.png ├── UploadConfig.png ├── UploadConfirmation.png ├── UploadLocation.png ├── exampleofinputs.png ├── menustart.png └── reconfigure.JPG ├── jmeterconfig.sh ├── jmeterconfig_2.sh ├── lambda_call_lambda_to_run_redshift_script.py ├── lambda_run_redshift_script.py ├── menu-script.sh ├── permissionlist.md ├── redshift_poc_automation ├── .DS_Store ├── __init__.py └── stacks │ ├── .DS_Store │ ├── data_sharing_consumer_stack.py │ ├── data_sharing_stack.py │ ├── dms_on_prem_to_redshift_stack.py │ ├── dmsinstance_stack.py │ ├── jmeter_stack.py │ ├── redshift-stack-test.py │ ├── redshift_stack.py │ ├── redshiftload_stack.py │ ├── redshiftrole_stack.py │ ├── redshiftserverless_stack.py │ ├── sct_stack.py │ └── vpc_stack.py ├── requirements.txt ├── scripts ├── Redshift Load Test.jmx ├── delete_buckets.py ├── delete_secrets.py ├── deploy.sh ├── destroy.sh ├── detach_policy.py ├── jmeter.bat ├── loadTPcH3TB.txt ├── redshift-jdbc42-2.0.0.4.jar ├── restart_session.sh └── shell_menu │ ├── bash-menu-cli-commands.sh │ ├── menu-script.sh │ ├── menu-welcome-message.sh │ ├── miscdetails.sh │ └── permission-check.sh ├── sctconfig.sh ├── sctconfig_2.sh ├── setup.py ├── source.bat ├── user-config-sample.json └── user-config-template.json /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/.DS_Store -------------------------------------------------------------------------------- /BigQuery-Redshift-Migration/BQtoS3Glue.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from awsglue.transforms import * 3 | from awsglue.utils import getResolvedOptions 4 | from pyspark.context import SparkContext 5 | from awsglue.context import GlueContext 6 | from awsglue.job import Job 7 | from pprint import pprint 8 | import boto3 9 | from boto3.dynamodb.conditions import Key 10 | import json 11 | import datetime 12 | from decimal import Decimal 13 | 14 | aws_session = boto3.Session() 15 | client = aws_session.client('s3') 16 | glue = boto3.client('glue') 17 | gluejobname="bq-s3-mig" 18 | 19 | args = getResolvedOptions(sys.argv, ["JOB_NAME", 'tablename', 'connectionname']) 20 | sc = SparkContext() 21 | glueContext = GlueContext(sc) 22 | spark = glueContext.spark_session 23 | job = Job(glueContext) 24 | job.init(args["JOB_NAME"], args) 25 | 26 | bucket = 'bigquery-migration-bucket' 27 | import_tablename = args['tablename'] 28 | import_connectionname = args['connectionname'] 29 | 30 | in_import_tablename = import_tablename.split(".",1)[1] 31 | in_import_tablename_db = import_tablename.split(".")[2] 32 | 33 | dynamodb = boto3.resource('dynamodb') 34 | table = dynamodb.Table('bq_to_s3_tracking_v4') 35 | 36 | if in_import_tablename.replace('/','') == 'employee.metadata_table': ## Update this to generic table name and folder names 37 | v_filepath = 's3://bigquery-migration-bucket/metadata-table-data/' + in_import_tablename + '/' 38 | else: 39 | v_filepath = 's3://s3loadertest-useast1/s3-redshift-loader-source/' + in_import_tablename + '/' 40 | 41 | table.update_item( 42 | Key={ 43 | 'table_name': in_import_tablename_db 44 | }, 45 | UpdateExpression="set updated_date = :r, load_status = :p, s3_path = :q", 46 | ExpressionAttributeValues={ 47 | ':r': str(datetime.datetime.now()), 48 | ':p': 'running', 49 | ':q': v_filepath 50 | }, 51 | ReturnValues="UPDATED_NEW" 52 | ) 53 | 54 | # Script generated for node Google BigQuery Connector 0.22.0 for AWS Glue 3.0 55 | GoogleBigQueryConnector0220forAWSGlue30_node1 = ( 56 | glueContext.create_dynamic_frame.from_options( 57 | connection_type="marketplace.spark", 58 | connection_options={ 59 | "parentProject": "ultra-unfolding-347804", 60 | "table": import_tablename, 61 | "connectionName": import_connectionname, 62 | }, 63 | transformation_ctx="GoogleBigQueryConnector0220forAWSGlue30_node1", 64 | ) 65 | ) 66 | 67 | # Script generated for node S3 bucket 68 | S3bucket_node3 = glueContext.write_dynamic_frame.from_options( 69 | frame=GoogleBigQueryConnector0220forAWSGlue30_node1, 70 | connection_type="s3", 71 | format="json", 72 | connection_options={"path": v_filepath, "partitionKeys": []}, 73 | transformation_ctx="S3bucket_node3", 74 | ) 75 | 76 | table.update_item( 77 | Key={ 78 | 'table_name': in_import_tablename_db 79 | }, 80 | UpdateExpression="set updated_date = :r, load_status = :p, s3_path = :q", 81 | ExpressionAttributeValues={ 82 | ':r': str(datetime.datetime.now()), 83 | ':p': 'completed', 84 | ':q': v_filepath 85 | }, 86 | ReturnValues="UPDATED_NEW" 87 | ) 88 | 89 | # update the dynamoDB table for status : Complete or aborted. 90 | job.commit() -------------------------------------------------------------------------------- /CDKstaging.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Description: 'Amazon Redshift POC resource auto-creation (To be deployed in your AWS account to execute the Redshift POC)' 3 | 4 | Metadata: 5 | AWS::CloudFormation::Interface: 6 | ParameterGroups: 7 | - 8 | Label: 9 | default: "Target Infrastructure" 10 | Parameters: 11 | - ConfigurationFile 12 | - 13 | Label: 14 | default: "Staging Infrastructure" 15 | Parameters: 16 | - EC2InstanceAMI 17 | - KeyPair 18 | - OnPremisesCIDR 19 | - SubnetID 20 | - 21 | Label: 22 | default: "Secret input" 23 | Parameters: 24 | - SourceDBPassword 25 | - TargetRedshiftPassword 26 | ParameterLabels: 27 | ConfigurationFile: 28 | default: "Configuration File" 29 | EC2InstanceAMI: 30 | default: "EC2 AMI" 31 | KeyPair: 32 | default: "Key Pair" 33 | OnPremisesCIDR: 34 | default: "On Prem CIDR" 35 | SubnetID: 36 | default: "Subnet ID" 37 | SourceDBPassword: 38 | default: "Source Password" 39 | type: String 40 | TargetRedshiftPassword: 41 | default: "Redshift Password" 42 | type: String 43 | 44 | Conditions: 45 | DBHasValue: !Not 46 | - !Equals 47 | - !Ref SourceDBPassword 48 | - '' 49 | RedshiftHasValue: !Not 50 | - !Equals 51 | - !Ref TargetRedshiftPassword 52 | - '' 53 | 54 | 55 | Parameters: 56 | SubnetID: 57 | Description: Select the subnet to launch the first staging instance in - ensure it is public and public IPs are auto-assigned 58 | Type: AWS::EC2::Subnet::Id 59 | ConfigurationFile: 60 | Description: The location (URI) for the configuration file on S3 61 | Type: String 62 | EC2InstanceAMI: 63 | Description: AMI for the Amazon Linux 2 based EC2 instance. Please don't change this parameter unless needed for some compliance requirement. 64 | Type: "AWS::SSM::Parameter::Value" 65 | Default: "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2" 66 | KeyPair: 67 | Description: Name of the keypair created in your account to be used in case you need to login to the EC2 instance 68 | Type: AWS::EC2::KeyPair::KeyName 69 | OnPremisesCIDR: 70 | Description: IP range (CIDR notation) of the infrastructure which needs to access the staging EC2 71 | Type: String 72 | Default: 10.0.0.0/8 73 | MinLength: '9' 74 | MaxLength: '18' 75 | AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})" 76 | ConstraintDescription: Must be a valid IP CIDR range of the form x.x.x.x/x. 77 | SourceDBPassword: 78 | Description: Password for the source system outside the target infrastructure (Optional -- use when connecting to external database) 79 | Default: '' 80 | Type: String 81 | NoEcho: 'true' 82 | TargetRedshiftPassword: 83 | Description: Password for the Redshfit cluster used in the architecture (Optional -- use when connecting to existing Redshift cluster) 84 | Default: '' 85 | Type: String 86 | NoEcho: 'true' 87 | 88 | Resources: 89 | 90 | SQLSecrets: 91 | Type: AWS::SecretsManager::Secret 92 | Condition: DBHasValue 93 | Properties: 94 | Name: 'SourceDBPassword' 95 | Description: 'Credentials for source system in DMS demo' 96 | SecretString: !Sub ${SourceDBPassword} 97 | RedshiftSecrets: 98 | Type: AWS::SecretsManager::Secret 99 | Condition: RedshiftHasValue 100 | Properties: 101 | Name: 'RedshiftPassword' 102 | Description: 'Redshift Cluster Secret' 103 | SecretString: !Sub ${TargetRedshiftPassword} 104 | 105 | LambdaExecutionRole: 106 | Type: AWS::IAM::Role 107 | Properties: 108 | AssumeRolePolicyDocument: 109 | Version: '2012-10-17' 110 | Statement: 111 | - Effect: Allow 112 | Principal: 113 | Service: 114 | - lambda.amazonaws.com 115 | Action: 116 | - sts:AssumeRole 117 | Path: "/" 118 | Policies: 119 | - PolicyName: root 120 | PolicyDocument: 121 | Version: '2012-10-17' 122 | Statement: 123 | - Effect: Allow 124 | Action: 125 | - logs:CreateLogGroup 126 | - logs:CreateLogStream 127 | - logs:PutLogEvents 128 | Resource: arn:aws:logs:*:*:* 129 | - Effect: Allow 130 | Action: 131 | - ec2:DescribeSubnets 132 | - ec2:DescribeVpcs 133 | Resource: "*" 134 | 135 | # NOTE: Pay special attention to the indentatiion in the Python code below. 136 | # Lines that appear blank are likely not blank, but have leading spaces. 137 | 138 | GetAttFromParam: 139 | Type: AWS::Lambda::Function 140 | Properties: 141 | Description: Look up info from a VPC or subnet ID 142 | Handler: index.handler 143 | MemorySize: 128 144 | Role: !GetAtt LambdaExecutionRole.Arn 145 | Runtime: "python3.6" 146 | Timeout: 30 147 | Code: 148 | ZipFile: | 149 | import json 150 | import boto3 151 | import cfnresponse 152 | import logging 153 | 154 | def handler(event, context): 155 | logger = logging.getLogger() 156 | logger.setLevel(logging.INFO) 157 | 158 | # initialize our responses, assume failure by default 159 | 160 | response_data = {} 161 | response_status = cfnresponse.FAILED 162 | 163 | logger.info('Received event: {}'.format(json.dumps(event))) 164 | 165 | if event['RequestType'] == 'Delete': 166 | response_status = cfnresponse.SUCCESS 167 | cfnresponse.send(event, context, response_status, response_data) 168 | 169 | try: 170 | ec2=boto3.client('ec2') 171 | except Exception as e: 172 | logger.info('boto3.client failure: {}'.format(e)) 173 | cfnresponse.send(event, context, response_status, response_data) 174 | 175 | name_filter = event['ResourceProperties']['NameFilter'] 176 | name_filter_parts = name_filter.split('-') 177 | resource_type=name_filter_parts[0] 178 | 179 | try: 180 | subnets = ec2.describe_subnets(SubnetIds=[name_filter]) 181 | except Exception as e: 182 | logger.info('ec2.describe_subnets failure: {}'.format(e)) 183 | cfnresponse.send(event, context, response_status, response_data) 184 | 185 | number_of_subnets = len(subnets['Subnets']) 186 | logger.info('number of subnets returned: {}'.format(number_of_subnets)) 187 | 188 | if number_of_subnets == 1: 189 | VpcId = subnets['Subnets'][0]['VpcId'] 190 | response_data['VpcId'] = VpcId 191 | 192 | logger.info('subnet VpcId {}'.format(VpcId)) 193 | 194 | response_status = cfnresponse.SUCCESS 195 | cfnresponse.send(event, context, response_status, response_data) 196 | 197 | elif number_of_subnets == 0: 198 | logger.info('no matching subnet for filter {}'.format(name_filter)) 199 | cfnresponse.send(event, context, response_status, response_data) 200 | 201 | else: 202 | logger.info('multiple matching subnets for filter {}'.format(name_filter)) 203 | cfnresponse.send(event, context, response_status, response_data) 204 | 205 | SubnetInfo: 206 | Type: Custom::SubnetInfo 207 | Properties: 208 | ServiceToken: !GetAtt GetAttFromParam.Arn 209 | NameFilter: !Ref SubnetID 210 | 211 | DMSFullAccessAAA: 212 | Type: AWS::IAM::ManagedPolicy 213 | Properties: 214 | Description: Policy for creating DMS resources 215 | Path: / 216 | PolicyDocument: 217 | Version: "2012-10-17" 218 | Statement: 219 | - Effect: Allow 220 | Action: "dms:*" 221 | Resource: "*" 222 | 223 | RootRole: 224 | Type: 'AWS::IAM::Role' 225 | Properties: 226 | AssumeRolePolicyDocument: 227 | Version: '2012-10-17' 228 | Statement: 229 | - Effect: Allow 230 | Principal: 231 | Service: 232 | - ec2.amazonaws.com 233 | Action: 234 | - 'sts:AssumeRole' 235 | Path: / 236 | ManagedPolicyArns: 237 | - "arn:aws:iam::aws:policy/IAMFullAccess" 238 | - "arn:aws:iam::aws:policy/AWSCloudFormationFullAccess" 239 | - "arn:aws:iam::aws:policy/AmazonSSMFullAccess" 240 | - "arn:aws:iam::aws:policy/AmazonRedshiftFullAccess" 241 | - "arn:aws:iam::aws:policy/AmazonS3FullAccess" 242 | - "arn:aws:iam::aws:policy/SecretsManagerReadWrite" 243 | - "arn:aws:iam::aws:policy/AmazonEC2FullAccess" 244 | - Ref: DMSFullAccessAAA 245 | 246 | InstanceProfileEC2Instance: 247 | Type: AWS::IAM::InstanceProfile 248 | Properties: 249 | Path: "/" 250 | Roles: 251 | - Ref: RootRole 252 | 253 | SecurityGroupEc2: 254 | Type: AWS::EC2::SecurityGroup 255 | Properties: 256 | GroupDescription: 'Launching EC2 security group' 257 | SecurityGroupIngress: 258 | - CidrIp: !Ref OnPremisesCIDR 259 | Description : Allow inbound access for on prem users on SSH port for the launching EC2 instance 260 | IpProtocol: tcp 261 | FromPort: 22 262 | IpProtocol: tcp 263 | ToPort: 22 264 | VpcId: !GetAtt SubnetInfo.VpcId 265 | 266 | SecurityGroupSelfReference: 267 | Type: AWS::EC2::SecurityGroupIngress 268 | Properties: 269 | Description: Self Referencing Rule 270 | FromPort: -1 271 | IpProtocol: -1 272 | GroupId: !GetAtt [SecurityGroupEc2, GroupId] 273 | SourceSecurityGroupId: !GetAtt [SecurityGroupEc2, GroupId] 274 | ToPort: -1 275 | 276 | StagingEC2Instance: 277 | Type: AWS::EC2::Instance 278 | CreationPolicy: 279 | ResourceSignal: 280 | Timeout: PT45M 281 | Properties: 282 | KeyName: !Ref KeyPair 283 | InstanceType: "t3.micro" 284 | IamInstanceProfile: !Ref InstanceProfileEC2Instance 285 | Tags: 286 | - Key: Name 287 | Value: 288 | Fn::Join: 289 | - "-" 290 | - - Ref: AWS::StackName 291 | - EC2Instance 292 | BlockDeviceMappings: 293 | - DeviceName: "/dev/sda1" 294 | Ebs: 295 | DeleteOnTermination: true 296 | VolumeType: gp2 297 | VolumeSize: 30 298 | ImageId: !Ref EC2InstanceAMI 299 | NetworkInterfaces: 300 | - DeleteOnTermination: true 301 | DeviceIndex: "0" 302 | SubnetId: !Ref SubnetID 303 | GroupSet: 304 | - Ref: SecurityGroupEc2 305 | UserData: 306 | Fn::Base64: !Sub | 307 | #!/bin/bash -e 308 | yum update -y 309 | yum -y install git 310 | yum -y install python3 311 | yum -y install python3-pip 312 | yum -y install aws-cfn-bootstrap 313 | yum -y install gcc gcc-c++ python3 python3-devel unixODBC unixODBC-devel 314 | mkdir /root/.aws 315 | echo "[default]" > /root/.aws/config 316 | echo "region = ${AWS::Region}" >> /root/.aws/config 317 | curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash 318 | . /.nvm/nvm.sh 319 | nvm install node 320 | npm install -g aws-cdk 321 | git clone -b main https://github.com/aws-samples/amazon-redshift-infrastructure-automation.git 322 | cd amazon-redshift-infrastructure-automation 323 | python3 -m venv .env 324 | source .env/bin/activate 325 | pip install -r requirements.txt 326 | pip install aws_cdk.aws_dms 327 | pip install aws_cdk.aws_redshift 328 | pip install boto3 329 | pip install aws_cdk.aws_cloudformation 330 | pip install aws_cdk.custom_resources 331 | pip install aws_cdk.aws_glue 332 | aws s3 cp ${ConfigurationFile} ./user-config.json 333 | # 334 | # Run CDK App 335 | # 336 | export STACK_NAME=${AWS::StackName} 337 | cdk deploy --all --require-approval never 338 | deactivate 339 | /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource StagingEC2Instance --region ${AWS::Region} 340 | 341 | Outputs: 342 | SourceAccountNumber: 343 | Description: "Extract Source Account Number" 344 | Value: !Ref AWS::AccountId 345 | VPC: 346 | Description: "VPC used" 347 | Value: !GetAtt SubnetInfo.VpcId 348 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /Config: -------------------------------------------------------------------------------- 1 | package.Aws-analytics-automation = { 2 | interfaces = (1.0); 3 | 4 | # Use NoOpBuild. See https://w.amazon.com/index.php/BrazilBuildSystem/NoOpBuild 5 | build-system = no-op; 6 | build-tools = { 7 | 1.0 = { 8 | NoOpBuild = 1.0; 9 | }; 10 | }; 11 | 12 | # Use runtime-dependencies for when you want to bring in additional 13 | # packages when deploying. 14 | # Use dependencies instead if you intend for these dependencies to 15 | # be exported to other packages that build against you. 16 | dependencies = { 17 | 1.0 = { 18 | }; 19 | }; 20 | 21 | runtime-dependencies = { 22 | 1.0 = { 23 | }; 24 | }; 25 | 26 | }; 27 | -------------------------------------------------------------------------------- /InputCIDR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/InputCIDR.png -------------------------------------------------------------------------------- /InputRegion.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/InputRegion.png -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AWS Analytics Automation Toolkit 2 | 3 | As analytics solutions have moved away from the one-size-fits-all model to choosing the right tool for the right function, architectures have become more optimized and performant while simultaneously becoming more complex. Solutions leveraging Amazon Redshift will often be used alongside services including AWS DMS, AWS AppSync, AWS Glue, AWS SCT, Amazon Sagemaker, Amazon QuickSight, and more. One of the core challenges of building these solutions can oftentimes be the integration of these services. 4 | 5 | This solution takes advantage of the repeated integrations between different services in common use cases, and leverages the [AWS CDK](https://aws.amazon.com/cdk/) to automate the provisioning of AWS analytics services, primarily Amazon Redshift. --> Deployment consists of running through a bash menu to indicate the resources to be used, and this solution takes those inputs to auto-provision the required infrastructure dynamically. <-- 6 | 7 | PLEASE NOTE: This solution is meant for proof of concept or demo use cases, and not for production workloads. 8 | 9 | ## Table of Contents 10 | 11 | 1. [Overview of Deployment](#overview-of-deployment) 12 | 1. [Prerequisites](#prerequisites) 13 | 1. [Deployment Steps](#deployment-steps) 14 | 1. [Post deployment](#post-deployment) 15 | 1. [Clean up](#clean-up) 16 | 1. [Troubleshooting](#troubleshooting) 17 | 1. [Feedback](#feedback) 18 | 19 | ## Overview of Deployment 20 | 21 | This project leverages [CloudShell](https://aws.amazon.com/cloudshell/), a browser-based shell service, to programmatically initiate the deployment through the AWS console. To achieve this, the user will clone this repository and run the deployment command. Afterwards a bash menu will appear helping the user in specifying the desired service configurations. 22 | 23 | The following sections give further details of how to complete these steps. 24 | 25 | ## Prerequisites 26 | 27 | Prior to deployment, some resources need to be preconfigured: 28 | * Please verify that you will be deploying this solution in a [region that supports CloudShell](https://docs.aws.amazon.com/general/latest/gr/cloudshell.html) 29 | * Execute the deployment with an IAM user with permissions to use: 30 | - AWS CloudShell 31 | - AWS Identity and Access Management (IAM) 32 | - AWS CloudFormation 33 | - Amazon SSM 34 | - Amazon Redshift 35 | - Amazon S3 36 | - AWS Secrets Manager 37 | - Amazon EC2 38 | - AWS Database Migration Service (DMS) 39 | - For a more granular list of permissions, please see [here](./permissionlist.md) 40 | * [OPTIONAL] If using SCT or JMeter: 41 | * create a key pair that can be accessed (see [the documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html#having-ec2-create-your-key-pair) on how to create a new one) 42 | * ensure your VPC contains subnets which can access outbound internet; via an internet gateway or NAT gateway 43 | * [OPTIONAL] If using an external database, open source firewalls/ security groups to allow for traffic from AWS 44 | * [OPTIONAL] If using Datasharing please ensure both producer and consumer cluster are in the same region/account. Publically enabled clusters are supported. For all other considerations please refer to the [Datasharing Considerations](https://docs.aws.amazon.com/redshift/latest/dg/considerations.html) documentation. 45 | 46 | If these are complete, continue to [deployment steps](#deployment-steps). If you come across errors, please refer to the [troubleshooting](#troubleshooting) section -- if the error isn't addressed there, please submit the feedback using the [Issues](https://github.com/aws-samples/amazon-redshift-infrastructure-automation/issues) tab of this repo. 47 | 48 | 49 | ## Deployment Steps 50 | 51 | 1. Open [CloudShell](https://console.aws.amazon.com/cloudshell/home?) 52 | 53 | 4. Clone the Git repository 54 | 55 | `git clone https://github.com/aws-samples/amazon-redshift-infrastructure-automation.git` 56 | 57 | 5. Run the deployment script 58 | 59 | `~/amazon-redshift-infrastructure-automation/scripts/deploy.sh` 60 | 61 | 6. Follow the prompts and specify desired resources using Y/y/N/n or corresponding numbers. Press *Enter* to confirm your selection. Towards the end users will also be able to change any inputs. 62 | 63 | ![Menu Start](./images/menustart.png) 64 | 65 | Example of using Y/y/N/n or numbers corresponding to inputs 66 | 67 | ![Example Inputs](./images/exampleofinputs.png) 68 | 69 | 7. At the end of the menu you may go back and reconfigure any desired inputs. Otherwise input 6 and launch resources! 70 | 71 | ![Reconfigure](./images/reconfigure.JPG) 72 | 73 | 74 | 9. Depending on your resource configuration, you may receive some additional input prompts: 75 | 76 | | Prompt | Input | Description | 77 | |-----------|-----------|---------------| 78 | |![Input Database Password](./images/InputDBPassword.png)|Password of external database|If are using an external database, will create a Secrets Manager secret with the password value| 79 | |![Input Redshift Password](./images/InputRedshiftPassword.png)|Password of existing Redshift cluster|If are giving a Redshift endpoint in the user_config.json file, will create a Secrets Manager secret with the password for the cluster database| 80 | 81 | 82 | ### Post deployment 83 | 84 | Once the script has been run, you can monitor the deployment of CloudFormation stacks through the CloudShell terminal, or with the CloudFormation console. 85 | 86 | 87 | ## Clean up 88 | 89 | 1. Open [CloudShell](https://aws.amazon.com/cloudshell/) 90 | 2. run the destroy script: `~/amazon-redshift-infrastructure-automation/scripts/destroy.sh` 91 | 3. enter the stackname when prompted, and click Yes when prompted, to destroy the stack. 92 | 93 | 94 | ## Troubleshooting 95 | 96 | * Error: `User: [IAM-USER-ARN] is not authorized to perform: [ACTION] on resource: [RESOURCE-ARN]` 97 | 98 | User running CloudShell doesn't have the appropriate permissions required - can use a separate IAM user with appropriate permissions: 99 | 100 | NOTE: User running the deployment (logged into the console) still needs **AWSCloudShellFullAccess** permissions 101 | 1. Open the [IAM console](https://console.aws.amazon.com/iamv2/home?#/home) 102 | 2. Under **Users**, select **Add users** 103 | 3. Create a new user 104 | 105 | ![New user](./images/NewUsers.png) 106 | 107 | 4. Select **Next: Permissions** 108 | 5. Add the following policies: 109 | - IAMFullAccess 110 | - AWSCloudFormationFullAccess 111 | - AmazonSSMFullAccess 112 | - AmazonRedshiftFullAccess 113 | - AmazonS3FullAccess 114 | - SecretsManagerReadWrite 115 | - AmazonEC2FullAccess 116 | - Create custom DMS policy called **AmazonDMSRoleCustom** -- select **Create policy** with the following permissions: 117 | ``` 118 | { 119 | "Version": "2012-10-17", 120 | "Statement": [ 121 | { 122 | "Effect": "Allow", 123 | "Action": "dms:*", 124 | "Resource": "*" 125 | } 126 | ] 127 | } 128 | ``` 129 | 130 | ![Policies](./images/Policies.png) 131 | 132 | 6. Get and download the CSV containing the Access Key and Secret Access Key for this user -- these will be used with Cloudshell: 133 | 134 | ![Access Keys](./images/AccessKeys.png) 135 | 136 | 7. When first open CloudShell, run 137 | 138 | 'aws configure' 139 | 140 | 8. Enter the Access Key and Secret Access Key downloaded for the IAM user created in the [Prerequisites](#prerequisites) 141 | 142 | ![Input Config](./images/InputConfig.png) 143 | * Error: `An error occurred (InvalidRequestException) when calling the CreateSecret operation: You can't create this secret because a secret with this name is already scheduled for deletion.` 144 | 145 | This occurs when you use a repeated stack name for the deployment, which results in a repeat of a secret name in Secrets Manager. Either use a new stack name when prompted for it, or delete the secrets by replacing `[STACK NAME]` with the stack name used for the deployment in the following commands and running them in CloudShell: 146 | 147 | `aws secretsmanager delete-secret --secret-id [STACK NAME]-SourceDBPassword --force-delete-without-recovery` 148 | 149 | `aws secretsmanager delete-secret --secret-id [STACK NAME]-RedshiftPassword --force-delete-without-recovery` 150 | 151 | `aws secretsmanager delete-secret --secret-id [STACK NAME]-RedshiftClusterSecretAA --force-delete-without-recovery` 152 | 153 | Then rerun: 154 | 155 | `~/amazon-redshift-infrastructure-automation/scripts/deploy.sh` 156 | 157 | ## Feedback 158 | 159 | Our aim is to make this tool as dynamic and comprehensive as possible, so we’d love to hear your feedback. Let us know your experience deploying the solution, and share any other use cases that the automation solution doesn’t yet support. Please use the [Issues](https://github.com/aws-samples/amazon-redshift-infrastructure-automation/issues) tab under this repo, and we’ll use that to guide our roadmap. 160 | 161 | ## Security 162 | 163 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 164 | 165 | ## License 166 | 167 | This library is licensed under the MIT-0 License. See the LICENSE file. 168 | -------------------------------------------------------------------------------- /Redshift-Loader/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/.DS_Store -------------------------------------------------------------------------------- /Redshift-Loader/CODE-OF-CONDUCT: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. -------------------------------------------------------------------------------- /Redshift-Loader/CONTRIBUTING: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. -------------------------------------------------------------------------------- /Redshift-Loader/CloudFormation/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/CloudFormation/.DS_Store -------------------------------------------------------------------------------- /Redshift-Loader/Images/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/Images/.gitkeep -------------------------------------------------------------------------------- /Redshift-Loader/Images/cfn_lambda_role.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/Images/cfn_lambda_role.png -------------------------------------------------------------------------------- /Redshift-Loader/Images/redshift_loader.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/Images/redshift_loader.png -------------------------------------------------------------------------------- /Redshift-Loader/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /Redshift-Loader/Sample_file_autoloader.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/Sample_file_autoloader.zip -------------------------------------------------------------------------------- /Redshift-Loader/Test_Data_Files/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/Test_Data_Files/.gitkeep -------------------------------------------------------------------------------- /Redshift-Loader/Test_Data_Files/Test_Table_ddls.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE "public"."customer_tbl1"(c_customer_sk integer NOT NULL encode az64, 2 | c_customer_id character(16) NOT NULL encode lzo, 3 | c_current_cdemo_sk integer encode az64, 4 | c_current_hdemo_sk integer encode az64, 5 | c_current_addr_sk integer encode az64, 6 | c_first_shipto_date_sk integer encode az64, 7 | c_first_sales_date_sk integer encode az64, 8 | c_salutation character(10) encode lzo, 9 | c_first_name character(20) encode lzo, 10 | c_last_name character(30) encode lzo, 11 | c_preferred_cust_flag character(1) encode lzo, 12 | c_birth_day integer encode az64, 13 | c_birth_month integer encode az64, 14 | c_birth_year integer encode az64, 15 | c_birth_country character varying(20) encode lzo, 16 | c_login character(13) encode lzo, 17 | c_email_address character(50) encode lzo, 18 | c_last_review_date_sk integer encode az64)distkey(c_customer_sk); 19 | 20 | CREATE TABLE "public"."customer_tbl2"(c_customer_sk integer NOT NULL encode az64, 21 | c_customer_id character(16) NOT NULL encode lzo, 22 | c_current_cdemo_sk integer encode az64, 23 | c_current_hdemo_sk integer encode az64, 24 | c_current_addr_sk integer encode az64, 25 | c_first_shipto_date_sk integer encode az64, 26 | c_first_sales_date_sk integer encode az64, 27 | c_salutation character(10) encode lzo, 28 | c_first_name character(20) encode lzo, 29 | c_last_name character(30) encode lzo, 30 | c_preferred_cust_flag character(1) encode lzo, 31 | c_birth_day integer encode az64, 32 | c_birth_month integer encode az64, 33 | c_birth_year integer encode az64, 34 | c_birth_country character varying(20) encode lzo, 35 | c_login character(13) encode lzo, 36 | c_email_address character(50) encode lzo, 37 | c_last_review_date_sk integer encode az64)distkey(c_customer_sk); 38 | CREATE TABLE "public"."customer_tbl3"(c_customer_sk integer NOT NULL encode az64, 39 | c_customer_id character(16) NOT NULL encode lzo, 40 | c_current_cdemo_sk integer encode az64, 41 | c_current_hdemo_sk integer encode az64, 42 | c_current_addr_sk integer encode az64, 43 | c_first_shipto_date_sk integer encode az64, 44 | c_first_sales_date_sk integer encode az64, 45 | c_salutation character(10) encode lzo, 46 | c_first_name character(20) encode lzo, 47 | c_last_name character(30) encode lzo, 48 | c_preferred_cust_flag character(1) encode lzo, 49 | c_birth_day integer encode az64, 50 | c_birth_month integer encode az64, 51 | c_birth_year integer encode az64, 52 | c_birth_country character varying(20) encode lzo, 53 | c_login character(13) encode lzo, 54 | c_email_address character(50) encode lzo, 55 | c_last_review_date_sk integer encode az64)distkey(c_customer_sk); 56 | CREATE TABLE "public"."customer_tbl4"(c_customer_sk integer NOT NULL encode az64, 57 | c_customer_id character(16) NOT NULL encode lzo, 58 | c_current_cdemo_sk integer encode az64, 59 | c_current_hdemo_sk integer encode az64, 60 | c_current_addr_sk integer encode az64, 61 | c_first_shipto_date_sk integer encode az64, 62 | c_first_sales_date_sk integer encode az64, 63 | c_salutation character(10) encode lzo, 64 | c_first_name character(20) encode lzo, 65 | c_last_name character(30) encode lzo, 66 | c_preferred_cust_flag character(1) encode lzo, 67 | c_birth_day integer encode az64, 68 | c_birth_month integer encode az64, 69 | c_birth_year integer encode az64, 70 | c_birth_country character varying(20) encode lzo, 71 | c_login character(13) encode lzo, 72 | c_email_address character(50) encode lzo, 73 | c_last_review_date_sk integer encode az64)distkey(c_customer_sk); 74 | CREATE TABLE "public"."customer_tbl5"(c_customer_sk integer NOT NULL encode az64, 75 | c_customer_id character(16) NOT NULL encode lzo, 76 | c_current_cdemo_sk integer encode az64, 77 | c_current_hdemo_sk integer encode az64, 78 | c_current_addr_sk integer encode az64, 79 | c_first_shipto_date_sk integer encode az64, 80 | c_first_sales_date_sk integer encode az64, 81 | c_salutation character(10) encode lzo, 82 | c_first_name character(20) encode lzo, 83 | c_last_name character(30) encode lzo, 84 | c_preferred_cust_flag character(1) encode lzo, 85 | c_birth_day integer encode az64, 86 | c_birth_month integer encode az64, 87 | c_birth_year integer encode az64, 88 | c_birth_country character varying(20) encode lzo, 89 | c_login character(13) encode lzo, 90 | c_email_address character(50) encode lzo, 91 | c_last_review_date_sk integer encode az64)distkey(c_customer_sk); 92 | CREATE TABLE "public"."customer_tbl6"(c_customer_sk integer NOT NULL encode az64, 93 | c_customer_id character(16) NOT NULL encode lzo, 94 | c_current_cdemo_sk integer encode az64, 95 | c_current_hdemo_sk integer encode az64, 96 | c_current_addr_sk integer encode az64, 97 | c_first_shipto_date_sk integer encode az64, 98 | c_first_sales_date_sk integer encode az64, 99 | c_salutation character(10) encode lzo, 100 | c_first_name character(20) encode lzo, 101 | c_last_name character(30) encode lzo, 102 | c_preferred_cust_flag character(1) encode lzo, 103 | c_birth_day integer encode az64, 104 | c_birth_month integer encode az64, 105 | c_birth_year integer encode az64, 106 | c_birth_country character varying(20) encode lzo, 107 | c_login character(13) encode lzo, 108 | c_email_address character(50) encode lzo, 109 | c_last_review_date_sk integer encode az64)distkey(c_customer_sk); 110 | 111 | CREATE TABLE "public"."rs_customer_fact"(c_customer_sk integer NOT NULL encode az64, 112 | c_customer_id character(16) NOT NULL encode lzo, 113 | c_current_cdemo_sk integer encode az64, 114 | c_current_hdemo_sk integer encode az64, 115 | c_current_addr_sk integer encode az64, 116 | c_first_shipto_date_sk integer encode az64, 117 | c_first_sales_date_sk integer encode az64, 118 | c_salutation character(10) encode lzo, 119 | c_first_name character(20) encode lzo, 120 | c_last_name character(30) encode lzo, 121 | c_preferred_cust_flag character(1) encode lzo, 122 | c_birth_day integer encode az64, 123 | c_birth_month integer encode az64, 124 | c_birth_year integer encode az64, 125 | c_birth_country character varying(20) encode lzo, 126 | c_login character(13) encode lzo, 127 | c_email_address character(50) encode lzo, 128 | c_last_review_date_sk integer encode az64)distkey(c_customer_sk); -------------------------------------------------------------------------------- /Redshift-Loader/Test_Data_Files/Testing_Script.sql: -------------------------------------------------------------------------------- 1 | truncate table public.s3_data_loader_util_config; 2 | truncate table s3_data_loader_util; 3 | truncate table s3_data_loader_log; 4 | 5 | truncate table customer_tbl; 6 | truncate table customer_tbl1; 7 | truncate table customer_tbl2; 8 | truncate table customer_tbl3; 9 | truncate table customer_tbl4; 10 | truncate table customer_tbl5; 11 | truncate table customer_tbl6; 12 | 13 | 14 | select 'customer_tbl' ,count(*) from customer_tbl union 15 | select 'customer_tbl1' ,count(*) from customer_tbl1 union 16 | select 'customer_tbl2' ,count(*) from customer_tbl2 union 17 | select 'customer_tbl3' ,count(*) from customer_tbl3 union 18 | select 'customer_tbl4' ,count(*) from customer_tbl4 union 19 | select 'customer_tbl5' ,count(*) from customer_tbl5 union 20 | select 'customer_tbl6' ,count(*) from customer_tbl6 21 | union 22 | select 'rs_customer_fact' ,count(*) from rs_customer_fact; 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /Redshift-Loader/Test_Data_Files/customer.txt_0000_part_00.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/Test_Data_Files/customer.txt_0000_part_00.gz -------------------------------------------------------------------------------- /Redshift-Loader/Test_Data_Files/customer.txt_0000_part_01.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/Test_Data_Files/customer.txt_0000_part_01.gz -------------------------------------------------------------------------------- /Redshift-Loader/Test_Data_Files/customer.txt_0000_part_02.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/Test_Data_Files/customer.txt_0000_part_02.gz -------------------------------------------------------------------------------- /Redshift-Loader/Test_Data_Files/customer.txt_0000_part_03.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/Test_Data_Files/customer.txt_0000_part_03.gz -------------------------------------------------------------------------------- /Redshift-Loader/loader_ui/README.md: -------------------------------------------------------------------------------- 1 | # Amazon Redshift S3 Auto Loader UI 2 | 3 | ## Description 4 | The Amazon Redshift S3 Auto Loader UI allows you to monitor your data loading processes without having to track down each individual load in the AWS console. Using this UI assumes that the Redshift Auto Loader was already deployed in your AWS account to load data from S3 to Redshift. 5 | 6 | Please read through the following to learn about the architecture behind this user interface and how to deploy it using your AWS account. 7 | 8 | ## Table of Contents 9 | 10 | [Architecture](#architecture) 11 | 12 | - [Process Flow](#process-flow) 13 | 14 | - [Components](#components) 15 | 16 | [Prerequisites](#prerequisites) 17 | 18 | [User Interface Content](#user-interface-content) 19 | 20 | [Cost](#cost) 21 | 22 | [Roadmap](#roadmap) 23 | 24 | ## Architecture 25 | Below is the architecture diagram of the user interface. 26 | 27 | ![architecture diagram](loader_ui_architecture_diagram.png) 28 | 29 | ### Process Flow 30 | Each of the following steps correlate with one image in the diagram based on the labeled step number to visualize the process flow of deploying the user interface. 31 | 32 | 1. The user downloads the **LoaderUI.yaml** file from the correct public repository in GitHub to upload to CloudFormation. 33 | 2. The user creates a new stack in CloudFormation and uploads the **LoaderUI.yaml** file to begin the automatic provisioning of AWS services required to deploy the user interface. 34 | 3. A Lambda function is created that copies static files to a target S3 bucket and generates a config.js file that is also placed in the same target S3 bucket. 35 | 4. The Lambda function is invoked and completes both the copying of static files and the generation of a config.js file which are placed in the target S3 bucket that was generated by the CloudFormation stack. 36 | 5. All the necessary files including the static files and config.js file now exist in the target S3 bucket which points to a CloudFront distribution that was created by the CloudFormation stack. 37 | 6. The user clicks on a CloudFront URL that was generated as an output parameter by the CloudFormation stack. 38 | 7. The user is redirected to an Amazon Cognito sign-in page. The user needs to manually create a user in the user pool that was generated by the CloudFormation stack so they can log in and access the user interface. 39 | 8. After the user has created their account and successfully logged in using the Amazon Cognito sign-in page, an IAM role and permissions are assigned to the user's account which grants them reading privileges to DynamoDB tables generated from using the auto loader. 40 | 9. The DynamoDB tables that were generated from using the auto loader are read to the user interface after the user was assigned their IAM role and permissions. 41 | 10. The user can now access the user interface webpage that displays the DynamoDB tables that were originally generated from using the auto loader. 42 | 43 | ### Components 44 | The following AWS resources are provisioned during the launch of the CloudFormation stack for deploying the user interface. The logical IDs of each resource are listed under each service that is used during the launch of the CloudFormation stack. The type of each resource is listed in parentheses next to each logical ID. 45 | #### [AWS Lambda](https://aws.amazon.com/lambda/) 46 | * CreateConfigFileAndCopyStaticFiles (AWS::Lambda::Function) 47 | 48 | #### [Amazon CloudFront](https://aws.amazon.com/cloudfront/) 49 | * CloudFrontDistribution (AWS::CloudFront::Distribution) 50 | * CloudFrontOriginAccessIdentity (AWS::CloudFront::CloudFrontOriginAccessIdentity) 51 | 52 | #### [Amazon Cognito](https://aws.amazon.com/cognito/) 53 | * CognitoAppClient (AWS::Cognito::UserPoolClient) 54 | * CognitoIdentityPool (AWS::Cognito::IdentityPool) 55 | * CognitoIdentityPoolRoleAttachment (AWS::Cognito::IdentityPoolRoleAttachment) 56 | * CognitoUserPool (AWS::Cognito::UserPool) 57 | * CognitoUserPoolDomain (AWS::Cognito::UserPoolDomain) 58 | 59 | #### [AWS Identity and Access Management (IAM)](https://aws.amazon.com/iam/) 60 | * DynamoDBAuthIAMRole (AWS::IAM::Role) 61 | * LambdaAuthIAMRole (AWS::IAM::Role) 62 | 63 | #### [Amazon S3](https://aws.amazon.com/s3/) 64 | * S3Bucket (AWS::S3::Bucket) 65 | * S3BucketReadPolicy (AWS::S3::BucketPolicy) 66 | 67 | #### [AWS CloudFormation](https://aws.amazon.com/cloudformation/) 68 | * LambdaInvoke (AWS::CloudFormation::CustomResource) 69 | 70 | ## Prerequisites 71 | The following items are prerequisites for deploying the user interface: 72 | * Deployed the Redshift Auto Loader tool 73 | * Downloaded the **LoaderUI.yaml** file from the correct public repository in GitHub 74 | 75 | If you have not deployed the Redshift Auto Loader tool yet, please go [here](https://github.com/aws-samples/amazon-redshift-infrastructure-automation/tree/main/Redshift-Loader) for the information and steps to deploy it before attempting to deploy the user interface. 76 | 77 | ## User Interface Content 78 | There are two panes that make up the main content of the UI: 79 | * Load details overview 80 | * Loader configuration 81 | 82 | Under **Load details overview**, there are three tables: 83 | 1. The first table counts the number of loads by status and does not have a table header. 84 | 2. The second table is the **Copy Command Details** table which pulls from the **s3_data_loader_log** table in DynamoDB. 85 | 3. The third table is the **S3 File Details** table which pulls from the **s3_data_loader_file_metadata** table in DynamoDB. 86 | 87 | The **Load details overview** pane has a dropdown button that allows you to filter all three tables based on one of three time frames: **Last 24 Hours**, **Last 7 Days**, and **All Time**. There is one refresh button that exists on top of the first table that allows the user to refresh the first two tables in the **Load details overview** pane. There is another refresh button that exists on top of the **S3 File Details** table that allows the user to individually refresh that table. 88 | 89 | Under **Loader configuration**, there are two tables: 90 | 1. The first table is the **Redshift Table Details** table which pulls from the **s3_data_loader_table_config** table in DynamoDB. 91 | 2. The second table is the **Loader Parameter Details** table which pulls from the **s3_data_loader_params** table in DynamoDB. 92 | 93 | The **Loader configuration** pane has one refresh button on top of each of the two tables for a total of two refresh buttons on the pane. Each refresh button individually refreshes their respective table, similar to how the refresh button on top of the **S3 File Details** table in the **Load details overview** pane individually refreshes the **S3 File Details** table. 94 | 95 | ## Cost 96 | AWS IAM is free to use where more information can be found [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html). The rest of the services offer a free tier but will start accumulating an expense for whatever resources you use that exceeds the free tier. 97 | 98 | Please refer to the following pricing information for each service: 99 | * [AWS Lambda](https://aws.amazon.com/lambda/pricing/) 100 | * [Amazon CloudFront](https://aws.amazon.com/cloudfront/pricing/) 101 | * [Amazon Cognito](https://aws.amazon.com/cognito/pricing/) 102 | * [Amazon S3](https://aws.amazon.com/s3/pricing/) 103 | * [AWS CloudFormation](https://aws.amazon.com/cloudformation/pricing/) 104 | 105 | ## Roadmap 106 | Here is a list of items and features that are currently on the roadmap that plan on being implemented in the future. 107 | 1. Automatically copy static files into the target S3 bucket resource from the loader_ui folder in the auto loader’s public GitHub repository instead of the public Redshift blogs S3 bucket. 108 | 2. Add an initial user from CloudFormation and output a username and password that allows the person who is deploying the UI to login immediately instead of having to first manually create a user in Amazon Cognito. 109 | 3. Modify one to multiple columns in the Loader Configuration tables using a Modify button. 110 | 4. For the first time setting up and deploying the UI, the initial user who is an auth admin can configure a password within the CloudFormation template as a parameter before deployment. 111 | * Auth admin user can choose to include a non-admin user as a second user that can be configured as a parameter in the CloudFormation template. 112 | 5. Create additional folders to load data either from the original S3 bucket or an additional S3 bucket if the solution was already deployed. 113 | 6. Add and remove users directly from the UI. 114 | 7. Configure email notifications of errors and issues seen in a set time frame to be sent to an email address provided by the user. The user will be able to choose what they are notified about from the following: successful loads, failed loads, or all loads. 115 | 8. Load data to additional Redshift clusters. 116 | 9. Configure parameters: rate at which loading occurs, number of copy commands run per load, Redshift cluster identifier name, database username, database name, database schema name, Redshift IAM Role ARN, copy command options, copy command schedule, and source S3 bucket. 117 | 10. Navigate the rows in the copy command details table using pagination. 118 | 11. Sort and filter rows in the tables by an attribute. 119 | * A key attribute to filter by in the copy command details table is copy command status which includes the following: FINISHED, Execution, and FAILED. 120 | * Key attributes to sort by in the copy command details table are finish timestamp and log timestamp which would allow the table to be sorted by the newest or oldest loads. 121 | 12. Limit how many rows are returned in the copy command details table. 122 | 13. Display the number of copy commands sitting in Simple Queue Service (SQS). 123 | 14. Add new users to one of two groups: admin or non-admin. 124 | -------------------------------------------------------------------------------- /Redshift-Loader/loader_ui/images/AWS_logo_RGB.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/loader_ui/images/AWS_logo_RGB.png -------------------------------------------------------------------------------- /Redshift-Loader/loader_ui/images/aws_logo_favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/loader_ui/images/aws_logo_favicon.ico -------------------------------------------------------------------------------- /Redshift-Loader/loader_ui/images/loading.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/loader_ui/images/loading.gif -------------------------------------------------------------------------------- /Redshift-Loader/loader_ui/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Amazon Redshift S3 Auto Loader UI 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 |

28 | 29 |

Amazon Redshift S3 Auto Loader

30 | 31 |


32 | 33 |
34 | 35 |
36 | 41 |

Load details overview

42 |
43 | 44 |
45 | 46 | 47 | 48 | 49 |
Executing LoadsFailed LoadsCompleted LoadsAttempted Loads
Please login to read the load metrics.
50 | 51 |

52 | 53 |
54 |
55 |

Copy Command Details

56 |
57 | 58 |
59 | 60 | 61 | 62 |
Copy Command SQLCopy Command StatusFinish Timestamp (UTC)Log Timestamp (UTC)Redshift Query IDRedshift SchemaRedshift Table NameS3 Manifest File PathS3 Table Name
Please login to read the loader copy command details.
63 |
64 | 65 |
66 |
67 | 68 |

S3 File Details

69 |
70 | 71 | 72 | 73 | 74 |
S3 Table NameFile Created Timestamp (UTC)File Content LengthS3 File Path
Please login to read the S3 file details.
75 | 76 |
77 | 78 |

79 | 80 |
81 | 82 |
83 |

Loader configuration

84 |
85 | 86 |

Redshift Table Details

87 |
88 | 89 | 90 | 91 | 92 |
S3 Table NameAdditional Copy OptionsIAM RoleLoad StatusMax File Proccessed Timestamp (UTC)Redshift SchemaRedshift Table NameSchema Detection CompletedSchema Detection Failed
Please login to read the loader Redshift table details.
93 | 94 |

95 | 96 |
97 |
98 | 99 |

Loader Parameter Details

100 |
101 | 102 | 103 | 104 | 105 |
Redshift Cluster IdentifierCopy Default OptionsInitiate Schema Detection Global VarNo of Copy CommandsRedshift DatabaseRedshift IAM RoleRedshift UserSchedule Frequency
Please login to read the loader parameter details.
106 | 107 |
108 | 109 | 110 | 111 | -------------------------------------------------------------------------------- /Redshift-Loader/loader_ui/loader_ui_architecture_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/Redshift-Loader/loader_ui/loader_ui_architecture_diagram.png -------------------------------------------------------------------------------- /Redshift-Loader/loader_ui/script.js: -------------------------------------------------------------------------------- 1 | var idToken = null; 2 | 3 | // Checks if the user is logged in with a valid token, otherwise they will be redirected to the sign-in page. 4 | // If the UI webpage times out, they will still be on the webpage but their tables and metrics 5 | // will not load where the user will have to press the logout button to sign in again. 6 | function checkLoginAndLoadInfo() { 7 | idToken = window.location.hash.substr(1).split('&')[0].split('id_token=')[1] 8 | if (idToken != null) { 9 | document.getElementById("loaderUserInterfaceContent").style.visibility = 'visible' 10 | window.location.href = cloudFrontURLString + idToken + "&expires_in=3600&token_type=Bearer" 11 | document.getElementById("welcomeMsg").innerHTML = "Signed in!"; 12 | auth(); 13 | timeDropdownFunctionCalls(); 14 | populateRedshiftTableDetails(); 15 | populateLoaderParameterDetails(); 16 | } else { 17 | document.getElementById("loaderUserInterfaceContent").style.visibility = 'hidden' 18 | window.location.href = cognitoSignInSignUpURL 19 | } 20 | } 21 | 22 | // Sets the credentials based on parameter variables pulled from the config.js file that is created from 23 | // launching the CloudFormation template. 24 | function auth() { 25 | var creds = {}; 26 | AWS.config.update({ 27 | region: awsAcctRegion, 28 | }); 29 | 30 | creds["IdentityPoolId"] = cognitoIdentityPoolId; 31 | creds["Logins"]={} 32 | creds["Logins"][loginsKey]=idToken; 33 | AWS.config.credentials = new AWS.CognitoIdentityCredentials(creds); 34 | } 35 | 36 | // Sets the screen to display the Cognito sign-in page. 37 | function loadLoginPage() { 38 | window.location.href = cognitoSignInSignUpURL 39 | } 40 | 41 | // Captures the current timestamp and calculates an adjusted timestamp based on the user's 42 | // selection from the time filter dropdown on the UI webpage. The function returns an array 43 | // containing both timestamps that were formatted using MomentJS. 44 | function subtractHoursFromCurrentTimeStamp() { 45 | let timeSelect = document.getElementById('timeDropdown'); 46 | let timeSelectValue = timeSelect.options[timeSelect.selectedIndex].value; 47 | let numHours = 0; 48 | const userCurrentTimeStamp = new Date(); 49 | let adjustedTimeStamp = "" 50 | let formattedCurrentTimeStamp = ""; 51 | let formattedAdjustedTimeStamp = ""; 52 | let hoursDiff = userCurrentTimeStamp.getTimezoneOffset() / 60; 53 | 54 | userCurrentTimeStamp.setHours(userCurrentTimeStamp.getHours() + hoursDiff) // Converts captured timestamp to represent UTC 55 | 56 | formattedCurrentTimeStamp = moment(userCurrentTimeStamp).format('YYYY-MM-DD HH:mm:ss') 57 | 58 | if (timeSelectValue == 24) { 59 | numHours = 24 60 | } else if (timeSelectValue == 168) { 61 | numHours = 168 62 | } else if (timeSelectValue == "allTime") { 63 | numHours = 0 64 | } 65 | 66 | adjustedTimeStamp = userCurrentTimeStamp.setHours(userCurrentTimeStamp.getHours() - numHours) 67 | 68 | formattedAdjustedTimeStamp = moment(adjustedTimeStamp).format('YYYY-MM-DD HH:mm:ss') 69 | 70 | return [formattedCurrentTimeStamp, formattedAdjustedTimeStamp] 71 | } 72 | 73 | // Calls the populateCopyCommandDetailsAndCountCopyCommands() and populateFileDetails() functions 74 | // to combine them into one function for populating all the tables in the "Load details overview" pane by 75 | // using the time filter dropdown. 76 | function timeDropdownFunctionCalls() { 77 | 78 | populateCopyCommandDetailsAndCountCopyCommands() 79 | populateFileDetails() 80 | } 81 | 82 | // Populates the Copy Command Details table and counts the number of loads by status by calling the countCopyCommands() 83 | // function. This function works with the timestamp values of the array returned by the subtractHoursFromCurrentTimeStamp() 84 | // function. If the two timestamp values are equal to each other, this means that the user chose to see the load details 85 | // over the course of all their time with using the auto loader. The scan filter is dropped if "All Time" was selected. 86 | function populateCopyCommandDetailsAndCountCopyCommands() { 87 | 88 | var docClient = new AWS.DynamoDB.DocumentClient(); 89 | let timeStamps = subtractHoursFromCurrentTimeStamp(); 90 | let currentTime = timeStamps[0]; 91 | let adjustedTime = timeStamps[1]; 92 | const currentTimeDateObject = new Date(currentTime); 93 | const adjustedTimeDateObject = new Date(adjustedTime); 94 | 95 | console.log("populateCopyCommandDetails' Current Date & Time: " + currentTime) 96 | console.log("populateCopyCommandDetails' Adjusted Date & Time: " + adjustedTime) 97 | 98 | console.log("populate Copy Command Details") 99 | $("#copyCommandDetails tbody")[0].innerHTML = ''; 100 | 101 | if (currentTimeDateObject.getTime() != adjustedTimeDateObject.getTime()) { 102 | var params = { 103 | TableName: 's3_data_loader_log', 104 | ScanFilter: { 105 | 'finish_timestamp': { 106 | ComparisonOperator: "BETWEEN", 107 | AttributeValueList: [ 108 | adjustedTime, 109 | currentTime 110 | ] 111 | } 112 | } 113 | }; 114 | } else { 115 | var params = { 116 | TableName: 's3_data_loader_log' 117 | }; 118 | } 119 | 120 | docClient.scan(params, function(err,result){ 121 | if (err) { 122 | console.log('Error: ' + JSON.stringify(err)); 123 | } else { 124 | console.log(result); 125 | if (result.Items.length > 0) { 126 | $("#copyCommandDetails tbody")[0].innerHTML = result.Items.map(row => " \ 127 | " + row.copy_command_sql + "\ 128 | " + row.copy_command_status + "\ 129 | " + row.finish_timestamp + "\ 130 | " + row.log_timestamp + "\ 131 | " + row.redshift_query_Id + "\ 132 | " + row.redshift_schema + "\ 133 | " + row.redshift_table_name + "\ 134 | " + row.s3_manifest_file_path + "\ 135 | " + row.s3_table_name).join(""); 136 | } else { 137 | $("#copyCommandDetails tbody")[0].innerHTML = "There are no copy command details to display." 138 | } 139 | 140 | countCopyCommands() 141 | } 142 | }); 143 | } 144 | 145 | // Populates the S3 File Details table and works with the timestamp values of the array returned by 146 | // the subtractHoursFromCurrentTimeStamp() function, similar to the populateCopyCommandDetailsAndCountCopyCommands() function. 147 | function populateFileDetails() { 148 | 149 | var docClient = new AWS.DynamoDB.DocumentClient(); 150 | let timeStamps = subtractHoursFromCurrentTimeStamp(); 151 | let currentTime = timeStamps[0]; 152 | let adjustedTime = timeStamps[1]; 153 | const currentTimeDateObject = new Date(currentTime); 154 | const adjustedTimeDateObject = new Date(adjustedTime); 155 | 156 | console.log("populateFileDetails' Current Date & Time: " + currentTime) 157 | console.log("populateFileDetails' Adjusted Date & Time: " + adjustedTime) 158 | 159 | console.log("populate S3 File Details") 160 | $("#fileDetails tbody")[0].innerHTML = ''; 161 | 162 | if (currentTimeDateObject.getTime() != adjustedTimeDateObject.getTime()) { 163 | var params = { 164 | TableName: 's3_data_loader_file_metadata', 165 | ScanFilter: { 166 | 'file_created_timestamp': { 167 | ComparisonOperator: "BETWEEN", 168 | AttributeValueList: [ 169 | adjustedTime, 170 | currentTime 171 | ] 172 | } 173 | } 174 | }; 175 | } else { 176 | var params = { 177 | TableName: 's3_data_loader_file_metadata' 178 | }; 179 | } 180 | 181 | docClient.scan(params, function(err,result){ 182 | if (err) { 183 | console.log('Error: ' + JSON.stringify(err)); 184 | } else { 185 | console.log(result); 186 | if (result.Items.length > 0) { 187 | $("#fileDetails tbody")[0].innerHTML = result.Items.map(row => " \ 188 | " + row.s3_table_name + "\ 189 | " + row.file_created_timestamp + "\ 190 | " + row.file_content_length + "\ 191 | " + row.s3_file_path).join(""); 192 | } else { 193 | $("#fileDetails tbody")[0].innerHTML = "There are no S3 file details to display." 194 | } 195 | } 196 | }); 197 | } 198 | 199 | // Populates the Redshift Table Details table in the "Loader configuration" pane. 200 | function populateRedshiftTableDetails() { 201 | 202 | var docClient = new AWS.DynamoDB.DocumentClient(); 203 | 204 | console.log("populate Redshift Table Details") 205 | $("#redshiftTableDetails tbody")[0].innerHTML = ''; 206 | var params = { 207 | TableName: 's3_data_loader_table_config' 208 | }; 209 | docClient.scan(params, function(err,result){ 210 | if (err) { 211 | console.log('Error: ' + JSON.stringify(err)); 212 | } else { 213 | console.log(result); 214 | if (result.Items.length > 0) { 215 | $("#redshiftTableDetails tbody")[0].innerHTML = result.Items.map(row => " \ 216 | " + row.s3_table_name + "\ 217 | " + row.additional_copy_options + "\ 218 | " + row.iam_role + "\ 219 | " + row.load_status + "\ 220 | " + row.max_file_proccessed_timestamp + "\ 221 | " + row.redshift_schema + "\ 222 | " + row.redshift_table_name + "\ 223 | " + row.schema_detection_completed + "\ 224 | " + row.schema_detection_failed).join(""); 225 | } else { 226 | $("#redshiftTableDetails tbody")[0].innerHTML = "There are no Redshift table details to display." 227 | } 228 | } 229 | }); 230 | } 231 | 232 | // Populates the Loader Parameter Details table in the "Loader configuration" pane. 233 | function populateLoaderParameterDetails() { 234 | 235 | var docClient = new AWS.DynamoDB.DocumentClient(); 236 | 237 | console.log("populate Loader Parameter Details") 238 | $("#loaderParameterDetails tbody")[0].innerHTML = ''; 239 | var params = { 240 | TableName: 's3_data_loader_params' 241 | }; 242 | docClient.scan(params, function(err,result){ 243 | if (err) { 244 | console.log('Error: ' + JSON.stringify(err)); 245 | } else { 246 | console.log(result); 247 | if (result.Items.length > 0) { 248 | $("#loaderParameterDetails tbody")[0].innerHTML = result.Items.map(row => " \ 249 | " + row.redshift_cluster_identifier + "\ 250 | " + row.copy_default_options + "\ 251 | " + row.initiate_schema_detection_global_var + "\ 252 | " + row.no_of_copy_commands + "\ 253 | " + row.redshift_database + "\ 254 | " + row.redshift_iam_role + "\ 255 | " + row.redshift_user + "\ 256 | " + row.schedule_frequency).join(""); 257 | } else { 258 | $("#loaderParameterDetails tbody")[0].innerHTML = "There are no loader parameter details to display." 259 | } 260 | } 261 | }); 262 | } 263 | 264 | // Counts the number of loads by status directly from the rows returned in the Copy Command Details table 265 | // on the UI webpage itself. 266 | function countCopyCommands() { 267 | 268 | let numRowsFinished = $("#copyCommandDetails tr:contains('FINISHED')").length; 269 | let numRowsFailed = $("#copyCommandDetails tr:contains('FAILED')").length; 270 | let numRowsExecuting = $("#copyCommandDetails tr:contains('Execution')").length; 271 | let numRowsAttempted = numRowsExecuting + numRowsFailed + numRowsFinished 272 | let countsRow = document.createElement('tr'); 273 | let countsRowExecutingValue = document.createElement('td'); 274 | let countsRowFailedValue = document.createElement('td'); 275 | let countsRowCompletedValue = document.createElement('td'); 276 | let countsRowAttemptedValue = document.createElement('td'); 277 | 278 | console.log("Counting loads") 279 | 280 | console.log(numRowsExecuting + " Loads Executing") 281 | console.log(numRowsFailed + " Loads Failed") 282 | console.log(numRowsFinished + " Loads Finished") 283 | console.log(numRowsAttempted + " Loads Attempted") 284 | 285 | countsRowCompletedValue.innerHTML = numRowsFinished; 286 | countsRowFailedValue.innerHTML = numRowsFailed; 287 | countsRowAttemptedValue.innerHTML = numRowsAttempted; 288 | countsRowExecutingValue.innerHTML = numRowsExecuting; 289 | 290 | console.log("populate metrics table") 291 | 292 | countsRow.appendChild(countsRowExecutingValue) 293 | countsRow.appendChild(countsRowFailedValue) 294 | countsRow.appendChild(countsRowCompletedValue) 295 | countsRow.appendChild(countsRowAttemptedValue) 296 | 297 | document.getElementById("metricsTblTBody").innerHTML = "" 298 | document.getElementById("metricsTblTBody").appendChild(countsRow) 299 | } -------------------------------------------------------------------------------- /Redshift-Loader/loader_ui/styles.css: -------------------------------------------------------------------------------- 1 | body, head { 2 | font-family: Arial, Helvetica, sans-serif; 3 | } 4 | 5 | body { 6 | background-color: #eaeded; 7 | padding: 15px; 8 | } 9 | 10 | .sectionBlock { 11 | background-color: white; 12 | padding: 20px; 13 | width: 78%; 14 | margin: auto; 15 | overflow: auto; 16 | } 17 | 18 | .sectionBlockHeadingContent { 19 | width: 78%; 20 | margin: auto; 21 | } 22 | 23 | h1 { 24 | font-size: 2.5em; 25 | text-align: center; 26 | } 27 | 28 | h2 { 29 | border-bottom: 2px solid black; 30 | padding-bottom: 10px; 31 | } 32 | 33 | h1, h2 { 34 | margin: auto; 35 | } 36 | 37 | a:link { 38 | text-decoration: none; 39 | } 40 | 41 | .refreshBtn { 42 | background-color: #ff9900; 43 | border-radius: 8px; 44 | padding: 5px; 45 | margin: 10px 0px; 46 | width: 40px; 47 | font-size: 20px; 48 | float: right; 49 | } 50 | 51 | .refreshBtn:hover { 52 | background-color: #99cbe4; 53 | } 54 | 55 | .refreshBtn:active { 56 | background-color: #0073bb; 57 | } 58 | 59 | select { 60 | background-color: #ff9900; 61 | padding: 4px; 62 | font-weight: bold; 63 | float: right; 64 | border: 2px solid; 65 | outline: none; 66 | } 67 | 68 | select:hover { 69 | background-color: #99cbe4; 70 | } 71 | 72 | #loginBtn { 73 | background-color: #ff9900; 74 | border-radius: 8px; 75 | padding: 6px; 76 | float: right; 77 | font-weight: bold; 78 | } 79 | 80 | #loginBtn:hover { 81 | background-color: #99cbe4; 82 | } 83 | 84 | #loginBtn:active { 85 | background-color: #0073bb; 86 | } 87 | 88 | table { 89 | clear: right; 90 | border-collapse: collapse; 91 | width: 78%; 92 | margin: auto; 93 | } 94 | 95 | th { 96 | padding: 28px; 97 | } 98 | 99 | .detailsTbl td { 100 | padding: 18px; 101 | text-align: center; 102 | } 103 | 104 | tbody tr:nth-child(odd) { 105 | background-color: #d5dbdb; 106 | } 107 | 108 | tbody tr:nth-child(even) { 109 | background-color: #f2f3f3; 110 | } 111 | 112 | tr th { 113 | background-color: #f2f3f3; 114 | } 115 | 116 | #metricsTblTBody td { 117 | font-size: 1.75em; 118 | text-align: center; 119 | padding: 6px; 120 | } 121 | 122 | #metricsTbl th { 123 | font-size: 1.2em; 124 | } 125 | 126 | #welcomeMsg { 127 | clear: right; 128 | float: right; 129 | font-weight: bold; 130 | } 131 | 132 | #AWSLogo { 133 | width: 105.94px; 134 | height: 63.33; 135 | } 136 | 137 | #copyCommandDetailsWindow { 138 | max-height: 1620px; 139 | overflow-y: scroll; 140 | } -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | 4 | import json 5 | import boto3 6 | import os 7 | from aws_cdk import core 8 | from aws_cdk.core import Tags 9 | from redshift_poc_automation.stacks.vpc_stack import VpcStack 10 | from redshift_poc_automation.stacks.redshift_stack import RedshiftStack 11 | from redshift_poc_automation.stacks.redshiftserverless_stack import RedshiftServerlessStack 12 | from redshift_poc_automation.stacks.redshiftrole_stack import RSDefaultRole 13 | from redshift_poc_automation.stacks.redshiftload_stack import RedshiftLoadStack 14 | from redshift_poc_automation.stacks.dms_on_prem_to_redshift_stack import DmsOnPremToRedshiftStack 15 | from redshift_poc_automation.stacks.sct_stack import SctOnPremToRedshiftStack 16 | from redshift_poc_automation.stacks.jmeter_stack import JmeterStack 17 | from redshift_poc_automation.stacks.data_sharing_stack import DataSharingProducerStack 18 | from redshift_poc_automation.stacks.data_sharing_consumer_stack import DataSharingConsumerStack 19 | 20 | 21 | app = core.App() 22 | 23 | my_region = boto3.session.Session().region_name 24 | account_id = boto3.client('sts').get_caller_identity().get('Account') 25 | env = {'account': account_id, 'region': my_region} 26 | config = json.load(open("user-config.json")) 27 | 28 | # Get the values of the user-config file 29 | vpc_id = config.get('vpc_id') 30 | vpc_config = config.get('vpc') 31 | 32 | redshift_endpoint = config.get('redshift_endpoint') 33 | redshift_config = config.get('redshift') 34 | loadtpc = redshift_config.get('loadTPCdata') 35 | datasharing_config=config.get('datasharing') 36 | 37 | dms_on_prem_to_redshift_target = config.get('dms_migration_to_redshift_target') 38 | sct_on_prem_to_redshift_target = config.get('sct_on_prem_to_redshift_target') 39 | jmeter = config.get('jmeter') 40 | dms_on_prem_to_redshift_config = config.get('dms_migration') 41 | external_database_config = config.get('external_database') 42 | other_config = config.get('other') 43 | datashare = config.get('datashare') 44 | stackname = os.getenv('STACK_NAME') 45 | onprem_cidr = os.getenv('ONPREM_CIDR') 46 | 47 | glue_crawler_s3_target = "N/A" 48 | glue_crawler_s3_config = "N/A" 49 | 50 | # VPC Stack for hosting secure API & other resources 51 | vpc_stack = VpcStack( 52 | app, 53 | f"{stackname}-vpc-stack", 54 | env=env, 55 | vpc_id=vpc_id, 56 | vpc_config=vpc_config, 57 | onprem_cidr=onprem_cidr, 58 | stack_log_level="INFO", 59 | description="AWS Analytics Automation: Custom Multi-AZ VPC" 60 | ) 61 | Tags.of(vpc_stack).add("project", stackname) 62 | 63 | redshift_serverless_endpoint = config.get('redshift_serverless_endpoint') 64 | redshift_serverless_config = config.get('redshift_serverless') 65 | 66 | if redshift_serverless_endpoint != "N/A": 67 | redshift_serverless_stack = RedshiftServerlessStack( 68 | app, 69 | f"{stackname}-redshift-serverless-stack", 70 | env=env, 71 | vpc=vpc_stack, 72 | redshift_serverless_endpoint=redshift_serverless_endpoint, 73 | redshift_serverless_config=redshift_serverless_config, 74 | stack_log_level="INFO", 75 | description="AWS Analytics Automation: Deploy Redshift Serverless Endpoint" 76 | ) 77 | redshift_serverless_stack.add_dependency(vpc_stack) 78 | Tags.of(redshift_serverless_stack).add("project", stackname) 79 | 80 | # Deploy Redshift cluster and load data" 81 | if redshift_endpoint != "N/A": 82 | 83 | redshift_stack = RedshiftStack( 84 | app, 85 | f"{stackname}-redshift-stack", 86 | env=env, 87 | vpc=vpc_stack, 88 | redshift_endpoint=redshift_endpoint, 89 | redshift_config=redshift_config, 90 | stack_log_level="INFO", 91 | description="AWS Analytics Automation: Deploy Redshift cluster" 92 | ) 93 | redshift_stack.add_dependency(vpc_stack); 94 | Tags.of(redshift_stack).add("project", stackname) 95 | 96 | if redshift_endpoint == "CREATE": 97 | if loadtpc == "Y" or loadtpc == "y": 98 | redshiftrole_stack = RSDefaultRole( 99 | app, 100 | f"{stackname}-redshiftrole-stack", 101 | env=env, 102 | cluster=redshift_stack.redshift, 103 | defaultrole=redshift_stack.cluster_iam_role.role_arn, 104 | stack_log_level="INFO", 105 | description="AWS Analytics Automation: Modify Redshift Role" 106 | ) 107 | redshiftrole_stack.add_dependency(redshift_stack); 108 | redshiftload_stack = RedshiftLoadStack( 109 | app, 110 | f"{stackname}-redshiftload-stack", 111 | env=env, 112 | cluster=redshift_stack.redshift, 113 | defaultrole=redshift_stack.cluster_iam_role.role_arn, 114 | redshift_config=redshift_config, 115 | stack_log_level="INFO", 116 | description="AWS Analytics Automation: Load TPC Data" 117 | ) 118 | redshiftload_stack.add_dependency(redshift_stack); 119 | redshiftload_stack.add_dependency(redshiftrole_stack); 120 | 121 | Tags.of(redshiftrole_stack).add("project", stackname) 122 | Tags.of(redshiftload_stack).add("project", stackname) 123 | 124 | 125 | # DMS OnPrem to Redshift Stack for migrating database to redshift 126 | if dms_on_prem_to_redshift_target == "CREATE": 127 | dms_on_prem_to_redshift_stack = DmsOnPremToRedshiftStack( 128 | app, 129 | f"{stackname}-dms-stack", 130 | env=env, 131 | vpc = vpc_stack, 132 | dmsmigration_config = dms_on_prem_to_redshift_config, 133 | source_config = external_database_config, 134 | cluster=redshift_stack, 135 | stack_log_level="INFO", 136 | description="AWS Analytics Automation: DMS endpoints and tasks" 137 | ) 138 | dms_on_prem_to_redshift_stack.add_dependency(redshift_stack); 139 | Tags.of(dms_on_prem_to_redshift_stack).add("project", stackname) 140 | 141 | # SCT OnPrem to Redshift Stack for migrating database to redshift 142 | if sct_on_prem_to_redshift_target == "CREATE": 143 | sct_on_prem_to_redshift_stack = SctOnPremToRedshiftStack( 144 | app, 145 | f"{stackname}-sct-stack", 146 | env=env, 147 | other_config=other_config, 148 | vpc=vpc_stack, 149 | stack_log_level="INFO", 150 | onprem_cidr=onprem_cidr, 151 | description="AWS Analytics Automation: SCT install on new EC2 Instance" 152 | ) 153 | sct_on_prem_to_redshift_stack.add_dependency(redshift_stack); 154 | Tags.of(sct_on_prem_to_redshift_stack).add("project", stackname) 155 | 156 | if jmeter == "CREATE": 157 | jmeter_stack = JmeterStack( 158 | app, 159 | f"{stackname}-jmeter-stack", 160 | env=env, 161 | # cluster=redshift_stack, 162 | other_config=other_config, 163 | # redshift_config=redshift_config, 164 | vpc=vpc_stack, 165 | stack_log_level="INFO", 166 | onprem_cidr=onprem_cidr, 167 | description="AWS Analytics Automation: Jmeter install on new EC2 Instance" 168 | ) 169 | # jmeter_stack.add_dependency(redshift_stack); 170 | Tags.of(jmeter_stack).add("project", stackname) 171 | 172 | # Glue Crawler Stack to crawl s3 locations 173 | if glue_crawler_s3_target != "N/A": 174 | glue_crawler_stack = GlueCrawlerStack( 175 | app, 176 | f"{stackname}-glue-crawler-stack", 177 | env=env, 178 | glue_crawler_s3_config=glue_crawler_s3_config, 179 | stack_log_level="INFO", 180 | description="AWS Analytics Automation: Deploy Glue Crawler for S3 data lake" 181 | ) 182 | glue_crawler_stack.add_dependency(vpc_stack); 183 | # Data Sharing 184 | if datashare == "CREATE": 185 | ds_producer_stack = DataSharingProducerStack( 186 | app, 187 | f"{stackname}-datasharingproducerstack", 188 | #env=env, 189 | #cluster=redshift_stack.redshift, 190 | defaultrole='arn:aws:iam::572911389735:role/aws-service-role/redshift.amazonaws.com/AWSServiceRoleForRedshift', 191 | datasharing_config=datasharing_config, 192 | stack_log_level="INFO", 193 | description="AWS Analytics Automation: Data Sharing Producer Stack" 194 | ) 195 | 196 | ds_consumer_stack = DataSharingConsumerStack( 197 | app, 198 | f"{stackname}-datasharingconsumerstack", 199 | #env=env, 200 | #cluster=redshift_stack.redshift, 201 | defaultrole='arn:aws:iam::572911389735:role/aws-service-role/redshift.amazonaws.com/AWSServiceRoleForRedshift', 202 | datasharing_config=datasharing_config, 203 | stack_log_level="INFO", 204 | description="AWS Analytics Automation: Data Sharing Consumer Stack" 205 | ) 206 | ds_consumer_stack.add_dependency(ds_producer_stack); 207 | 208 | # Stack Level Tagging 209 | _tags_lst = app.node.try_get_context("tags") 210 | 211 | if _tags_lst: 212 | for _t in _tags_lst: 213 | for k, v in _t.items(): 214 | core.Tags.of(app).add(k, v, apply_to_launched_instances=True) 215 | 216 | 217 | app.synth() 218 | -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "python3 app.py", 3 | "context": { 4 | "project": "redshift-aaa", 5 | "@aws-cdk/core:enableStackNameDuplicates": "true", 6 | "aws-cdk:enableDiffNoFail": "true", 7 | "@aws-cdk/core:stackRelativeExports": "true", 8 | "@aws-cdk/aws-ecr-assets:dockerIgnoreSupport": true, 9 | "@aws-cdk/aws-secretsmanager:parseOwnedSecretName": true, 10 | "@aws-cdk/aws-kms:defaultKeyPolicies": true, 11 | "@aws-cdk/aws-s3:grantWriteWithoutAcl": true, 12 | "@aws-cdk/aws-ecs-patterns:removeDefaultDesiredCount": true 13 | } 14 | } 15 | 16 | -------------------------------------------------------------------------------- /images/AccessKeys.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/AccessKeys.png -------------------------------------------------------------------------------- /images/InputCIDR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/InputCIDR.png -------------------------------------------------------------------------------- /images/InputConfig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/InputConfig.png -------------------------------------------------------------------------------- /images/InputDBPassword.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/InputDBPassword.png -------------------------------------------------------------------------------- /images/InputRedshiftPassword.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/InputRedshiftPassword.png -------------------------------------------------------------------------------- /images/InputRegion.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/InputRegion.png -------------------------------------------------------------------------------- /images/InputStack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/InputStack.png -------------------------------------------------------------------------------- /images/NewUsers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/NewUsers.png -------------------------------------------------------------------------------- /images/Policies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/Policies.png -------------------------------------------------------------------------------- /images/UploadConfig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/UploadConfig.png -------------------------------------------------------------------------------- /images/UploadConfirmation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/UploadConfirmation.png -------------------------------------------------------------------------------- /images/UploadLocation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/UploadLocation.png -------------------------------------------------------------------------------- /images/exampleofinputs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/exampleofinputs.png -------------------------------------------------------------------------------- /images/menustart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/menustart.png -------------------------------------------------------------------------------- /images/reconfigure.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/images/reconfigure.JPG -------------------------------------------------------------------------------- /jmeterconfig.sh: -------------------------------------------------------------------------------- 1 | 20 | -------------------------------------------------------------------------------- /lambda_call_lambda_to_run_redshift_script.py: -------------------------------------------------------------------------------- 1 | import json 2 | import boto3 3 | import cfnresponse 4 | 5 | def handler(event, context): 6 | print(event) 7 | if event['RequestType'] == 'Delete': 8 | cfnresponse.send(event, context, cfnresponse.SUCCESS, {'Data': 'No Action Needed'}) 9 | # Check if this is a Create and we're failing Creates 10 | elif event['RequestType'] == 'Create' and event['ResourceProperties'].get('FailCreate', False): 11 | raise RuntimeError('Create failure requested') 12 | else: 13 | action = event['ResourceProperties']['Action'] 14 | redshift_host = event['ResourceProperties']['RedshiftHost'] 15 | redshift_db = event['ResourceProperties']['RedshiftDb'] 16 | redshift_user = event['ResourceProperties']['RedshiftUser'] 17 | script_s3_path = event['ResourceProperties']['ScriptS3Path'] 18 | redshift_iam_role = event['ResourceProperties']['RedshiftIamRole'] 19 | lambda_arn = event['ResourceProperties']['LambdaArn'] 20 | lambda_payload = { 21 | "input": { 22 | "Action": action, 23 | "RedshiftHost": redshift_host, 24 | "RedshiftDb": redshift_db, 25 | "RedshiftUser": redshift_user, 26 | "RedshiftIamRole": redshift_iam_role, 27 | "ScriptS3Path": script_s3_path, 28 | "redshift_iam_role": redshift_iam_role 29 | } 30 | } 31 | response = boto3.client('lambda').invoke( 32 | FunctionName=lambda_arn, 33 | InvocationType='Event', 34 | Payload=json.dumps(lambda_payload) 35 | ) 36 | print(response) 37 | cfnresponse.send(event, context, cfnresponse.SUCCESS, {'Data': 'Create complete'}) 38 | 39 | if __name__ == "__main__": 40 | event ={ 41 | 'RequestType': 'Create', 42 | 'ServiceToken': 'arn:aws:lambda:us-east-1:855402123041:function:redshift-demo-redshift-bo-SingletonLambda130c8acc4-NLBM0M7107K', 43 | 'ResponseURL': 'https://cloudformation-custom-resource-response-useast1.s3.amazonaws.com/arn%3Aaws%3Acloudformation%3Aus-east-1%3A855402123041%3Astack/redshift-demo-redshift-bootstrap-stack2/89df0570-a250-11eb-84de-0a43cdcd5209%7CcustomResourceLambdaCallRunRedshiftScript%7C51ca3d8a-e891-4e35-a840-bc0ef5ccfc9d?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20210421T032106Z&X-Amz-SignedHeaders=host&X-Amz-Expires=7200&X-Amz-Credential=AKIA6L7Q4OWTYMT6AIEY%2F20210421%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Signature=650a79b649f7971b784a4685fc7af602b75c3dcbc401119b433a6cf28b99bd5b', 44 | 'StackId': 'arn:aws:cloudformation:us-east-1:855402123041:stack/redshift-demo-redshift-bootstrap-stack2/89df0570-a250-11eb-84de-0a43cdcd5209', 45 | 'RequestId': '51ca3d8a-e891-4e35-a840-bc0ef5ccfc9d', 46 | 'LogicalResourceId': 'customResourceLambdaCallRunRedshiftScript', 47 | 'ResourceType': 'AWS::CloudFormation::CustomResource', 48 | 'ResourceProperties': { 49 | 'ServiceToken': 'arn:aws:lambda:us-east-1:855402123041:function:redshift-demo-redshift-bo-SingletonLambda130c8acc4-NLBM0M7107K', 50 | 'Action': 'RUN_REDSHIFT_SCRIPT', 51 | 'RedshiftHost': 'redshift-demo-redshift-stack-redshiftcluster-wo73jo4zwwtg.cxzy7wkirtem.us-east-1.redshift.amazonaws.com', 52 | 'LambdaArn': 'arn:aws:lambda:us-east-1:855402123041:function:redshift-demo-redshift-bo-lambdaRunRedshiftScript0-4EKDBQRJD21F', 53 | 'RedshiftIamRole': 'arn:aws:iam::855402123041:role/redshift-demo-redshift-st-redshiftClusterRole4D302-1I44UET6SAEVE', 54 | 'RedshiftDb': 'dev', 55 | 'ScriptS3Path': 's3://event-driven-app-with-lambda-redshift/scripts/test_script.sql', 56 | 'RedshiftUser': 'awsuser' 57 | } 58 | } 59 | 60 | context = "" 61 | handler(event, context) 62 | 63 | -------------------------------------------------------------------------------- /lambda_run_redshift_script.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import traceback 3 | 4 | def handler(event, context): 5 | print(event) 6 | action = event['input'].get('Action') 7 | redshift_host = event['input'].get('RedshiftHost') 8 | redshift_db = event['input'].get('RedshiftDb') 9 | redshift_user = event['input'].get('RedshiftUser') 10 | redshift_iam_role = event['input'].get('RedshiftIamRole') 11 | script_s3_path = event['input'].get('ScriptS3Path') 12 | sql_query_id = event['input'].get('sql_query_id') 13 | 14 | try: 15 | if action == "RUN_REDSHIFT_SCRIPT": 16 | res = {'sql_id': run_sql(redshift_host, redshift_db, redshift_user, redshift_iam_role, script_s3_path)} 17 | elif action == "SQL_STATUS": 18 | res = {'status': sql_status(sql_query_id)} 19 | else: 20 | raise ValueError("Invalid Task: " + action) 21 | except Exception as e: 22 | print(e) 23 | print(traceback.format_exc()) 24 | raise 25 | print(res) 26 | return res 27 | 28 | 29 | def get_config_from_s3(script_s3_path): 30 | path_parts = script_s3_path.replace("s3://", "").split("/") 31 | bucket = path_parts.pop(0) 32 | key = "/".join(path_parts) 33 | obj = boto3.client('s3').get_object(Bucket=bucket, Key=key) 34 | return obj['Body'].read().decode('utf-8') 35 | 36 | 37 | def run_sql(redshift_host, redshift_db, redshift_user, redshift_iam_role, script_s3_path, with_event=True, 38 | run_type='ASYNC'): 39 | cluster_identifier = redshift_host.split('.')[0] 40 | script = get_config_from_s3(script_s3_path).format(redshift_iam_role) 41 | 42 | res = boto3.client("redshift-data").execute_statement(Database=redshift_db, DbUser=redshift_user, Sql=script, 43 | ClusterIdentifier=cluster_identifier, WithEvent=with_event) 44 | query_id = res["Id"] 45 | statuses = ["STARTED", "FAILED", "FINISHED"] if run_type == 'ASYNC' else ["FAILED", "FINISHED"] 46 | done = False 47 | while not done: 48 | status = sql_status(query_id) 49 | if status in statuses: 50 | print(query_id + ":" + status) 51 | break 52 | return query_id 53 | 54 | 55 | def sql_status(query_id): 56 | res = boto3.client("redshift-data").describe_statement(Id=query_id) 57 | status = res["Status"] 58 | if status == "FAILED": 59 | raise Exception('Error:' + res["Error"]) 60 | return status.strip('"') 61 | 62 | 63 | if __name__ == "__main__": 64 | event ={ 65 | 'input': { 66 | 'Action': 'RUN_REDSHIFT_SCRIPT', 67 | 'RedshiftHost': 'redshift-demo-redshift-stack-redshiftcluster-wo73jo4zwwtg.cxzy7wkirtem.us-east-1.redshift.amazonaws.com', 68 | 'RedshiftDb': 'dev', 69 | 'RedshiftUser': 'awsuser', 70 | 'RedshiftIamRole': 'arn:aws:iam::855402123041:role/redshift-demo-redshift-st-redshiftClusterRole4D302-1I44UET6SAEVE', 71 | 'ScriptS3Path': 's3://event-driven-app-with-lambda-redshift/scripts/test_script.sql', 72 | 'redshift_iam_role': 'arn:aws:iam::855402123041:role/redshift-demo-redshift-st-redshiftClusterRole4D302-1I44UET6SAEVE' 73 | } 74 | } 75 | 76 | context = "" 77 | handler(event, context) 78 | 79 | -------------------------------------------------------------------------------- /redshift_poc_automation/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/redshift_poc_automation/.DS_Store -------------------------------------------------------------------------------- /redshift_poc_automation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/redshift_poc_automation/__init__.py -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/redshift_poc_automation/stacks/.DS_Store -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/data_sharing_consumer_stack.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from typing import Any 3 | import json 4 | # from constructs import Construct 5 | from aws_cdk import ( 6 | core, 7 | aws_iam as iam, 8 | aws_redshift as aws_redshift 9 | ) 10 | 11 | from aws_cdk.custom_resources import ( 12 | AwsCustomResource, 13 | AwsCustomResourcePolicy, 14 | AwsSdkCall, 15 | PhysicalResourceId, 16 | ) 17 | 18 | 19 | class DataSharingConsumerStack(core.Stack): 20 | 21 | def __init__( 22 | self, 23 | scope: core.Construct, 24 | id: str, 25 | # cluster: aws_redshift.CfnCluster, 26 | defaultrole: str, 27 | datasharing_config: dict, 28 | stack_log_level: str, 29 | log_retention=None, 30 | **kwargs 31 | ) -> None: 32 | super().__init__(scope, id, **kwargs) 33 | stackname = id.split('-')[0] 34 | 35 | # database_name = redshift_config.get('database_name') 36 | # master_user_name = redshift_config.get('master_user_name') 37 | 38 | DatashareName = datasharing_config.get('datashare_name') 39 | ProducerCluster = datasharing_config.get('producer_cluster_identifier') 40 | ProducerClusterDb = datasharing_config.get('producer_database_name') 41 | ProducerClusterMasterUser = datasharing_config.get('producer_username') 42 | ProducerSchemaName = datasharing_config.get('producer_schema_name') 43 | ConsumerCluster = datasharing_config.get('consumer_cluster_identifier') 44 | ConsumerClusterDb = datasharing_config.get('consumer_database_name') 45 | ConsumerClusterMasterUser = datasharing_config.get('consumer_username') 46 | 47 | client = boto3.client('redshift-data') 48 | boto_client = boto3.client('redshift') 49 | # f = open('./scripts/loadTPcH3TB.txt') 50 | 51 | # a = f.read() 52 | consumer_namespace = (boto_client.describe_clusters(ClusterIdentifier=ConsumerCluster)['Clusters'][0][ 53 | 'ClusterNamespaceArn']).split(":")[6] 54 | producer_namespace = (boto_client.describe_clusters(ClusterIdentifier=ProducerCluster)['Clusters'][0][ 55 | 'ClusterNamespaceArn']).split(":")[6] 56 | default_role = defaultrole 57 | # cluster_identifier = cluster.ref 58 | 59 | policy = AwsCustomResourcePolicy.from_sdk_calls( 60 | resources=AwsCustomResourcePolicy.ANY_RESOURCE) 61 | lambda_role = self.get_provisioning_lambda_role(construct_id=id) 62 | # lambda_role.add_to_policy(actions=["redshift:GetClusterCredentials"], resources=['*']) 63 | lambda_role.add_to_policy(iam.PolicyStatement(actions=["redshift:GetClusterCredentials"], resources=['*'])) 64 | 65 | consumer_statement = "CREATE DATABASE myconsumer_db FROM DATASHARE " + DatashareName + " OF NAMESPACE '" + producer_namespace + "'; CREATE EXTERNAL SCHEMA myconsumer_schema FROM REDSHIFT DATABASE myconsumer_db SCHEMA " + ProducerSchemaName + "; CREATE SCHEMA myview_schema;CREATE VIEW myview_schema.tickit_sales AS SELECT * FROM myconsumer_schema.tickit_sales WITH NO SCHEMA BINDING;" 66 | 67 | create_params = { 68 | "Database": ConsumerClusterDb, 69 | "Sql": consumer_statement, 70 | "ClusterIdentifier": ConsumerCluster, 71 | "DbUser": ConsumerClusterMasterUser 72 | } 73 | 74 | aws_custresource_con = AwsCustomResource(self, 75 | id=f'{id}-AWSCustomResource', 76 | policy=policy, 77 | log_retention=log_retention, 78 | on_update=AwsSdkCall( 79 | action='executeStatement', 80 | service='RedshiftData', 81 | parameters=create_params, 82 | physical_resource_id=PhysicalResourceId.of( 83 | ConsumerCluster), 84 | ), 85 | role=lambda_role) 86 | 87 | def get_provisioning_lambda_role(self, construct_id: str): 88 | return iam.Role( 89 | scope=self, 90 | id=f'{construct_id}-LambdaRole', 91 | assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'), 92 | managed_policies=[iam.ManagedPolicy.from_aws_managed_policy_name( 93 | "service-role/AWSLambdaBasicExecutionRole")], 94 | ) -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/data_sharing_stack.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from typing import Any 3 | import json 4 | #from constructs import Construct 5 | from aws_cdk import ( 6 | core, 7 | aws_iam as iam, 8 | aws_redshift as aws_redshift 9 | ) 10 | 11 | from aws_cdk.custom_resources import ( 12 | AwsCustomResource, 13 | AwsCustomResourcePolicy, 14 | AwsSdkCall, 15 | PhysicalResourceId, 16 | ) 17 | 18 | 19 | class DataSharingProducerStack(core.Stack): 20 | 21 | def __init__( 22 | self, 23 | scope: core.Construct, 24 | id: str, 25 | #cluster: aws_redshift.CfnCluster, 26 | defaultrole: str, 27 | datasharing_config: dict, 28 | stack_log_level: str, 29 | log_retention=None, 30 | **kwargs 31 | ) -> None: 32 | super().__init__(scope, id, **kwargs) 33 | stackname = id.split('-')[0] 34 | 35 | #database_name = redshift_config.get('database_name') 36 | #master_user_name = redshift_config.get('master_user_name') 37 | 38 | ProducerCluster = datasharing_config.get('producer_cluster_identifier') 39 | DatashareName = datasharing_config.get('datashare_name') 40 | ProducerClusterDb = datasharing_config.get('producer_database_name') 41 | ProducerClusterMasterUser = datasharing_config.get('producer_username') 42 | ProducerSchemaName = datasharing_config.get('producer_schema_name') 43 | ConsumerCluster = datasharing_config.get('consumer_cluster_identifier') 44 | ConsumerClusterDb = datasharing_config.get('consumer_database_name') 45 | ConsumerClusterMasterUser = datasharing_config.get('consumer_username') 46 | 47 | client = boto3.client('redshift-data') 48 | boto_client = boto3.client('redshift') 49 | #f = open('./scripts/loadTPcH3TB.txt') 50 | 51 | #a = f.read() 52 | consumer_namespace = (boto_client.describe_clusters(ClusterIdentifier=ConsumerCluster)['Clusters'][0]['ClusterNamespaceArn']).split(":")[6] 53 | #producer_namespace = (boto_client.describe_clusters(ClusterIdentifier=ProducerCluster)['Clusters'][0]['ClusterNamespaceArn']).split( ":")[6] 54 | default_role = defaultrole 55 | #cluster_identifier = cluster.ref 56 | 57 | policy = AwsCustomResourcePolicy.from_sdk_calls( 58 | resources=AwsCustomResourcePolicy.ANY_RESOURCE) 59 | lambda_role = self.get_provisioning_lambda_role(construct_id=id) 60 | # lambda_role.add_to_policy(actions=["redshift:GetClusterCredentials"], resources=['*']) 61 | lambda_role.add_to_policy(iam.PolicyStatement(actions=["redshift:GetClusterCredentials"], resources=['*'])) 62 | 63 | producer_statement = "CREATE DATASHARE " + DatashareName + "; ALTER DATASHARE " + DatashareName + " ADD SCHEMA " + ProducerSchemaName + "; ALTER DATASHARE " + DatashareName + " ADD ALL TABLES IN SCHEMA " + ProducerSchemaName + "; ALTER DATASHARE " + DatashareName + " SET INCLUDENEW = TRUE FOR SCHEMA " + ProducerSchemaName + "; ALTER DATASHARE " + DatashareName + " SET PUBLICACCESSIBLE TRUE; GRANT USAGE ON DATASHARE " + DatashareName + " TO NAMESPACE '" + consumer_namespace + "';" 64 | 65 | create_params = { 66 | "Database": ProducerClusterDb, 67 | "Sql": producer_statement, 68 | "ClusterIdentifier": ProducerCluster, 69 | "DbUser": ProducerClusterMasterUser 70 | } 71 | aws_custresource = AwsCustomResource(self, 72 | id=f'{id}-AWSCustomResource', 73 | policy=policy, 74 | log_retention=log_retention, 75 | on_update=AwsSdkCall( 76 | action='executeStatement', 77 | service='RedshiftData', 78 | parameters=create_params, 79 | physical_resource_id=PhysicalResourceId.of( 80 | ProducerCluster), 81 | ), 82 | role=lambda_role) 83 | 84 | def get_provisioning_lambda_role(self, construct_id: str): 85 | return iam.Role( 86 | scope=self, 87 | id=f'{construct_id}-LambdaRole', 88 | assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'), 89 | managed_policies=[iam.ManagedPolicy.from_aws_managed_policy_name( 90 | "service-role/AWSLambdaBasicExecutionRole")], 91 | ) -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/dms_on_prem_to_redshift_stack.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import aws_dms 2 | from aws_cdk import core 3 | from aws_cdk import aws_iam 4 | import boto3 5 | import getpass 6 | import json 7 | 8 | 9 | class GlobalArgs(): 10 | """ 11 | Helper to define global statics 12 | """ 13 | 14 | OWNER = "Redshift POC SSA team" 15 | ENVIRONMENT = "development" 16 | REPO_NAME = "redshift-demo" 17 | SOURCE_INFO = f"https://github.com/kaklis/RedshiftPOCAutomation" 18 | VERSION = "2021_03_15" 19 | SUPPORT_EMAIL = ["aws-redshift-poc-sa-amer@amazon.com"] 20 | 21 | class DmsOnPremToRedshiftStack(core.Stack): 22 | 23 | def __init__( 24 | self, 25 | scope: core.Construct, id: str, 26 | vpc, 27 | dmsmigration_config: dict, 28 | source_config: dict, 29 | cluster, 30 | stack_log_level: str, 31 | **kwargs 32 | 33 | ) -> None: 34 | super().__init__(scope, id, **kwargs) 35 | 36 | stackname = id.split('-')[0] 37 | 38 | #--------CREATE DMS INSTANCE-------- 39 | subnet_type = dmsmigration_config.get('subnet_type') 40 | dms_instance_type = dmsmigration_config.get('dms_instance_type') 41 | 42 | # DMS IAM Role 43 | self.dms_vpc_role() 44 | self.dms_cloudwatch_logs_role() 45 | self.dms_access_for_endpoint() 46 | 47 | publiclyaccessible = False 48 | if subnet_type == 'PUBLIC': 49 | subnets = vpc.get_vpc_public_subnet_ids 50 | publiclyaccessible = True 51 | elif subnet_type == 'PRIVATE': 52 | subnets = vpc.get_vpc_private_subnet_ids 53 | elif subnet_type == 'ISOLATED': 54 | subnets = vpc.get_vpc_private_isolated_subnet_ids 55 | 56 | dms_subnet_group = aws_dms.CfnReplicationSubnetGroup( 57 | self, 58 | "DMSsubnetgroup", 59 | replication_subnet_group_description="Subnet group for DMS replication instance", 60 | subnet_ids=subnets 61 | ) 62 | 63 | security_group_id = vpc.get_vpc_security_group_id 64 | 65 | self.dms_instance = aws_dms.CfnReplicationInstance( 66 | self, 67 | "DMSInstance", 68 | replication_instance_class=dms_instance_type, 69 | allocated_storage=50, 70 | allow_major_version_upgrade=None, 71 | auto_minor_version_upgrade=None, 72 | multi_az=False, 73 | publicly_accessible=publiclyaccessible, 74 | replication_subnet_group_identifier=dms_subnet_group.ref, 75 | vpc_security_group_ids=[security_group_id] 76 | ) 77 | 78 | #--------CREATE DMS MIGRATION TASK-------- 79 | source_db = source_config.get('source_db') 80 | source_engine = source_config.get('source_engine') 81 | source_schema = source_config.get('source_schema') 82 | source_host = source_config.get('source_host') 83 | source_user = source_config.get('source_user') 84 | source_port = int(source_config.get('source_port')) 85 | migration_type = dmsmigration_config.get('migration_type') 86 | 87 | secret_name = stackname+"-SourceDBPassword" 88 | region_name = boto3.session.Session().region_name 89 | 90 | session = boto3.session.Session() 91 | client = session.client( 92 | service_name='secretsmanager', 93 | region_name=region_name, 94 | ) 95 | 96 | try: 97 | source_pwd = client.get_secret_value( 98 | SecretId=secret_name 99 | )['SecretString'] 100 | except Exception: 101 | source_pwd = getpass.getpass(prompt='Source DB Password: ') 102 | client.create_secret( 103 | Name=secret_name, 104 | SecretString=source_pwd, 105 | Description='Source database password for DMS' 106 | ) 107 | pass 108 | 109 | 110 | 111 | if cluster.get_cluster_type: 112 | target_pwd = client.get_secret_value( 113 | SecretId=cluster.get_cluster_secret 114 | )['SecretString'] 115 | else: 116 | target_pwd = cluster.get_cluster_password 117 | 118 | tablemappings="""{ 119 | "rules": [ 120 | { 121 | "rule-type": "selection", 122 | "rule-id": "1", 123 | "rule-name": "1", 124 | "object-locator": { 125 | "schema-name": "%"""+source_schema + """", 126 | "table-name": "%" 127 | }, 128 | "rule-action": "include", 129 | "filters": [] 130 | } 131 | ] 132 | }""" 133 | 134 | self.dms_endpoint_tgt = aws_dms.CfnEndpoint( 135 | self, 136 | "DMSendpointtgt", 137 | endpoint_type="target", 138 | engine_name="redshift", 139 | database_name=f"{cluster.get_cluster_dbname}", 140 | password=f"{target_pwd}", 141 | username=f"{cluster.get_cluster_user}", 142 | server_name=f"{cluster.get_cluster_host}", 143 | port=5439 144 | ) 145 | 146 | self.dms_endpoint_src = aws_dms.CfnEndpoint( 147 | self, 148 | "DMSendpointsrc", 149 | endpoint_type="source", 150 | engine_name=source_engine, 151 | database_name=source_db, 152 | password=source_pwd, 153 | port=source_port, 154 | username=source_user, 155 | server_name=source_host, 156 | ) 157 | 158 | dms_task = aws_dms.CfnReplicationTask( 159 | self, 160 | "DMSreplicationtask", 161 | migration_type=migration_type, 162 | replication_instance_arn=self.dms_instance.ref, 163 | source_endpoint_arn=self.get_srcendpoint_id, 164 | target_endpoint_arn=self.get_tgtendpoint_id, 165 | table_mappings=tablemappings 166 | ) 167 | 168 | 169 | def dms_vpc_role(self): 170 | client = boto3.client('iam') 171 | try: 172 | response = client.get_role(RoleName='dms-vpc-role') 173 | except: 174 | try: 175 | role_policy_document = { 176 | "Version": "2012-10-17", 177 | "Statement": [ 178 | { 179 | "Effect": "Allow", 180 | "Principal": { 181 | "Service": [ 182 | "dms.amazonaws.com" 183 | ] 184 | }, 185 | "Action": "sts:AssumeRole" 186 | } 187 | ] 188 | } 189 | client.create_role( 190 | RoleName='dms-vpc-role', 191 | AssumeRolePolicyDocument=json.dumps(role_policy_document) 192 | ) 193 | client.attach_role_policy( 194 | RoleName='dms-vpc-role', 195 | PolicyArn='arn:aws:iam::aws:policy/service-role/AmazonDMSVPCManagementRole' 196 | ) 197 | except Exception as e: 198 | print(e) 199 | 200 | def dms_cloudwatch_logs_role(self): 201 | client = boto3.client('iam') 202 | try: 203 | response = client.get_role(RoleName='dms-cloudwatch-logs-role') 204 | except: 205 | try: 206 | role_policy_document = { 207 | "Version": "2012-10-17", 208 | "Statement": [ 209 | { 210 | "Effect": "Allow", 211 | "Principal": { 212 | "Service": "dms.amazonaws.com" 213 | }, 214 | "Action": "sts:AssumeRole" 215 | } 216 | ] 217 | } 218 | client.create_role( 219 | RoleName='dms-cloudwatch-logs-role', 220 | AssumeRolePolicyDocument=json.dumps(role_policy_document) 221 | ) 222 | client.attach_role_policy( 223 | RoleName='dms-cloudwatch-logs-role', 224 | PolicyArn='arn:aws:iam::aws:policy/service-role/AmazonDMSCloudWatchLogsRole' 225 | ) 226 | except Exception as e: 227 | print(e) 228 | 229 | def dms_access_for_endpoint(self): 230 | client = boto3.client('iam') 231 | try: 232 | response = client.get_role(RoleName='dms-access-for-endpoint') 233 | except: 234 | try: 235 | role_policy_document = { 236 | "Version": "2012-10-17", 237 | "Statement": [ 238 | { 239 | "Sid": "1", 240 | "Effect": "Allow", 241 | "Principal": { 242 | "Service": "dms.amazonaws.com" 243 | }, 244 | "Action": "sts:AssumeRole" 245 | }, 246 | { 247 | "Sid": "2", 248 | "Effect": "Allow", 249 | "Principal": { 250 | "Service": "redshift.amazonaws.com" 251 | }, 252 | "Action": "sts:AssumeRole" 253 | } 254 | ] 255 | } 256 | client.create_role( 257 | RoleName='dms-access-for-endpoint', 258 | AssumeRolePolicyDocument=json.dumps(role_policy_document) 259 | ) 260 | client.attach_role_policy( 261 | RoleName='dms-access-for-endpoint', 262 | PolicyArn='arn:aws:iam::aws:policy/service-role/AmazonDMSRedshiftS3Role' 263 | ) 264 | except Exception as e: 265 | print(e) 266 | 267 | 268 | 269 | @property 270 | def get_repinstance_id(self): 271 | return self.dms.ref 272 | 273 | @property 274 | def get_tgtendpoint_id(self): 275 | return self.dms_endpoint_tgt.ref 276 | 277 | @property 278 | def get_srcendpoint_id(self): 279 | return self.dms_endpoint_src.ref 280 | -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/dmsinstance_stack.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import aws_dms 2 | from aws_cdk import aws_iam 3 | from aws_cdk import core 4 | import boto3 5 | import json 6 | 7 | class GlobalArgs(): 8 | """ 9 | Helper to define global statics 10 | """ 11 | 12 | OWNER = "Redshift POC SSA team" 13 | ENVIRONMENT = "development" 14 | REPO_NAME = "redshift-demo" 15 | SOURCE_INFO = f"https://github.com/kaklis/RedshiftPOCAutomation" 16 | VERSION = "2021_03_15" 17 | SUPPORT_EMAIL = ["aws-redshift-poc-sa-amer@amazon.com"] 18 | 19 | class DmsInstanceStack(core.Stack): 20 | 21 | def __init__( 22 | self, 23 | scope: core.Construct, id: str, 24 | vpc, 25 | stack_log_level: str, 26 | dms_config: dict, 27 | **kwargs 28 | 29 | ) -> None: 30 | super().__init__(scope, id, **kwargs) 31 | 32 | # DMS IAM Role 33 | self.dms_vpc_role() 34 | self.dms_cloudwatch_logs_role() 35 | self.dms_access_for_endpoint() 36 | 37 | # 38 | # try: 39 | # dms_vpc_role = aws_iam.ManagedPolicy.from_aws_managed_policy_name("dms-vpc-role") 40 | # except: 41 | # dms_vpc_role = aws_iam.Role( 42 | # self, "dmsvpcrole", 43 | # assumed_by=aws_iam.ServicePrincipal( 44 | # "dms.amazonaws.com"), 45 | # managed_policies=[ 46 | # aws_iam.ManagedPolicy.from_aws_managed_policy_name( 47 | # "AmazonDMSVPCManagementRole" 48 | # ) 49 | # ], 50 | # role_name = "dms-vpc-role" 51 | # ) 52 | subnet_type = dms_config.get('subnet_type') 53 | dms_instance_type = dms_config.get('dms_instance_type') 54 | if subnet_type == 'PUBLIC': 55 | subnets = vpc.get_vpc_public_subnet_ids 56 | else: 57 | subnets = vpc.get_vpc_private_subnet_ids 58 | 59 | 60 | dms_subnet_group = aws_dms.CfnReplicationSubnetGroup( 61 | self, 62 | "DMSsubnetgroup", 63 | replication_subnet_group_description="Subnet group for DMS replication instance", 64 | subnet_ids=subnets 65 | ) 66 | 67 | security_group_id = vpc.get_vpc_security_group_id 68 | 69 | self.dms_instance = aws_dms.CfnReplicationInstance( 70 | self, 71 | "DMSInstance", 72 | replication_instance_class=dms_instance_type, 73 | allocated_storage=50, 74 | allow_major_version_upgrade=None, 75 | auto_minor_version_upgrade=None, 76 | multi_az=False, 77 | publicly_accessible=True, 78 | replication_subnet_group_identifier=dms_subnet_group.ref, 79 | vpc_security_group_ids=[security_group_id] 80 | ) 81 | 82 | def dms_vpc_role(self): 83 | client = boto3.client('iam') 84 | try: 85 | response = client.get_role(RoleName='dms-vpc-role') 86 | except: 87 | try: 88 | role_policy_document = { 89 | "Version": "2012-10-17", 90 | "Statement": [ 91 | { 92 | "Effect": "Allow", 93 | "Principal": { 94 | "Service": [ 95 | "dms.amazonaws.com" 96 | ] 97 | }, 98 | "Action": "sts:AssumeRole" 99 | } 100 | ] 101 | } 102 | client.create_role( 103 | RoleName='dms-vpc-role', 104 | AssumeRolePolicyDocument=json.dumps(role_policy_document) 105 | ) 106 | client.attach_role_policy( 107 | RoleName='dms-vpc-role', 108 | PolicyArn='arn:aws:iam::aws:policy/service-role/AmazonDMSVPCManagementRole' 109 | ) 110 | except Exception as e: 111 | print(e) 112 | 113 | def dms_cloudwatch_logs_role(self): 114 | client = boto3.client('iam') 115 | try: 116 | response = client.get_role(RoleName='dms-cloudwatch-logs-role') 117 | except: 118 | try: 119 | role_policy_document = { 120 | "Version": "2012-10-17", 121 | "Statement": [ 122 | { 123 | "Effect": "Allow", 124 | "Principal": { 125 | "Service": "dms.amazonaws.com" 126 | }, 127 | "Action": "sts:AssumeRole" 128 | } 129 | ] 130 | } 131 | client.create_role( 132 | RoleName='dms-cloudwatch-logs-role', 133 | AssumeRolePolicyDocument=json.dumps(role_policy_document) 134 | ) 135 | client.attach_role_policy( 136 | RoleName='dms-cloudwatch-logs-role', 137 | PolicyArn='arn:aws:iam::aws:policy/service-role/AmazonDMSCloudWatchLogsRole' 138 | ) 139 | except Exception as e: 140 | print(e) 141 | 142 | def dms_access_for_endpoint(self): 143 | client = boto3.client('iam') 144 | try: 145 | response = client.get_role(RoleName='dms-access-for-endpoint') 146 | except: 147 | try: 148 | role_policy_document = { 149 | "Version": "2012-10-17", 150 | "Statement": [ 151 | { 152 | "Sid": "1", 153 | "Effect": "Allow", 154 | "Principal": { 155 | "Service": "dms.amazonaws.com" 156 | }, 157 | "Action": "sts:AssumeRole" 158 | }, 159 | { 160 | "Sid": "2", 161 | "Effect": "Allow", 162 | "Principal": { 163 | "Service": "redshift.amazonaws.com" 164 | }, 165 | "Action": "sts:AssumeRole" 166 | } 167 | ] 168 | } 169 | client.create_role( 170 | RoleName='dms-access-for-endpoint', 171 | AssumeRolePolicyDocument=json.dumps(role_policy_document) 172 | ) 173 | client.attach_role_policy( 174 | RoleName='dms-access-for-endpoint', 175 | PolicyArn='arn:aws:iam::aws:policy/service-role/AmazonDMSRedshiftS3Role' 176 | ) 177 | except Exception as e: 178 | print(e) 179 | 180 | @property 181 | def get_repinstance_id(self): 182 | return self.dms_instance.ref 183 | -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/jmeter_stack.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import aws_iam 2 | from aws_cdk import aws_ec2 3 | from aws_cdk import core 4 | from aws_cdk import aws_secretsmanager 5 | import boto3 6 | import json 7 | 8 | 9 | class JmeterStack(core.Stack): 10 | 11 | def __init__( 12 | self, 13 | scope: core.Construct, id: str, 14 | # cluster, 15 | other_config: dict, 16 | # redshift_config: dict, 17 | vpc, 18 | stack_log_level: str, 19 | onprem_cidr: str, 20 | **kwargs 21 | 22 | ) -> None: 23 | super().__init__(scope, id, **kwargs) 24 | 25 | keyname = other_config.get('key_name') 26 | jmeter_node_type = other_config.get('jmeter_node_type') 27 | # redshift_host = cluster.get_cluster_host 28 | # redshift_db = cluster.get_cluster_dbname 29 | # redshift_user = cluster.get_cluster_user 30 | secret_arn = 'RedshiftClusterSecretAA' 31 | amiID = 'ami-042e0580ee1b9e2af' 32 | 33 | account_id = boto3.client('sts').get_caller_identity().get('Account') 34 | 35 | with open("./jmeterconfig.sh") as f: 36 | user_data = f.read() 37 | 38 | with open("./jmeterconfig_2.sh") as f_2: 39 | user_data_2 = f_2.read() 40 | 41 | # Instance Role and SSM Managed Policy 42 | 43 | role_policy_document = { 44 | "Version": "2012-10-17", 45 | "Statement": [ 46 | { 47 | "Effect": "Allow", 48 | "Principal": { 49 | "AWS": "arn:aws:iam::" + account_id + ":root" 50 | }, 51 | "Action": "sts:AssumeRole" 52 | } 53 | ] 54 | } 55 | client = boto3.client('iam') 56 | roles = client.list_roles() 57 | Role_list = roles['Roles'] 58 | for key in Role_list: 59 | name = key['RoleName'] 60 | if name == 'window-cli-role': 61 | windowcliexists = 1 62 | else: 63 | windowcliexists = 0 64 | 65 | adminrole = aws_iam.Role( 66 | self, 67 | id='windows-cli-role', 68 | assumed_by=aws_iam.ArnPrincipal("arn:aws:iam::" + account_id + ":root"), 69 | role_name='windows-cli-role' 70 | ) 71 | adminrole.add_managed_policy(aws_iam.ManagedPolicy.from_aws_managed_policy_name("AmazonS3ReadOnlyAccess")) 72 | 73 | role = aws_iam.Role(self, "WindowsCLIrole", assumed_by=aws_iam.ServicePrincipal("ec2.amazonaws.com")) 74 | 75 | role.add_to_policy(aws_iam.PolicyStatement( 76 | actions=["sts:AssumeRole"], 77 | resources=["arn:aws:iam::" + account_id + ":role/windows-cli-role"], 78 | effect=aws_iam.Effect.ALLOW 79 | )) 80 | role.add_managed_policy(aws_iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AmazonEC2RoleforSSM")) 81 | role.add_managed_policy(aws_iam.ManagedPolicy.from_aws_managed_policy_name("SecretsManagerReadWrite")) 82 | 83 | ### TAKE THIS OUT SO THAT INSTANCE IS NOT PUBLIC ### 84 | if aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType('PUBLIC')) != None: 85 | subnet = aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType('PUBLIC')) 86 | elif aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType('PRIVATE')) != None: 87 | subnet = aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType('PRIVATE')) 88 | else: 89 | subnet = aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType('ISOLATED')) 90 | 91 | # my_security_group = aws_ec2.SecurityGroup(self, "SecurityGroup", 92 | # vpc=vpc.vpc, 93 | # description="Allow ssh access to ec2 instances", 94 | # allow_all_outbound=True 95 | # ) 96 | # my_security_group.add_ingress_rule(aws_ec2.Peer.ipv4('10.200.0.0/24'), aws_ec2.Port.tcp(22), "allow ssh access from the world") 97 | # my_security_group.add_ingress_rule(my_security_group, aws_ec2.Port.all_tcp(), "self-referencing rule") 98 | my_security_group = vpc.get_vpc_security_group 99 | 100 | my_security_group.add_ingress_rule(peer=aws_ec2.Peer.ipv4(onprem_cidr), connection=aws_ec2.Port.tcp(3389), 101 | description="RDP from anywhere") 102 | 103 | custom_ami = aws_ec2.WindowsImage(aws_ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_BASE); 104 | # Instance 105 | firstcommand = "\naws configure set role_arn arn:aws:iam::" + account_id + ":role/windows-cli-role\n" 106 | input_data = user_data + firstcommand + user_data_2 107 | instance = aws_ec2.Instance(self, "Instance", 108 | instance_type=aws_ec2.InstanceType(jmeter_node_type), 109 | machine_image=custom_ami, 110 | vpc=vpc.vpc, 111 | vpc_subnets=subnet, 112 | key_name=keyname, 113 | role=role, 114 | security_group=my_security_group, 115 | # resource_signal_timeout=core.Duration.minutes(5), 116 | user_data=aws_ec2.UserData.custom(input_data) 117 | ) 118 | -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/redshift-stack-test.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import aws_redshift 2 | from aws_cdk import aws_iam 3 | from aws_cdk import aws_secretsmanager 4 | from aws_cdk import core 5 | import json 6 | import boto3 7 | from aws_cdk import aws_ec2 8 | from redshift_poc_automation.stacks.redshiftrole_stack import RSDefaultRole 9 | from redshift_poc_automation.stacks.redshiftload_stack import RedshiftLoadStack 10 | import builtins 11 | import getpass 12 | 13 | 14 | class RedshiftStack(core.Stack): 15 | 16 | def __init__( 17 | self, 18 | scope: core.Construct, id: str, 19 | vpc, 20 | redshift_endpoint: str, 21 | redshift_config: dict, 22 | stack_log_level: str, 23 | **kwargs 24 | ) -> None: 25 | super().__init__(scope, id, **kwargs) 26 | stackname = id.split('-')[0] 27 | redshift_client = boto3.client('redshift') 28 | 29 | if redshift_endpoint != "CREATE": 30 | ec2_client = boto3.resource('ec2') 31 | cluster_identifier = redshift_endpoint.split('.')[0] 32 | self.redshift = redshift_client.describe_clusters(ClusterIdentifier=cluster_identifier)['Clusters'][0] 33 | 34 | self.password = stackname+'-RedshiftPassword' 35 | 36 | region_name = boto3.session.Session().region_name 37 | session = boto3.session.Session() 38 | client = session.client( 39 | service_name='secretsmanager', 40 | region_name=region_name, 41 | ) 42 | 43 | try: 44 | source_pwd = client.get_secret_value( 45 | SecretId=stackname+'-RedshiftPassword' 46 | )['SecretString'] 47 | except Exception: 48 | source_pwd = getpass.getpass(prompt='Redshift cluster password: ') 49 | client.create_secret( 50 | Name=stackname+'-RedshiftPassword', 51 | SecretString=source_pwd, 52 | Description='Password of Redshift cluster' 53 | ) 54 | 55 | redshift_sg_id = self.redshift['VpcSecurityGroups'][0]['VpcSecurityGroupId'] 56 | redshift_sg_name = ec2_client.SecurityGroup(redshift_sg_id).group_name 57 | 58 | redshift_sg = aws_ec2.SecurityGroup.from_security_group_id(self, redshift_sg_name, redshift_sg_id) 59 | 60 | dms_sg = vpc.get_vpc_security_group 61 | redshift_sg.add_ingress_rule(peer=dms_sg, connection=aws_ec2.Port.all_traffic(), description="DMS input.") 62 | 63 | else: 64 | 65 | cluster_identifier = redshift_config.get('cluster_identifier') 66 | database_name = redshift_config.get('database_name') 67 | node_type = redshift_config.get('node_type') 68 | number_of_nodes = int(redshift_config.get('number_of_nodes')) 69 | master_user_name = redshift_config.get('master_user_name') 70 | subnet_type = redshift_config.get('subnet_type') 71 | encryption = redshift_config.get('encryption') 72 | loadtpc = redshift_config.get('loadTPCdata') 73 | 74 | # Create Cluster Password ## MUST FIX EXCLUDE CHARACTERS FEATURE AS IT STILL INCLUDES SINGLE QUOTES SOMETIMES WHICH WILL FAIL 75 | self.cluster_masteruser_secret = aws_secretsmanager.Secret( 76 | self, 77 | "RedshiftClusterSecret", 78 | description="Redshift Cluster Secret", 79 | secret_name=stackname+'-RedshiftClusterSecretAA', 80 | generate_secret_string=aws_secretsmanager.SecretStringGenerator( 81 | exclude_punctuation=True, password_length=10), 82 | removal_policy=core.RemovalPolicy.DESTROY 83 | ) 84 | 85 | # IAM Role for Cluster 86 | self.cluster_iam_role = aws_iam.Role( 87 | self, "redshiftClusterRole", 88 | assumed_by=aws_iam.ServicePrincipal( 89 | "redshift.amazonaws.com"), 90 | managed_policies=[ 91 | aws_iam.ManagedPolicy.from_aws_managed_policy_name( 92 | "AmazonS3ReadOnlyAccess" 93 | ) 94 | ] 95 | ) 96 | self.cluster_masteruser_secret.grant_read(self.cluster_iam_role) 97 | 98 | publiclyaccessible = False 99 | # Subnet Group for Cluster 100 | if subnet_type == 'PUBLIC': 101 | self.cluster_subnet_group = aws_redshift.CfnClusterSubnetGroup( 102 | self, 103 | "redshiftDemoClusterSubnetGroup", 104 | subnet_ids=vpc.get_vpc_public_subnet_ids, 105 | description="Redshift Demo Cluster Subnet Group" 106 | ) 107 | publiclyaccessible = True 108 | elif subnet_type == 'PRIVATE': 109 | self.cluster_subnet_group = aws_redshift.CfnClusterSubnetGroup( 110 | self, 111 | "redshiftDemoClusterSubnetGroup", 112 | subnet_ids=vpc.get_vpc_private_subnet_ids, 113 | description="Redshift Demo Cluster Subnet Group" 114 | ) 115 | elif subnet_type == 'ISOLATED': 116 | self.cluster_subnet_group = aws_redshift.CfnClusterSubnetGroup( 117 | self, 118 | "redshiftDemoClusterSubnetGroup", 119 | subnet_ids=vpc.get_vpc_private_isolated_subnet_ids, 120 | description="Redshift Demo Cluster Subnet Group" 121 | ) 122 | 123 | if number_of_nodes > 1: 124 | clustertype = "multi-node" 125 | else: 126 | clustertype = "single-node" 127 | number_of_nodes = None 128 | 129 | # Encryption boolean to True if Y or y 130 | if encryption == "Y": 131 | encryptcluster = bool(1) 132 | elif encryption == "y": 133 | encryptcluster = bool(1) 134 | else: 135 | encryptcluster = bool(0) 136 | 137 | security_group_id = vpc.get_vpc_security_group_id 138 | 139 | self.redshift = aws_redshift.CfnCluster( 140 | self, 141 | cluster_identifier, 142 | db_name=database_name, 143 | master_username=master_user_name, 144 | cluster_type=clustertype, 145 | master_user_password=self.cluster_masteruser_secret.secret_value.to_string(), 146 | # master_user_password=master_password, 147 | iam_roles=[self.cluster_iam_role.role_arn], 148 | node_type=f"{node_type}", 149 | encrypted=encryptcluster, 150 | number_of_nodes=number_of_nodes, 151 | publicly_accessible=publiclyaccessible, 152 | cluster_subnet_group_name=self.cluster_subnet_group.ref, 153 | vpc_security_group_ids=[security_group_id] 154 | ) 155 | 156 | ########################################### 157 | ################# OUTPUTS ################# 158 | ########################################### 159 | # output_1 = core.CfnOutput( 160 | # self, 161 | # "RedshiftCluster", 162 | # value=f"{self.demo_cluster.attr_endpoint_address}", 163 | # description=f"RedshiftCluster Endpoint" 164 | # ) 165 | 166 | # output_2 = core.CfnOutput( 167 | # self, 168 | # "RedshiftClusterPassword", 169 | # value=( 170 | # f"https://console.aws.amazon.com/secretsmanager/home?region=" 171 | # f"{core.Aws.REGION}" 172 | # f"#/secret?name=" 173 | # f"{self.cluster_masteruser_secret.secret_arn}" 174 | # ), 175 | # description=f"Redshift Cluster Password in Secrets Manager" 176 | # ) 177 | # output_3 = core.CfnOutput( 178 | # self, 179 | # "RedshiftIAMRole", 180 | # value=( 181 | # f"{self.cluster_iam_role.role_arn}" 182 | # ), 183 | # description=f"Redshift Cluster IAM Role Arn" 184 | # ) 185 | 186 | ############## FIX bug in CDK. Always returns None ######################### 187 | 188 | # output_4 = core.CfnOutput( 189 | # self, 190 | # "RedshiftClusterIdentifier", 191 | # value=( 192 | # f"{self.demo_cluster.cluster_identifier}" 193 | # ), 194 | # description=f"Redshift Cluster Identifier" 195 | # ) 196 | 197 | # properties to share with other stacks 198 | @property 199 | def get_cluster(self): 200 | if type(self.redshift) == dict: 201 | return self.redshift 202 | return self.redshift 203 | 204 | @property 205 | def get_cluster_type(self): 206 | return (type(self.redshift) == dict) 207 | 208 | @property 209 | def get_cluster_dbname(self) -> builtins.str: 210 | if type(self.redshift) == dict: 211 | return self.redshift['DBName'] 212 | return self.redshift.db_name 213 | 214 | @property 215 | def get_cluster_user(self) -> builtins.str: 216 | if type(self.redshift) == dict: 217 | return self.redshift['MasterUsername'] 218 | return self.redshift.master_username 219 | 220 | @property 221 | def get_cluster_password(self) -> builtins.str: 222 | return self.redshift.master_user_password 223 | 224 | @property 225 | def get_cluster_host(self) -> builtins.str: 226 | if type(self.redshift) == dict: 227 | return self.redshift['Endpoint']['Address'] 228 | return self.redshift.attr_endpoint_address 229 | 230 | @property 231 | def get_cluster_iam_role(self) -> builtins.str: 232 | if type(self.redshift) == dict: 233 | return self.redshift['IamRoles'][0]['IamRoleArn'] 234 | return self.cluster_iam_role.role_arn 235 | 236 | @property 237 | def get_cluster_secret(self) -> builtins.str: 238 | if type(self.redshift) == dict: 239 | return self.password 240 | return self.cluster_masteruser_secret.secret_name 241 | 242 | ############## FIX bug in CDK. Always returns None ######################### 243 | @property 244 | def get_cluster_endpoint(self) -> builtins.str: 245 | return str(self.redshift.attr_endpoint_address) 246 | 247 | @property 248 | def get_cluster_identifier(self) -> builtins.str: 249 | return str(self.redshift.attr_endpoint_address).split('.')[0] 250 | 251 | @property 252 | def get_cluster_availability_zone(self) -> builtins.str: 253 | if type(self.redshift) == dict: 254 | return self.redshift['AvailabilityZone'] 255 | return str(self.redshift.availability_zone) 256 | -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/redshift_stack.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import aws_redshift 2 | from aws_cdk import aws_iam 3 | from aws_cdk import aws_secretsmanager 4 | from aws_cdk import core 5 | import json 6 | import boto3 7 | from aws_cdk import aws_ec2 8 | from redshift_poc_automation.stacks.redshiftrole_stack import RSDefaultRole 9 | from redshift_poc_automation.stacks.redshiftload_stack import RedshiftLoadStack 10 | import builtins 11 | import getpass 12 | 13 | 14 | class RedshiftStack(core.Stack): 15 | 16 | def __init__( 17 | self, 18 | scope: core.Construct, id: str, 19 | vpc, 20 | redshift_endpoint: str, 21 | redshift_config: dict, 22 | stack_log_level: str, 23 | **kwargs 24 | ) -> None: 25 | super().__init__(scope, id, **kwargs) 26 | stackname = id.split('-')[0] 27 | redshift_client = boto3.client('redshift') 28 | if redshift_endpoint != "CREATE": 29 | ec2_client = boto3.resource('ec2') 30 | cluster_identifier = redshift_endpoint.split('.')[0] 31 | self.redshift = redshift_client.describe_clusters(ClusterIdentifier=cluster_identifier)['Clusters'][0] 32 | 33 | self.password = stackname+'-RedshiftPassword' 34 | 35 | region_name = boto3.session.Session().region_name 36 | session = boto3.session.Session() 37 | client = session.client( 38 | service_name='secretsmanager', 39 | region_name=region_name, 40 | ) 41 | 42 | try: 43 | source_pwd = client.get_secret_value( 44 | SecretId=stackname+'-RedshiftPassword' 45 | )['SecretString'] 46 | except Exception: 47 | source_pwd = getpass.getpass(prompt='Redshift cluster password: ') 48 | client.create_secret( 49 | Name=stackname+'-RedshiftPassword', 50 | SecretString=source_pwd, 51 | Description='Password of Redshift cluster' 52 | ) 53 | 54 | redshift_sg_id = self.redshift['VpcSecurityGroups'][0]['VpcSecurityGroupId'] 55 | redshift_sg_name = ec2_client.SecurityGroup(redshift_sg_id).group_name 56 | 57 | redshift_sg = aws_ec2.SecurityGroup.from_security_group_id(self, redshift_sg_name, redshift_sg_id) 58 | 59 | dms_sg = vpc.get_vpc_security_group 60 | redshift_sg.add_ingress_rule(peer=dms_sg, connection=aws_ec2.Port.all_traffic(), description="DMS input.") 61 | 62 | else: 63 | 64 | cluster_identifier = redshift_config.get('cluster_identifier') 65 | database_name = redshift_config.get('database_name') 66 | node_type = redshift_config.get('node_type') 67 | number_of_nodes = int(redshift_config.get('number_of_nodes')) 68 | master_user_name = redshift_config.get('master_user_name') 69 | subnet_type = redshift_config.get('subnet_type') 70 | encryption = redshift_config.get('encryption') 71 | loadtpc = redshift_config.get('loadTPCdata') 72 | 73 | # Create Cluster Password ## MUST FIX EXCLUDE CHARACTERS FEATURE AS IT STILL INCLUDES SINGLE QUOTES SOMETIMES WHICH WILL FAIL 74 | self.cluster_masteruser_secret = aws_secretsmanager.Secret( 75 | self, 76 | "RedshiftClusterSecret", 77 | description="Redshift Cluster Secret", 78 | secret_name=stackname+'-RedshiftClusterSecretAA', 79 | generate_secret_string=aws_secretsmanager.SecretStringGenerator( 80 | exclude_punctuation=True, password_length=10), 81 | removal_policy=core.RemovalPolicy.DESTROY 82 | ) 83 | 84 | # IAM Role for Cluster 85 | self.cluster_iam_role = aws_iam.Role( 86 | self, "redshiftClusterRole", 87 | assumed_by=aws_iam.ServicePrincipal( 88 | "redshift.amazonaws.com"), 89 | managed_policies=[ 90 | aws_iam.ManagedPolicy.from_aws_managed_policy_name( 91 | "AmazonS3ReadOnlyAccess" 92 | ) 93 | ] 94 | ) 95 | self.cluster_masteruser_secret.grant_read(self.cluster_iam_role) 96 | 97 | publiclyaccessible = False 98 | # Subnet Group for Cluster 99 | if subnet_type == 'PUBLIC': 100 | self.cluster_subnet_group = aws_redshift.CfnClusterSubnetGroup( 101 | self, 102 | "redshiftDemoClusterSubnetGroup", 103 | subnet_ids=vpc.get_vpc_public_subnet_ids, 104 | description="Redshift Demo Cluster Subnet Group" 105 | ) 106 | publiclyaccessible = True 107 | elif subnet_type == 'PRIVATE': 108 | self.cluster_subnet_group = aws_redshift.CfnClusterSubnetGroup( 109 | self, 110 | "redshiftDemoClusterSubnetGroup", 111 | subnet_ids=vpc.get_vpc_private_subnet_ids, 112 | description="Redshift Demo Cluster Subnet Group" 113 | ) 114 | elif subnet_type == 'ISOLATED': 115 | self.cluster_subnet_group = aws_redshift.CfnClusterSubnetGroup( 116 | self, 117 | "redshiftDemoClusterSubnetGroup", 118 | subnet_ids=vpc.get_vpc_private_isolated_subnet_ids, 119 | description="Redshift Demo Cluster Subnet Group" 120 | ) 121 | 122 | if number_of_nodes > 1: 123 | clustertype = "multi-node" 124 | else: 125 | clustertype = "single-node" 126 | number_of_nodes = None 127 | 128 | # Encryption boolean to True if Y or y 129 | if encryption == "Y": 130 | encryptcluster = bool(1) 131 | elif encryption == "y": 132 | encryptcluster = bool(1) 133 | else: 134 | encryptcluster = bool(0) 135 | 136 | security_group_id = vpc.get_vpc_security_group_id 137 | 138 | self.redshift = aws_redshift.CfnCluster( 139 | self, 140 | cluster_identifier, 141 | db_name=database_name, 142 | master_username=master_user_name, 143 | cluster_type=clustertype, 144 | master_user_password=self.cluster_masteruser_secret.secret_value.to_string(), 145 | # master_user_password=master_password, 146 | iam_roles=[self.cluster_iam_role.role_arn], 147 | node_type=f"{node_type}", 148 | encrypted=encryptcluster, 149 | number_of_nodes=number_of_nodes, 150 | publicly_accessible=publiclyaccessible, 151 | cluster_subnet_group_name=self.cluster_subnet_group.ref, 152 | vpc_security_group_ids=[security_group_id] 153 | ) 154 | 155 | ########################################### 156 | ################# OUTPUTS ################# 157 | ########################################### 158 | # output_1 = core.CfnOutput( 159 | # self, 160 | # "RedshiftCluster", 161 | # value=f"{self.demo_cluster.attr_endpoint_address}", 162 | # description=f"RedshiftCluster Endpoint" 163 | # ) 164 | 165 | # output_2 = core.CfnOutput( 166 | # self, 167 | # "RedshiftClusterPassword", 168 | # value=( 169 | # f"https://console.aws.amazon.com/secretsmanager/home?region=" 170 | # f"{core.Aws.REGION}" 171 | # f"#/secret?name=" 172 | # f"{self.cluster_masteruser_secret.secret_arn}" 173 | # ), 174 | # description=f"Redshift Cluster Password in Secrets Manager" 175 | # ) 176 | # output_3 = core.CfnOutput( 177 | # self, 178 | # "RedshiftIAMRole", 179 | # value=( 180 | # f"{self.cluster_iam_role.role_arn}" 181 | # ), 182 | # description=f"Redshift Cluster IAM Role Arn" 183 | # ) 184 | 185 | ############## FIX bug in CDK. Always returns None ######################### 186 | 187 | # output_4 = core.CfnOutput( 188 | # self, 189 | # "RedshiftClusterIdentifier", 190 | # value=( 191 | # f"{self.demo_cluster.cluster_identifier}" 192 | # ), 193 | # description=f"Redshift Cluster Identifier" 194 | # ) 195 | 196 | # properties to share with other stacks 197 | @property 198 | def get_cluster(self): 199 | if type(self.redshift) == dict: 200 | return self.redshift 201 | return self.redshift 202 | 203 | @property 204 | def get_cluster_type(self): 205 | return (type(self.redshift) == dict) 206 | 207 | @property 208 | def get_cluster_dbname(self) -> builtins.str: 209 | if type(self.redshift) == dict: 210 | return self.redshift['DBName'] 211 | return self.redshift.db_name 212 | 213 | @property 214 | def get_cluster_user(self) -> builtins.str: 215 | if type(self.redshift) == dict: 216 | return self.redshift['MasterUsername'] 217 | return self.redshift.master_username 218 | 219 | @property 220 | def get_cluster_password(self) -> builtins.str: 221 | return self.redshift.master_user_password 222 | 223 | @property 224 | def get_cluster_host(self) -> builtins.str: 225 | if type(self.redshift) == dict: 226 | return self.redshift['Endpoint']['Address'] 227 | return self.redshift.attr_endpoint_address 228 | 229 | @property 230 | def get_cluster_iam_role(self) -> builtins.str: 231 | if type(self.redshift) == dict: 232 | return self.redshift['IamRoles'][0]['IamRoleArn'] 233 | return self.cluster_iam_role.role_arn 234 | 235 | @property 236 | def get_cluster_secret(self) -> builtins.str: 237 | if type(self.redshift) == dict: 238 | return self.password 239 | return self.cluster_masteruser_secret.secret_name 240 | 241 | ############## FIX bug in CDK. Always returns None ######################### 242 | @property 243 | def get_cluster_endpoint(self) -> builtins.str: 244 | return str(self.redshift.attr_endpoint_address) 245 | 246 | @property 247 | def get_cluster_identifier(self) -> builtins.str: 248 | return str(self.redshift.attr_endpoint_address).split('.')[0] 249 | 250 | @property 251 | def get_cluster_availability_zone(self) -> builtins.str: 252 | if type(self.redshift) == dict: 253 | return self.redshift['AvailabilityZone'] 254 | return str(self.redshift.availability_zone) 255 | -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/redshiftload_stack.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | from typing import Any 3 | from aws_cdk import ( 4 | core, 5 | aws_iam as iam, 6 | aws_redshift as aws_redshift 7 | ) 8 | 9 | from aws_cdk.custom_resources import ( 10 | AwsCustomResource, 11 | AwsCustomResourcePolicy, 12 | AwsSdkCall, 13 | PhysicalResourceId, 14 | ) 15 | 16 | class RedshiftLoadStack(core.Stack): 17 | 18 | def __init__( 19 | self, 20 | scope: core.Construct, id: str, 21 | cluster: aws_redshift.CfnCluster, 22 | defaultrole: str, 23 | redshift_config: dict, 24 | stack_log_level: str, 25 | log_retention=None, 26 | **kwargs 27 | ) -> None: 28 | super().__init__(scope, id, **kwargs) 29 | stackname = id.split('-')[0] 30 | 31 | database_name = redshift_config.get('database_name') 32 | master_user_name = redshift_config.get('master_user_name') 33 | 34 | client = boto3.client('redshift-data') 35 | f = open('./scripts/loadTPcH3TB.txt') 36 | 37 | a = f.read() 38 | 39 | default_role=defaultrole 40 | cluster_identifier=cluster.ref 41 | 42 | policy=AwsCustomResourcePolicy.from_sdk_calls( 43 | resources=AwsCustomResourcePolicy.ANY_RESOURCE) 44 | lambda_role = self.get_provisioning_lambda_role(construct_id=id) 45 | #lambda_role.add_to_policy(actions=["redshift:GetClusterCredentials"], resources=['*']) 46 | lambda_role.add_to_policy(iam.PolicyStatement(actions=["redshift:GetClusterCredentials"], resources=['*'])) 47 | 48 | create_params = { 49 | "Database": database_name, 50 | "Sql": a, 51 | "ClusterIdentifier": cluster_identifier, 52 | "DbUser": master_user_name 53 | } 54 | 55 | AwsCustomResource(self, 56 | id=f'{id}-AWSCustomResource', 57 | policy=policy, 58 | log_retention=log_retention, 59 | on_update=AwsSdkCall( 60 | action='executeStatement', 61 | service='RedshiftData', 62 | parameters=create_params, 63 | physical_resource_id=PhysicalResourceId.of( 64 | cluster_identifier), 65 | ), 66 | role=lambda_role) 67 | # You can set the lambda Timeout by passing the Timeout (Default: Duration.minutes(2) 68 | 69 | # Closing file 70 | f.close() 71 | 72 | # api_version=None uses the latest api 73 | #on_update = AwsSdkCall( 74 | # action='modifyClusterIamRoles', 75 | # service='Redshift', 76 | # parameters=create_params, 77 | # physical_resource_id=PhysicalResourceId.of( 78 | # cluster_identifier), 79 | #) 80 | #return on_update 81 | 82 | def get_provisioning_lambda_role(self, construct_id: str): 83 | return iam.Role( 84 | scope=self, 85 | id=f'{construct_id}-LambdaRole', 86 | assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'), 87 | managed_policies=[iam.ManagedPolicy.from_aws_managed_policy_name( 88 | "service-role/AWSLambdaBasicExecutionRole")], 89 | ) 90 | -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/redshiftrole_stack.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from aws_cdk import ( 3 | core, 4 | aws_iam as iam, 5 | aws_redshift as aws_redshift 6 | ) 7 | 8 | from aws_cdk.custom_resources import ( 9 | AwsCustomResource, 10 | AwsCustomResourcePolicy, 11 | AwsSdkCall, 12 | PhysicalResourceId, 13 | ) 14 | 15 | 16 | class RSDefaultRole(core.Stack): 17 | 18 | def __init__( 19 | self, 20 | scope: core.Construct, id: str, 21 | cluster: aws_redshift.CfnCluster, 22 | defaultrole: str, 23 | stack_log_level: str, 24 | log_retention=None, 25 | **kwargs 26 | ) -> None: 27 | super().__init__(scope, id, **kwargs) 28 | stackname = id.split('-')[0] 29 | 30 | default_role=defaultrole 31 | cluster_identifier=cluster.ref 32 | 33 | policy=AwsCustomResourcePolicy.from_sdk_calls( 34 | resources=AwsCustomResourcePolicy.ANY_RESOURCE) 35 | lambda_role = self.get_provisioning_lambda_role(construct_id=id) 36 | 37 | create_params = { 38 | "ClusterIdentifier": cluster_identifier, 39 | "DefaultIamRoleArn": default_role 40 | } 41 | 42 | AwsCustomResource(self, 43 | id=f'{id}-AWSCustomResource', 44 | policy=policy, 45 | log_retention=log_retention, 46 | on_update=AwsSdkCall( 47 | action='modifyClusterIamRoles', 48 | service='Redshift', 49 | parameters=create_params, 50 | physical_resource_id=PhysicalResourceId.of( 51 | cluster_identifier), 52 | ), 53 | # resource_type='Custom::AWS-S3-Object', 54 | role=lambda_role) 55 | # You can set the lambda Timeout by passing the Timeout (Default: Duration.minutes(2) 56 | 57 | 58 | # api_version=None uses the latest api 59 | #on_update = AwsSdkCall( 60 | # action='modifyClusterIamRoles', 61 | # service='Redshift', 62 | # parameters=create_params, 63 | # physical_resource_id=PhysicalResourceId.of( 64 | # cluster_identifier), 65 | #) 66 | #return on_update 67 | 68 | def get_provisioning_lambda_role(self, construct_id: str): 69 | return iam.Role( 70 | scope=self, 71 | id=f'{construct_id}-LambdaRole', 72 | assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'), 73 | managed_policies=[iam.ManagedPolicy.from_aws_managed_policy_name( 74 | "service-role/AWSLambdaBasicExecutionRole")], 75 | ) 76 | 77 | -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/redshiftserverless_stack.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import aws_redshiftserverless 2 | from aws_cdk import aws_iam 3 | from aws_cdk import aws_secretsmanager 4 | from aws_cdk import core 5 | from aws_cdk import aws_ec2 6 | import json 7 | import boto3 8 | from redshift_poc_automation.stacks.redshiftrole_stack import RSDefaultRole 9 | from redshift_poc_automation.stacks.redshiftload_stack import RedshiftLoadStack 10 | import builtins 11 | import getpass 12 | 13 | 14 | class RedshiftServerlessStack(core.Stack): 15 | 16 | def __init__( 17 | self, 18 | scope: core.Construct, id: str, 19 | vpc, 20 | redshift_serverless_endpoint: str, 21 | redshift_serverless_config: dict, 22 | stack_log_level: str, 23 | **kwargs 24 | ) -> None: 25 | super().__init__(scope, id, **kwargs) 26 | 27 | stackname = id.split('-')[0] 28 | redshift_serverless_client = boto3.client('redshift-serverless') 29 | 30 | if redshift_serverless_endpoint != "CREATE": 31 | #ToDo: 32 | ec2_client = boto3.resource('ec2') 33 | cluster_identifier = redshift_serverless_endpoint.split('.')[0] 34 | self.redshift = redshift_serverless_client.describe_clusters(ClusterIdentifier=cluster_identifier)['Clusters'][0] 35 | 36 | self.password = stackname+'-RedshiftPassword' 37 | 38 | region_name = boto3.session.Session().region_name 39 | session = boto3.session.Session() 40 | client = session.client( 41 | service_name='secretsmanager', 42 | region_name=region_name, 43 | ) 44 | 45 | try: 46 | source_pwd = client.get_secret_value( 47 | SecretId=stackname+'-RedshiftPassword' 48 | )['SecretString'] 49 | except Exception: 50 | source_pwd = getpass.getpass(prompt='Redshift serverless cluster password: ') 51 | client.create_secret( 52 | Name=stackname+'-RedshiftPassword', 53 | SecretString=source_pwd, 54 | Description='Password of Redshift cluster' 55 | ) 56 | 57 | redshift_sg_id = self.redshift['VpcSecurityGroups'][0]['VpcSecurityGroupId'] 58 | redshift_sg_name = ec2_client.SecurityGroup(redshift_sg_id).group_name 59 | 60 | redshift_sg = aws_ec2.SecurityGroup.from_security_group_id(self, redshift_sg_name, redshift_sg_id) 61 | 62 | dms_sg = vpc.get_vpc_security_group 63 | redshift_sg.add_ingress_rule(peer=dms_sg, connection=aws_ec2.Port.all_traffic(), description="DMS input.") 64 | 65 | else: 66 | 67 | namespace_name = redshift_serverless_config.get('namespace_name') 68 | workgroup_name = redshift_serverless_config.get('workgroup_name') 69 | base_capacity = redshift_serverless_config.get('base_capacity') 70 | ##admin_user_name = redshift_serverless_config.get('admin_user_name') 71 | ##admin_user_password = redshift_serverless_config.get('admin_user_password') 72 | database_name = redshift_serverless_config.get('database_name') 73 | 74 | # Create Cluster Password ## MUST FIX EXCLUDE CHARACTERS FEATURE AS IT STILL INCLUDES SINGLE QUOTES SOMETIMES WHICH WILL FAIL 75 | #self.cluster_masteruser_secret = aws_secretsmanager.Secret( 76 | # self, 77 | # "RedshiftClusterSecret", 78 | # description="Redshift Cluster Secret", 79 | # secret_name=stackname+'-RedshiftClusterSecretAA', 80 | # generate_secret_string=aws_secretsmanager.SecretStringGenerator( 81 | # exclude_punctuation=True, password_length=10), 82 | # removal_policy=core.RemovalPolicy.DESTROY 83 | #) 84 | 85 | # IAM Role for Cluster 86 | self.cluster_iam_role = aws_iam.Role( 87 | self, "redshiftServerlessClusterRole", 88 | assumed_by=aws_iam.ServicePrincipal( 89 | "redshift.amazonaws.com"), 90 | description="Added by Redshift Infrastructure Automation Toolkit", 91 | managed_policies=[ 92 | aws_iam.ManagedPolicy.from_aws_managed_policy_name( 93 | "AmazonS3ReadOnlyAccess" 94 | ), 95 | aws_iam.ManagedPolicy.from_aws_managed_policy_name( 96 | "AmazonRedshiftFullAccess" 97 | ) 98 | ] 99 | ) 100 | #self.cluster_masteruser_secret.grant_read(self.cluster_iam_role) 101 | 102 | security_group_id = vpc.get_vpc_security_group_id 103 | 104 | self.public_subnet_ids = vpc.get_vpc_public_subnet_ids 105 | 106 | # self.private_subnet_ids = vpc.get_vpc_private_subnet_ids 107 | 108 | self.namespace = aws_redshiftserverless.CfnNamespace(self, "MyCfnNamespace", 109 | namespace_name=namespace_name, 110 | 111 | # the properties below are optional 112 | #admin_username=admin_user_name, 113 | #admin_user_password=admin_user_password, 114 | db_name=database_name, 115 | #default_iam_role_arn=self.cluster_iam_role.role_arn, 116 | iam_roles=[self.cluster_iam_role.role_arn] 117 | ) 118 | 119 | #time.sleep(3) 120 | 121 | self.workgroup = aws_redshiftserverless.CfnWorkgroup(self, "MyCfnWorkgroup", 122 | workgroup_name=workgroup_name, 123 | 124 | # the properties below are optional 125 | base_capacity=base_capacity, 126 | namespace_name=namespace_name, 127 | #publicly_accessible=False, 128 | subnet_ids=self.public_subnet_ids, 129 | security_group_ids=[security_group_id] 130 | ) 131 | self.workgroup.add_depends_on(self.namespace) 132 | 133 | 134 | ########################################### 135 | ################# OUTPUTS ################# 136 | ########################################### 137 | # output_1 = core.CfnOutput( 138 | # self, 139 | # "RedshiftCluster", 140 | # value=f"{self.demo_cluster.attr_endpoint_address}", 141 | # description=f"RedshiftCluster Endpoint" 142 | # ) 143 | 144 | # output_2 = core.CfnOutput( 145 | # self, 146 | # "RedshiftClusterPassword", 147 | # value=( 148 | # f"https://console.aws.amazon.com/secretsmanager/home?region=" 149 | # f"{core.Aws.REGION}" 150 | # f"#/secret?name=" 151 | # f"{self.cluster_masteruser_secret.secret_arn}" 152 | # ), 153 | # description=f"Redshift Cluster Password in Secrets Manager" 154 | # ) 155 | # output_3 = core.CfnOutput( 156 | # self, 157 | # "RedshiftIAMRole", 158 | # value=( 159 | # f"{self.cluster_iam_role.role_arn}" 160 | # ), 161 | # description=f"Redshift Cluster IAM Role Arn" 162 | # ) 163 | 164 | ############## FIX bug in CDK. Always returns None ######################### 165 | 166 | # output_4 = core.CfnOutput( 167 | # self, 168 | # "RedshiftClusterIdentifier", 169 | # value=( 170 | # f"{self.demo_cluster.cluster_identifier}" 171 | # ), 172 | # description=f"Redshift Cluster Identifier" 173 | # ) 174 | 175 | # properties to share with other stacks 176 | # IMPORTANT: these methods are for Redshift Provisioned only 177 | @property 178 | def get_cluster(self): 179 | if type(self.redshift) == dict: ##needed for dms 180 | return self.redshift 181 | return self.redshift 182 | 183 | @property 184 | def get_cluster_type(self): 185 | return (type(self.redshift) == dict) #needed dms 186 | 187 | @property 188 | def get_cluster_dbname(self) -> builtins.str: #needed dms 189 | if type(self.redshift) == dict: 190 | return self.redshift['DBName'] 191 | return self.redshift.db_name 192 | 193 | @property 194 | def get_cluster_user(self) -> builtins.str: #needed dms 195 | if type(self.redshift) == dict: 196 | return self.redshift['MasterUsername'] 197 | return self.redshift.master_username 198 | 199 | @property 200 | def get_cluster_password(self) -> builtins.str: #needed dms 201 | return self.redshift.master_user_password 202 | 203 | @property 204 | def get_cluster_host(self) -> builtins.str: #needed dms 205 | if type(self.redshift) == dict: 206 | return self.redshift['Endpoint']['Address'] 207 | return self.redshift.attr_endpoint_address 208 | 209 | @property 210 | def get_cluster_iam_role(self) -> builtins.str: 211 | if type(self.redshift) == dict: 212 | return self.redshift['IamRoles'][0]['IamRoleArn'] 213 | return self.cluster_iam_role.role_arn 214 | 215 | @property 216 | def get_cluster_secret(self) -> builtins.str: 217 | if type(self.redshift) == dict: 218 | return self.password 219 | return self.cluster_masteruser_secret.secret_name 220 | 221 | ############## FIX bug in CDK. Always returns None ######################### 222 | @property 223 | def get_cluster_endpoint(self) -> builtins.str: 224 | return str(self.redshift.attr_endpoint_address) 225 | 226 | @property 227 | def get_cluster_identifier(self) -> builtins.str: 228 | return str(self.redshift.attr_endpoint_address).split('.')[0] 229 | 230 | @property 231 | def get_cluster_availability_zone(self) -> builtins.str: 232 | if type(self.redshift) == dict: 233 | return self.redshift['AvailabilityZone'] 234 | return str(self.redshift.availability_zone) 235 | -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/sct_stack.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import aws_iam 2 | from aws_cdk import aws_ec2 3 | from aws_cdk import core 4 | from aws_cdk import aws_secretsmanager 5 | import boto3 6 | import json 7 | 8 | 9 | class SctOnPremToRedshiftStack(core.Stack): 10 | 11 | def __init__( 12 | self, 13 | scope: core.Construct, id: str, 14 | other_config: dict, 15 | vpc, 16 | stack_log_level: str, 17 | onprem_cidr: str, 18 | **kwargs 19 | 20 | ) -> None: 21 | super().__init__(scope, id, **kwargs) 22 | 23 | keyname = other_config.get('key_name') 24 | secret_arn = 'RedshiftClusterSecretAA' 25 | amiID = 'ami-042e0580ee1b9e2af' 26 | 27 | account_id = boto3.client('sts').get_caller_identity().get('Account') 28 | 29 | with open("./sctconfig.sh") as f: 30 | user_data = f.read() 31 | 32 | with open("./sctconfig_2.sh") as f_2: 33 | user_data_2 = f_2.read() 34 | 35 | # Instance Role and SSM Managed Policy 36 | 37 | role_policy_document = { 38 | "Version": "2012-10-17", 39 | "Statement": [ 40 | { 41 | "Effect": "Allow", 42 | "Principal": { 43 | "AWS": "arn:aws:iam::" + account_id + ":root" 44 | }, 45 | "Action": "sts:AssumeRole" 46 | } 47 | ] 48 | } 49 | client = boto3.client('iam') 50 | roles = client.list_roles() 51 | Role_list = roles['Roles'] 52 | for key in Role_list: 53 | name = key['RoleName'] 54 | if name == 'window-cli-role': 55 | windowcliexists = 1 56 | else: 57 | windowcliexists = 0 58 | 59 | if windowcliexists == 1: 60 | adminrole = aws_iam.Role( 61 | self, 62 | id='windows-cli-role', 63 | assumed_by=aws_iam.ArnPrincipal("arn:aws:iam::" + account_id + ":root"), 64 | role_name='windows-cli-role' 65 | ) 66 | adminrole.add_managed_policy(aws_iam.ManagedPolicy.from_aws_managed_policy_name("AmazonS3ReadOnlyAccess")) 67 | 68 | role = aws_iam.Role(self, "WindowsCLIrole", assumed_by=aws_iam.ServicePrincipal("ec2.amazonaws.com")) 69 | 70 | role.add_to_policy(aws_iam.PolicyStatement( 71 | actions=["sts:AssumeRole"], 72 | resources=["arn:aws:iam::" + account_id + ":role/windows-cli-role"], 73 | effect=aws_iam.Effect.ALLOW 74 | )) 75 | role.add_managed_policy(aws_iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AmazonEC2RoleforSSM")) 76 | role.add_managed_policy(aws_iam.ManagedPolicy.from_aws_managed_policy_name("SecretsManagerReadWrite")) 77 | 78 | # secrets_client = boto3.client(service_name='secretsmanager', region_name='us-east-1') 79 | # get_secret_value_response = secrets_client.get_secret_value( 80 | # SecretId=secret_arn 81 | # ) 82 | # redshift_pwd = [value for value in get_secret_value_response.values()][3] 83 | 84 | ### TAKE THIS OUT SO THAT INSTANCE IS NOT PUBLIC ### 85 | if aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType('PUBLIC')) != None: 86 | subnet = aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType('PUBLIC')) 87 | elif aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType('PRIVATE_WITH_NAT')) != None: 88 | subnet = aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType('PRIVATE_WITH_NAT')) 89 | else: 90 | subnet = aws_ec2.SubnetSelection(subnet_type=aws_ec2.SubnetType('PRIVATE_ISOLATED')) 91 | 92 | # my_security_group = aws_ec2.SecurityGroup(self, "SecurityGroup", 93 | # vpc=vpc.vpc, 94 | # description="Allow ssh access to ec2 instances", 95 | # allow_all_outbound=True 96 | # ) 97 | # my_security_group.add_ingress_rule(aws_ec2.Peer.ipv4('10.200.0.0/24'), aws_ec2.Port.tcp(22), "allow ssh access from the world") 98 | # my_security_group.add_ingress_rule(my_security_group, aws_ec2.Port.all_tcp(), "self-referencing rule") 99 | my_security_group = vpc.get_vpc_security_group 100 | 101 | my_security_group.add_ingress_rule(peer=aws_ec2.Peer.ipv4(onprem_cidr), connection=aws_ec2.Port.tcp(3389), 102 | description="RDP from anywhere") 103 | 104 | custom_ami = aws_ec2.WindowsImage(aws_ec2.WindowsVersion.WINDOWS_SERVER_2019_ENGLISH_FULL_BASE); 105 | # Instance 106 | firstcommand = "\naws configure set role_arn arn:aws:iam::" + account_id + ":role/windows-cli-role\n" 107 | input_data = user_data + firstcommand + user_data_2 108 | instance = aws_ec2.Instance(self, "Instance", 109 | instance_type=aws_ec2.InstanceType("m5.large"), 110 | machine_image=custom_ami, 111 | vpc=vpc.vpc, 112 | vpc_subnets=subnet, 113 | key_name=keyname, 114 | role=role, 115 | security_group=my_security_group, 116 | # resource_signal_timeout=core.Duration.minutes(5), 117 | user_data=aws_ec2.UserData.custom(input_data) 118 | ) 119 | -------------------------------------------------------------------------------- /redshift_poc_automation/stacks/vpc_stack.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import aws_ec2 2 | from aws_cdk import core 3 | import random 4 | import string 5 | 6 | 7 | class GlobalArgs(): 8 | """ 9 | Helper to define global statics 10 | """ 11 | 12 | OWNER = "Redshift POC SSA team" 13 | ENVIRONMENT = "development" 14 | REPO_NAME = "redshift-demo" 15 | VERSION = "2021_03_15" 16 | 17 | class VpcStack(core.Stack): 18 | 19 | def __init__( 20 | self, 21 | scope: core.Construct, 22 | id: str, 23 | stack_log_level: str, 24 | vpc_id: str, 25 | onprem_cidr: str, 26 | vpc_config: dict, 27 | 28 | ** kwargs 29 | ) -> None: 30 | super().__init__(scope, id, **kwargs) 31 | 32 | if vpc_id != "CREATE": 33 | self.vpc = aws_ec2.Vpc.from_lookup( 34 | self, "vpc", 35 | vpc_id=vpc_id 36 | ) 37 | 38 | else: 39 | vpc_cidr = vpc_config.get('vpc_cidr') 40 | cidr_mask = int(vpc_config.get('cidr_mask')) 41 | number_of_az = int(vpc_config.get('number_of_az')) 42 | 43 | self.vpc = aws_ec2.Vpc( 44 | self, 45 | "RedshiftPOCVpc", 46 | cidr= vpc_cidr, 47 | max_azs=number_of_az, 48 | enable_dns_support=True, 49 | enable_dns_hostnames=True, 50 | subnet_configuration=[ 51 | aws_ec2.SubnetConfiguration( 52 | name="public_subnet", cidr_mask=cidr_mask, subnet_type=aws_ec2.SubnetType.PUBLIC 53 | ), 54 | aws_ec2.SubnetConfiguration( 55 | name="private_subnet", cidr_mask=cidr_mask, subnet_type=aws_ec2.SubnetType.PRIVATE 56 | ) 57 | ] 58 | ) 59 | 60 | letters = string.ascii_lowercase 61 | tail = ''.join(random.choice(letters) for i in range(5)) 62 | 63 | self.dms_security_group = aws_ec2.SecurityGroup( 64 | self, 65 | id = "sct-sg-dms-" + tail, 66 | vpc = self.vpc, 67 | security_group_name = "sct-sg-dms" + tail, 68 | description = "Gives DMS instance access to Redshift" 69 | ) 70 | self.dms_security_group.add_ingress_rule(peer=self.dms_security_group, connection=aws_ec2.Port.all_traffic(), description="Self-referencing rule.") 71 | self.dms_security_group.add_ingress_rule(peer=aws_ec2.Peer.ipv4(onprem_cidr), connection=aws_ec2.Port.tcp(22), description="SSH from anywhere") 72 | 73 | 74 | output_1 = core.CfnOutput( 75 | self, 76 | "New SG", 77 | value=f"{self.dms_security_group.security_group_id}", 78 | description="New security group of this VPC." 79 | ) 80 | 81 | # properties to share with other stacks 82 | @property 83 | def get_vpc(self): 84 | return self.vpc 85 | 86 | @property 87 | def get_vpc_public_subnet_ids(self): 88 | return self.vpc.select_subnets( 89 | subnet_type=aws_ec2.SubnetType.PUBLIC 90 | ).subnet_ids 91 | 92 | @property 93 | def get_vpc_private_isolated_subnet_ids(self): 94 | return self.vpc.select_subnets( 95 | subnet_type=aws_ec2.SubnetType.ISOLATED 96 | ).subnet_ids 97 | 98 | @property 99 | def get_vpc_private_subnet_ids(self): 100 | return self.vpc.select_subnets( 101 | subnet_type=aws_ec2.SubnetType.PRIVATE 102 | ).subnet_ids 103 | 104 | @property 105 | def get_vpc_security_group_id(self): 106 | return self.dms_security_group.security_group_id 107 | 108 | @property 109 | def get_vpc_security_group(self): 110 | return self.dms_security_group 111 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | pytest 3 | boto3 4 | -------------------------------------------------------------------------------- /scripts/delete_buckets.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import os 4 | 5 | stackname = os.getenv('STACK_NAME') 6 | 7 | region_name = boto3.session.Session().region_name 8 | session = boto3.session.Session() 9 | sm_client = session.client( 10 | service_name='s3', 11 | region_name=region_name, 12 | ) 13 | s3 = boto3.resource('s3') 14 | response = sm_client.list_buckets() 15 | 16 | for bucket in response['Buckets']: 17 | if 'cdktoolkit' in bucket['Name']: 18 | x = bucket['Name'] 19 | print(x) 20 | bucket = s3.Bucket(x) 21 | bucket.objects.all().delete() 22 | bucket.delete() 23 | -------------------------------------------------------------------------------- /scripts/delete_secrets.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import os 4 | 5 | stackname = os.getenv('STACK_NAME') 6 | 7 | region_name = boto3.session.Session().region_name 8 | session = boto3.session.Session() 9 | sm_client = session.client( 10 | service_name='secretsmanager', 11 | region_name=region_name, 12 | ) 13 | 14 | secrets_list = [f"{stackname}-SourceDBPassword", 15 | f"{stackname}-RedshiftPassword", 16 | f"{stackname}-RedshiftClusterSecretAA"] 17 | 18 | sm_response = sm_client.list_secrets() 19 | for secret in sm_response['SecretList']: 20 | if secret['Name'] in secrets_list: 21 | response = sm_client.delete_secret( 22 | SecretId=secret['Name'] 23 | ) 24 | if response['ResponseMetadata']['HTTPStatusCode'] == 200: 25 | print(f"{secret['Name']} successfully deleted") 26 | else: 27 | print(f"Error: {response}") 28 | -------------------------------------------------------------------------------- /scripts/deploy.sh: -------------------------------------------------------------------------------- 1 | echo "Installing dependencies...."; sudo yum -y install gcc gcc-c++ python3 python3-devel unixODBC unixODBC-devel aws-cfn-bootstrap > /dev/null; echo " done." 2 | echo "Installing aws-cdk...."; sudo npm install -g aws-cdk@2.x > /dev/null; echo " done." 3 | chmod +x ~/amazon-redshift-infrastructure-automation/scripts/shell_menu/menu-script.sh 4 | chmod +x ~/amazon-redshift-infrastructure-automation/scripts/shell_menu/bash-menu-cli-commands.sh 5 | chmod +x ~/amazon-redshift-infrastructure-automation/scripts/shell_menu/menu-welcome-message.sh 6 | chmod +x ~/amazon-redshift-infrastructure-automation/scripts/shell_menu/miscdetails.sh 7 | 8 | cd ~/amazon-redshift-infrastructure-automation 9 | python3 -m venv .env 10 | source .env/bin/activate 11 | echo "Installing requirements...."; pip install -r requirements.txt > /dev/null; echo " done." 12 | #LINE='~/amazon-redshift-infrastructure-automation/scripts/restart_session.sh' 13 | #FILE=~/.bashrc 14 | #grep -qF -- "$LINE" "$FILE" || echo "$LINE" >> "$FILE" 15 | aws configure set default.region us-east-1 16 | ~/amazon-redshift-infrastructure-automation/scripts/shell_menu/menu-welcome-message.sh 17 | read -r -p "Do you have an existing user-config.json file? (Yy/Nn): " answer 18 | case $answer in 19 | [Yy]* ) read -r -p "Please upload your user-config.json file and press ENTER to continue..." answer; 20 | source ~/amazon-redshift-infrastructure-automation/scripts/shell_menu/miscdetails.sh;; 21 | [Nn]* ) source ~/amazon-redshift-infrastructure-automation/scripts/shell_menu/menu-script.sh;; 22 | * ) echo "Please answer Y or N.";; 23 | esac 24 | export STACK_NAME=$stack 25 | export ONPREM_CIDR=$onprem_cidr 26 | #Need more elegant solution for handling exception here: 27 | [ -f ~/user-config.json ] && mv ~/user-config.json ~/amazon-redshift-infrastructure-automation/user-config.json 28 | export account_id=`aws sts get-caller-identity --query "Account" --output text` 29 | if [ "$loadTPCdata" = "Y" ]; 30 | then 31 | cdk bootstrap aws://$account_id/$current_region 32 | fi 33 | cdk deploy --all --require-approval never 34 | -------------------------------------------------------------------------------- /scripts/destroy.sh: -------------------------------------------------------------------------------- 1 | # Runs and deploys CDK 2 | sudo yum -y install gcc gcc-c++ python3 python3-devel unixODBC unixODBC-devel aws-cfn-bootstrap 3 | sudo npm install -g aws-cdk 4 | cd ~/amazon-redshift-infrastructure-automation 5 | python3 -m venv .env 6 | source .env/bin/activate 7 | pip install -r requirements.txt 8 | if [ -z "${STACK_NAME}" ] 9 | then 10 | read -p $'[Input Required] Enter a stack name: ' stack 11 | export STACK_NAME=$stack 12 | fi 13 | #if [ -z "${ONPREM_CIDR}" ] 14 | #then 15 | # read -p $'[Input Required] Enter your on prem CIDR range (format xxx.xxx.xxx.xxx/xx): ' onprem_cidr 16 | # export ONPREM_CIDR=$onprem_cidr 17 | #fi 18 | export ONPREM_CIDR=12.34.56.78/32 19 | python3 ./scripts/delete_buckets.py 20 | python3 ./scripts/detach_policy.py 21 | cdk destroy --all --require-approval never 22 | aws cloudformation delete-stack --stack-name CDKToolkit 23 | 24 | python3 << EOF 25 | import boto3 26 | import json 27 | import os 28 | stackname = os.getenv('STACK_NAME') 29 | region_name = boto3.session.Session().region_name 30 | session = boto3.session.Session() 31 | sm_client = session.client( 32 | service_name='secretsmanager', 33 | region_name=region_name, 34 | ) 35 | secrets_list = [f"{stackname}-SourceDBPassword", 36 | f"{stackname}-RedshiftPassword", 37 | f"{stackname}-RedshiftClusterSecretAA"] 38 | 39 | sm_response = sm_client.list_secrets() 40 | for secret in sm_response['SecretList']: 41 | if secret['Name'] in secrets_list: 42 | os.system('python /home/cloudshell-user/amazon-redshift-infrastructure-automation/scripts/delete_secrets.py') 43 | EOF 44 | -------------------------------------------------------------------------------- /scripts/detach_policy.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import os 4 | 5 | stackname = os.getenv('STACK_NAME') 6 | 7 | region_name = boto3.session.Session().region_name 8 | session = boto3.session.Session() 9 | sm_client = session.client( 10 | service_name='iam', 11 | region_name=region_name, 12 | ) 13 | iam = boto3.resource('iam') 14 | response = sm_client.list_roles() 15 | 16 | for role in response['Roles']: 17 | if 'WindowsCLIrole' in role['RoleName']: 18 | x = role['RoleName'] 19 | print(x) 20 | response = sm_client.detach_role_policy( 21 | RoleName=x, 22 | PolicyArn='arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore' 23 | ) 24 | -------------------------------------------------------------------------------- /scripts/jmeter.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | rem 3 | rem Licensed to the Apache Software Foundation (ASF) under one or more 4 | rem contributor license agreements. See the NOTICE file distributed with 5 | rem this work for additional information regarding copyright ownership. 6 | rem The ASF licenses this file to you under the Apache License, Version 2.0 7 | rem (the "License"); you may not use this file except in compliance with 8 | rem the License. You may obtain a copy of the License at 9 | rem 10 | rem http://www.apache.org/licenses/LICENSE-2.0 11 | rem 12 | rem Unless required by applicable law or agreed to in writing, software 13 | rem distributed under the License is distributed on an "AS IS" BASIS, 14 | rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | rem See the License for the specific language governing permissions and 16 | rem limitations under the License. 17 | rem 18 | 19 | rem ===================================================== 20 | rem Environment variables that can be defined externally: 21 | rem 22 | rem Do not set the variables in this script. Instead put them into a script 23 | rem setenv.bat in JMETER_HOME/bin to keep your customizations separate. 24 | rem 25 | rem DDRAW - (Optional) JVM options to influence usage of direct draw, 26 | rem e.g. '-Dsun.java2d.ddscale=true' 27 | rem 28 | rem JMETER_BIN - JMeter bin directory (must end in \) 29 | rem 30 | rem JMETER_COMPLETE_ARGS - if set indicates that JVM_ARGS is to be used exclusively instead 31 | rem of adding other options like HEAP or GC_ALGO 32 | rem 33 | rem JMETER_HOME - installation directory. Will be guessed from location of jmeter.bat 34 | rem 35 | rem JM_LAUNCH - java.exe (default) or javaw.exe 36 | rem 37 | rem JM_START - set this to 'start ""' to launch JMeter in a separate window 38 | rem this is used by the jmeterw.cmd script. 39 | rem 40 | rem JVM_ARGS - (Optional) Java options used when starting JMeter, e.g. -Dprop=val 41 | rem Defaults to '-Duser.language="en" -Duser.region="EN"' 42 | rem 43 | rem GC_ALGO - (Optional) JVM garbage collector options 44 | rem Defaults to '-XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:G1ReservePercent=20' 45 | rem 46 | rem HEAP - (Optional) JVM memory settings used when starting JMeter 47 | rem Defaults to '-Xms1g -Xmx1g -XX:MaxMetaspaceSize=256m' 48 | rem 49 | rem ===================================================== 50 | 51 | setlocal 52 | 53 | rem Guess JMETER_HOME if not defined 54 | set "CURRENT_DIR=%cd%" 55 | if not "%JMETER_HOME%" == "" goto gotHome 56 | set "JMETER_HOME=%CURRENT_DIR%" 57 | if exist "%JMETER_HOME%\bin\jmeter.bat" goto okHome 58 | cd .. 59 | set "JMETER_HOME=%cd%" 60 | cd "%CURRENT_DIR%" 61 | if exist "%JMETER_HOME%\bin\jmeter.bat" goto okHome 62 | set "JMETER_HOME=%~dp0\.." 63 | :gotHome 64 | 65 | if exist "%JMETER_HOME%\bin\jmeter.bat" goto okHome 66 | echo The JMETER_HOME environment variable is not defined correctly 67 | echo This environment variable is needed to run this program 68 | goto end 69 | :okHome 70 | 71 | rem Get standard environment variables 72 | if exist "%JMETER_HOME%\bin\setenv.bat" call "%JMETER_HOME%\bin\setenv.bat" 73 | 74 | if not defined JMETER_LANGUAGE ( 75 | rem Set language 76 | rem Default to en_EN 77 | set JMETER_LANGUAGE=-Duser.language="en" -Duser.region="EN" 78 | ) 79 | 80 | rem Minimal version to run JMeter 81 | set MINIMAL_VERSION=1.8.0 82 | 83 | 84 | rem --add-opens if JAVA 9 85 | set JAVA9_OPTS= 86 | 87 | 88 | for /f "tokens=3" %%g in ('java -version 2^>^&1 ^| findstr /i "version"') do ( 89 | rem @echo Debug Output: %%g 90 | set JAVAVER=%%g 91 | ) 92 | if not defined JAVAVER ( 93 | @echo Not able to find Java executable or version. Please check your Java installation. 94 | set ERRORLEVEL=2 95 | goto pause 96 | ) 97 | 98 | 99 | 100 | rem Check if version is from OpenJDK or Oracle Hotspot JVM prior to 9 containing 1.${version}.x 101 | rem JAVAVER will be equal to "9.0.4" (quotes are part of the value) for Oracle Java 9 102 | rem JAVAVER will be equal to "1.8.0_161" (quotes are part of the value) for Oracle Java 8 103 | rem so we extract 2 chars starting from index 1 104 | IF "%JAVAVER:~1,2%"=="1." ( 105 | set JAVAVER=%JAVAVER:"=% 106 | for /f "delims=. tokens=1-3" %%v in ("%JAVAVER%") do ( 107 | set current_minor=%%w 108 | ) 109 | ) else ( 110 | rem Java 9 at least 111 | set current_minor=9 112 | set JAVA9_OPTS=--add-opens java.desktop/sun.awt=ALL-UNNAMED --add-opens java.desktop/sun.swing=ALL-UNNAMED --add-opens java.desktop/javax.swing.text.html=ALL-UNNAMED --add-opens java.desktop/java.awt=ALL-UNNAMED --add-opens java.desktop/java.awt.font=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED 113 | ) 114 | 115 | 116 | for /f "delims=. tokens=1-3" %%v in ("%MINIMAL_VERSION%") do ( 117 | set minimal_minor=%%w 118 | ) 119 | 120 | if not defined current_minor ( 121 | @echo Not able to find Java executable or version. Please check your Java installation. 122 | set ERRORLEVEL=2 123 | goto pause 124 | ) 125 | rem @echo Debug: CURRENT=%current_minor% - MINIMAL=%minimal_minor% 126 | if %current_minor% LSS %minimal_minor% ( 127 | @echo Error: Java version -- %JAVAVER% -- is too low to run JMeter. Needs a Java version greater than or equal to %MINIMAL_VERSION% 128 | set ERRORLEVEL=3 129 | goto pause 130 | ) 131 | 132 | if not defined JM_LAUNCH ( 133 | set JM_LAUNCH=java.exe 134 | ) 135 | 136 | if exist jmeter.bat goto winNT1 137 | if not defined JMETER_BIN ( 138 | set JMETER_BIN=%~dp0 139 | ) 140 | 141 | :winNT1 142 | rem On NT/2K grab all arguments at once 143 | set JMETER_CMD_LINE_ARGS=%* 144 | 145 | rem The following link describes the -XX options: 146 | rem http://docs.oracle.com/javase/8/docs/technotes/tools/unix/java.html 147 | 148 | if not defined HEAP ( 149 | rem See the unix startup file for the rationale of the following parameters, 150 | rem including some tuning recommendations 151 | set HEAP=-Xms5g -Xmx5g -XX:MaxMetaspaceSize=1g 152 | ) 153 | 154 | rem Uncomment this to generate GC verbose file with Java prior to 9 155 | rem set VERBOSE_GC=-verbose:gc -Xloggc:gc_jmeter_%%p.log -XX:+PrintGCDetails -XX:+PrintGCCause -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC -XX:+PrintGCApplicationConcurrentTime -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCDateStamps -XX:+PrintAdaptiveSizePolicy 156 | 157 | rem Uncomment this to generate GC verbose file with Java 9 and above 158 | rem set VERBOSE_GC=-Xlog:gc*,gc+age=trace,gc+heap=debug:file=gc_jmeter_%%p.log 159 | rem You may want to add those settings 160 | rem -XX:+ParallelRefProcEnabled -XX:+PerfDisableSharedMem 161 | if not defined GC_ALGO ( 162 | set GC_ALGO=-XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:G1ReservePercent=20 163 | ) 164 | 165 | set SYSTEM_PROPS=-Djava.security.egd=file:/dev/urandom 166 | 167 | rem Always dump on OOM (does not cost anything unless triggered) 168 | set DUMP=-XX:+HeapDumpOnOutOfMemoryError 169 | 170 | rem Uncomment this if you run JMeter in DOCKER (need Java SE 8u131 or JDK 9) 171 | rem see https://blogs.oracle.com/java-platform-group/java-se-support-for-docker-cpu-and-memory-limits 172 | rem set RUN_IN_DOCKER=-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap 173 | 174 | rem Additional settings that might help improve GUI performance on some platforms 175 | rem See: http://www.oracle.com/technetwork/java/perf-graphics-135933.html 176 | 177 | if not defined DDRAW ( 178 | set DDRAW= 179 | rem Setting this flag to true turns off DirectDraw usage, which sometimes helps to get rid of a lot of rendering problems on Win32. 180 | rem set DDRAW=%DDRAW% -Dsun.java2d.noddraw=true 181 | 182 | rem Setting this flag to false turns off DirectDraw offscreen surfaces acceleration by forcing all createVolatileImage calls to become createImage calls, and disables hidden acceleration performed on surfaces created with createImage . 183 | rem set DDRAW=%DDRAW% -Dsun.java2d.ddoffscreen=false 184 | 185 | rem Setting this flag to true enables hardware-accelerated scaling. 186 | rem set DDRAW=%DDRAW% -Dsun.java2d.ddscale=true 187 | ) 188 | 189 | rem Collect the settings defined above 190 | if not defined JMETER_COMPLETE_ARGS ( 191 | set ARGS=%JAVA9_OPTS% %DUMP% %HEAP% %VERBOSE_GC% %GC_ALGO% %DDRAW% %SYSTEM_PROPS% %JMETER_LANGUAGE% %RUN_IN_DOCKER% 192 | ) else ( 193 | set ARGS= 194 | ) 195 | 196 | if "%JM_START%" == "start" ( 197 | set JM_START=start "Apache_JMeter" 198 | ) 199 | 200 | %JM_START% "%JM_LAUNCH%" %ARGS% %JVM_ARGS% -jar "C:\JMETER\apache-jmeter-5.6.3\bin\ApacheJMeter.jar" %JMETER_CMD_LINE_ARGS% 201 | 202 | rem If the errorlevel is not zero, then display it and pause 203 | 204 | if NOT errorlevel 0 goto pause 205 | if errorlevel 1 goto pause 206 | 207 | goto end 208 | 209 | :pause 210 | echo errorlevel=%ERRORLEVEL% 211 | pause 212 | 213 | :end 214 | -------------------------------------------------------------------------------- /scripts/redshift-jdbc42-2.0.0.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-redshift-infrastructure-automation/ce8cf02fecd016cf90377605c179e228d17eaef3/scripts/redshift-jdbc42-2.0.0.4.jar -------------------------------------------------------------------------------- /scripts/restart_session.sh: -------------------------------------------------------------------------------- 1 | # Reinstall required yum packages 2 | sudo yum -y install gcc gcc-c++ python3 python3-devel unixODBC unixODBC-devel aws-cfn-bootstrap 3 | # Reinstall CDK 4 | sudo npm install -g aws-cdk 5 | # Activates Python virtual environment 6 | source ~/amazon-redshift-infrastructure-automation/.env/bin/activate 7 | # Reinstall required python libraries 8 | pip install -r ~/amazon-redshift-infrastructure-automation/requirements.txt 9 | # clear screen 10 | clear -------------------------------------------------------------------------------- /scripts/shell_menu/bash-menu-cli-commands.sh: -------------------------------------------------------------------------------- 1 | aws ec2 --output text --query 'Vpcs[*].{VpcId:VpcId}' describe-vpcs > vpclist.txt 2 | 3 | aws redshift --output text --query 'Clusters[*].{Endpoint:Endpoint.Address}' describe-clusters > redshiftlist.txt 4 | 5 | aws redshift --output text --query 'Clusters[*].{ClusterIdentifier:ClusterIdentifier}' describe-clusters > redshiftidentifierlist.txt 6 | 7 | aws ec2 --output text describe-key-pairs --query 'KeyPairs[*].{KeyName:KeyName}' > keypairlist.txt -------------------------------------------------------------------------------- /scripts/shell_menu/menu-welcome-message.sh: -------------------------------------------------------------------------------- 1 | function box_out() 2 | { 3 | local s=("$@") b w 4 | for l in "${s[@]}"; do 5 | ((w<${#l})) && { b="$l"; w="${#l}"; } 6 | done 7 | tput setaf 3 8 | echo " -${b//?/-}- 9 | | ${b//?/ } |" 10 | for l in "${s[@]}"; do 11 | printf '| %s%*s%s |\n' "$(tput setaf 4)" "-$w" "$l" "$(tput setaf 3)" 12 | done 13 | echo "| ${b//?/ } | 14 | -${b//?/-}-" 15 | tput sgr 0 16 | } 17 | 18 | box_out "Welcome!" "This utility tool will help you create the required resources for $(whoami)" "Please review the pre-requisites at the following link before proceeding: " "" "https://github.com/aws-samples/amazon-redshift-infrastructure-automation#prerequisites" 19 | echo 20 | -------------------------------------------------------------------------------- /scripts/shell_menu/miscdetails.sh: -------------------------------------------------------------------------------- 1 | 2 | echo 3 | read -p "Please enter a region name(us-east-1, us-east-2, etc): " current_region 4 | echo 5 | read -p "Enter a stack name: " stack 6 | echo 7 | read -p "Enter your on prem CIDR range (format xxx.xxx.xxx.xxx/xx): " onprem_cidr 8 | -------------------------------------------------------------------------------- /scripts/shell_menu/permission-check.sh: -------------------------------------------------------------------------------- 1 | aws iam simulate-principal-policy output --policy-source-arn arn:aws:iam::465110942800:user/Test --action-names redshift:DescribeClusters 2 | -------------------------------------------------------------------------------- /sctconfig.sh: -------------------------------------------------------------------------------- 1 | 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | 4 | with open("README.md") as fp: 5 | long_description = fp.read() 6 | 7 | 8 | setuptools.setup( 9 | name="redshift_poc_automation", 10 | version="0.0.1", 11 | 12 | description="A sample CDK Python app", 13 | long_description=long_description, 14 | long_description_content_type="text/markdown", 15 | 16 | author="author", 17 | 18 | package_dir={"": "redshift_poc_automation"}, 19 | packages=setuptools.find_packages(where="redshift_poc_automation"), 20 | 21 | install_requires=[ 22 | "aws-cdk.core==1.174.0", 23 | "aws-cdk.aws_iam==1.174.0", 24 | "aws-cdk.aws_sqs==1.174.0", 25 | "aws-cdk.aws_sns==1.174.0", 26 | "aws-cdk.aws_sns_subscriptions==1.174.0", 27 | "aws-cdk.aws_s3==1.174.0", 28 | "aws_cdk.aws_ec2==1.174.0", 29 | "aws_cdk.aws_dms==1.174.0", 30 | "aws_cdk.aws_redshift==1.174.0", 31 | "aws_cdk.aws_cloudformation==1.174.0", 32 | "aws_cdk.custom_resources==1.174.0", 33 | "aws_cdk.aws_glue==1.174.0", 34 | "aws-cdk.aws_redshiftserverless==1.174.0" 35 | ], 36 | 37 | python_requires=">=3.6", 38 | 39 | classifiers=[ 40 | "Development Status :: 4 - Beta", 41 | 42 | "Intended Audience :: Developers", 43 | 44 | "License :: OSI Approved :: Apache Software License", 45 | 46 | "Programming Language :: JavaScript", 47 | "Programming Language :: Python :: 3 :: Only", 48 | "Programming Language :: Python :: 3.6", 49 | "Programming Language :: Python :: 3.7", 50 | "Programming Language :: Python :: 3.8", 51 | 52 | "Topic :: Software Development :: Code Generators", 53 | "Topic :: Utilities", 54 | 55 | "Typing :: Typed", 56 | ], 57 | ) 58 | -------------------------------------------------------------------------------- /source.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem The sole purpose of this script is to make the command 4 | rem 5 | rem source .venv/bin/activate 6 | rem 7 | rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. 8 | rem On Windows, this command just runs this batch file (the argument is ignored). 9 | rem 10 | rem Now we don't need to document a Windows command for activating a virtualenv. 11 | 12 | echo Executing .venv\Scripts\activate.bat for you 13 | .venv\Scripts\activate.bat 14 | -------------------------------------------------------------------------------- /user-config-sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "vpc_id": "vpc-fd5b3880", 3 | "redshift_endpoint": "CREATE", 4 | "redshift_serverless_endpoint": "CREATE", 5 | "dms_migration_to_redshift_target": "N/A", 6 | "sct_on_prem_to_redshift_target": "N/A", 7 | "jmeter": "CREATE", 8 | "datashare": "", 9 | "vpc": { 10 | "vpc_cidr": "10.1.0.0/16", 11 | "number_of_az": "3", 12 | "cidr_mask": "24" 13 | }, 14 | "redshift_serverless": { 15 | "namespace_name": "aaa-namespace-1", 16 | "workgroup_name": "aaa-workgroup-1", 17 | "base_capacity": 48, 18 | "database_name": "dev" 19 | }, 20 | "redshift": { 21 | "cluster_identifier": "target-cluster", 22 | "database_name": "dev", 23 | "node_type": "ra3.4xlarge", 24 | "number_of_nodes": "12", 25 | "master_user_name": "awsuser", 26 | "subnet_type": "PUBLIC", 27 | "encryption": "Y", 28 | "loadTPCdata": "Y" 29 | }, 30 | "datasharing":{ 31 | "datashare_name":"", 32 | "producer_cluster_identifier": "", 33 | "producer_database_name": "", 34 | "producer_username": "", 35 | "producer_schema_name": "", 36 | "consumer_cluster_identifier":"", 37 | "consumer_database_name": "", 38 | "consumer_username":"" 39 | }, 40 | "dms_migration": { 41 | "dms_instance_type": "dms.t3.medium", 42 | "subnet_type": "PUBLIC", 43 | "migration_type": "full-load" 44 | }, 45 | "external_database": { 46 | "source_db": "dms_sample", 47 | "source_engine": "sqlserver", 48 | "source_schema": "dbo", 49 | "source_host": "ec2-3-86-250-37.compute-1.amazonaws.com", 50 | "source_user": "awssct", 51 | "source_port": 1433 52 | }, 53 | "other": { 54 | "key_name": "cdkstaging2", 55 | "jmeter_node_type": "m5.xlarge" 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /user-config-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "vpc_id": "CREATE", 3 | "redshift_endpoint": "CREATE", 4 | "redshift_serverless_endpoint": "CREATE or N/A", 5 | "dms_migration_to_redshift_target": "CREATE", 6 | "sct_on_prem_to_redshift_target": "CREATE", 7 | "jmeter": "CREATE", 8 | 9 | "vpc": { 10 | "vpc_cidr": "__", 11 | "number_of_az": "__", 12 | "cidr_mask": "__" 13 | }, 14 | "redshift_serverless": { 15 | "namespace_name": "aaa-namespace-1", 16 | "workgroup_name": "aaa-workgroup-1", 17 | "base_capacity": 48, 18 | "database_name": "dev" 19 | }, 20 | "redshift": { 21 | "cluster_identifier": "__", 22 | "database_name": "__", 23 | "node_type": "__", 24 | "number_of_nodes": "__", 25 | "master_user_name": "__", 26 | "subnet_type": "__", 27 | "encryption": "__", 28 | "loadTPCdata": "__" 29 | 30 | }, 31 | "datasharing":{ 32 | "datashare_name":"", 33 | "producer_cluster_identifier": "", 34 | "producer_database_name": "", 35 | "producer_username": "", 36 | "producer_schema_name": "", 37 | "consumer_cluster_identifier":"", 38 | "consumer_database_name": "", 39 | "consumer_username":"" 40 | }, 41 | "dms_migration": { 42 | "dms_instance_type": "", 43 | "subnet_type": "__", 44 | "migration_type": "__" 45 | }, 46 | "external_database": { 47 | "source_db": "__", 48 | "source_engine": "__", 49 | "source_schema": "__", 50 | "source_host": "__", 51 | "source_user": "__", 52 | "source_port": 1111 53 | }, 54 | "other": { 55 | "key_name": "__", 56 | "jmeter_node_type": "__" 57 | } 58 | } 59 | --------------------------------------------------------------------------------