├── .github ├── solutionid_validator.sh └── workflows │ └── maintainer_workflows.yml ├── .gitignore ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── app_account.py ├── app_governance.py ├── assets └── images │ └── reference-architecture.png ├── base ├── deploy.sh ├── params │ └── dz_conn_b__params.json └── template.yaml ├── cdk.json ├── config ├── account │ └── a__config.py ├── common │ └── global_vars.py └── governance │ └── g_config.py ├── libs ├── python311 │ └── boto3-layer.zip └── python38 │ ├── oracledb-layer.zip │ └── pyodbc-layer.zip ├── requirements.txt ├── source.bat └── src ├── account ├── code │ ├── iam │ │ └── datazone_custom_environment_permission_boundary.json │ └── lambda │ │ ├── clean_environment_roles │ │ └── clean_environment_roles.py │ │ ├── manage_service_portfolio_environment_roles_access │ │ └── manage_service_portfolio_environment_roles_access.py │ │ └── update_environment_roles │ │ └── update_environment_roles.py └── stacks │ └── account_common_stack.py ├── consumer ├── code │ ├── lambda │ │ ├── copy_subscription_secret │ │ │ └── copy_subscription_secret.py │ │ ├── delete_subscription_secret │ │ │ └── delete_subscription_secret.py │ │ ├── remove_subscription_records │ │ │ └── remove_subscription_records.py │ │ └── update_subscription_records │ │ │ └── update_subscription_records.py │ └── stepfunctions │ │ ├── consumer_manage_subscription_grant_workflow.asl.json │ │ └── consumer_manage_subscription_revoke_workflow.asl.json ├── constructs │ ├── athena_connection.py │ ├── consumer_subscription_grant_workflow.py │ └── consumer_subscription_revoke_workflow.py └── stacks │ ├── consumer_service_portfolio_stack.py │ └── consumer_workflows_stack.py ├── governance ├── code │ ├── lambda │ │ ├── get_environment_details │ │ │ └── get_environment_details.py │ │ ├── get_subscription_details │ │ │ └── get_subscription_details.py │ │ └── start_subscription_workflow │ │ │ └── start_subscription_workflow.py │ └── stepfunctions │ │ ├── governance_manage_environment_active_workflow.asl.json │ │ ├── governance_manage_environment_delete_workflow.asl.json │ │ ├── governance_manage_subscription_grant_workflow.asl.json │ │ └── governance_manage_subscription_revoke_workflow.asl.json ├── constructs │ ├── governance_environment_active_workflow.py │ ├── governance_environment_delete_workflow.py │ ├── governance_subscription_grant_workflow.py │ └── governance_subscription_revoke_workflow.py └── stacks │ ├── governance_common_stack.py │ └── governance_workflows_stack.py └── producer ├── code ├── lambda │ ├── add_lf_tag_environment_dbs │ │ └── add_lf_tag_environment_dbs.py │ ├── delete_keep_subscription_secret │ │ └── delete_keep_subscription_secret.py │ ├── get_connection_details │ │ └── get_connection_details.py │ ├── grant_jdbc_subscription │ │ └── grant_jdbc_subscription.py │ ├── revoke_jdbc_subscription │ │ └── revoke_jdbc_subscription.py │ └── share_subscription_secret │ │ └── share_subscription_secret.py └── stepfunctions │ ├── producer_manage_subscription_grant_workflow.asl.json │ └── producer_manage_subscription_revoke_workflow.asl.json ├── constructs ├── glue_connection.py ├── producer_subscription_grant_workflow.py └── producer_subscription_revoke_workflow.py └── stacks ├── producer_common_stack.py ├── producer_service_portfolio_stack.py └── producer_workflows_stack.py /.github/solutionid_validator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #set -e 3 | 4 | echo "checking solution id $1" 5 | echo "grep -nr --exclude-dir='.github' "$1" ./.." 6 | result=$(grep -nr --exclude-dir='.github' "$1" ./..) 7 | if [ $? -eq 0 ] 8 | then 9 | echo "Solution ID $1 found\n" 10 | echo "$result" 11 | exit 0 12 | else 13 | echo "Solution ID $1 not found" 14 | exit 1 15 | fi 16 | 17 | export result 18 | -------------------------------------------------------------------------------- /.github/workflows/maintainer_workflows.yml: -------------------------------------------------------------------------------- 1 | # Workflows managed by aws-solutions-library-samples maintainers 2 | name: Maintainer Workflows 3 | on: 4 | # Triggers the workflow on push or pull request events but only for the "main" branch 5 | push: 6 | branches: [ "main" ] 7 | pull_request: 8 | branches: [ "main" ] 9 | types: [opened, reopened, edited] 10 | 11 | jobs: 12 | CheckSolutionId: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Run solutionid validator 17 | run: | 18 | chmod u+x ./.github/solutionid_validator.sh 19 | ./.github/solutionid_validator.sh ${{ vars.SOLUTIONID }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | package-lock.json 3 | __pycache__ 4 | .pytest_cache 5 | .venv 6 | *.egg-info 7 | temp 8 | .DS_Store 9 | output 10 | 11 | # CDK asset staging directory 12 | .cdk.staging 13 | cdk.out 14 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | CODEOWNERS @aws-solutions-library-samples/maintainers 2 | /.github/workflows/maintainer_workflows.yml @aws-solutions-library-samples/maintainers 3 | /.github/solutionid_validator.sh @aws-solutions-library-samples/maintainers 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT No Attribution 2 | 3 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 13 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 15 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 16 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | 18 | -------------------------------------------------------------------------------- /app_account.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import aws_cdk as cdk 3 | from importlib import import_module 4 | 5 | from src.account.stacks.account_common_stack import DataZoneConnectorsAccountCommonStack 6 | 7 | from src.producer.stacks.producer_common_stack import DataZoneConnectorsProducerCommonStack 8 | from src.producer.stacks.producer_workflows_stack import ProducerWorkflowsStack 9 | from src.producer.stacks.producer_service_portfolio_stack import ProducerServicePortfolioStack 10 | 11 | from src.consumer.stacks.consumer_workflows_stack import ConsumerWorkflowsStack 12 | from src.consumer.stacks.consumer_service_portfolio_stack import ConsumerServicePortfolioStack 13 | 14 | app = cdk.App() 15 | account = cdk.Aws.ACCOUNT_ID 16 | region = cdk.Aws.REGION 17 | 18 | env = cdk.Environment( 19 | account= account, 20 | region= region 21 | ) 22 | 23 | context_account_id = app.node.try_get_context("account") 24 | account_config_module = import_module(f'config.account.a_{context_account_id}_config') 25 | 26 | ACCOUNT_PROPS = getattr(account_config_module, 'ACCOUNT_PROPS') 27 | 28 | PRODUCER_PROPS = getattr(account_config_module, 'PRODUCER_PROPS') 29 | PRODUCER_WORKFLOW_PROPS = getattr(account_config_module, 'PRODUCER_WORKFLOW_PROPS') 30 | PRODUCER_SERVICE_PORTFOLIO_PROPS = getattr(account_config_module, 'PRODUCER_SERVICE_PORTFOLIO_PROPS') 31 | 32 | CONSUMER_WORKFLOW_PROPS = getattr(account_config_module, 'CONSUMER_WORKFLOW_PROPS') 33 | CONSUMER_SERVICE_PORTFOLIO_PROPS = getattr(account_config_module, 'CONSUMER_SERVICE_PORTFOLIO_PROPS') 34 | 35 | 36 | # ---------------- Account Stacks ------------------------ 37 | account_common_constructs = {} 38 | 39 | dz_conn_a_common_stack = DataZoneConnectorsAccountCommonStack( 40 | scope= app, 41 | construct_id = "dz-conn-a-common-stack", 42 | account_props = ACCOUNT_PROPS, 43 | env = env, 44 | description= "Guidance for Connecting Data Products with Amazon DataZone - Account Common Stack - (SO9317)" 45 | ) 46 | 47 | account_common_constructs = { 48 | **account_common_constructs, 49 | **dz_conn_a_common_stack.outputs 50 | } 51 | 52 | # ---------------- Producer Stacks ------------------------ 53 | dz_conn_p_common_stack = DataZoneConnectorsProducerCommonStack( 54 | scope= app, 55 | construct_id = "dz-conn-p-common-stack", 56 | account_props = ACCOUNT_PROPS, 57 | producer_props = PRODUCER_PROPS, 58 | common_constructs = account_common_constructs, 59 | env = env, 60 | description= "Guidance for Connecting Data Products with Amazon DataZone - Producer Common Stack - (SO9317)" 61 | ) 62 | 63 | dz_conn_p_common_stack.node.add_dependency(dz_conn_a_common_stack) 64 | 65 | account_common_constructs = { 66 | **account_common_constructs, 67 | **dz_conn_p_common_stack.outputs 68 | } 69 | 70 | dz_conn_p_workflows_stack = ProducerWorkflowsStack( 71 | scope= app, 72 | construct_id = "dz-conn-p-workflows-stack", 73 | account_props = ACCOUNT_PROPS, 74 | workflows_props = PRODUCER_WORKFLOW_PROPS, 75 | common_constructs = account_common_constructs, 76 | env = env, 77 | description= "Guidance for Connecting Data Products with Amazon DataZone - Producer Workflows Stack - (SO9317)" 78 | ) 79 | 80 | dz_conn_p_service_portfolio_stack = ProducerServicePortfolioStack( 81 | scope= app, 82 | construct_id = "dz-conn-p-service-portfolio-stack", 83 | account_props = ACCOUNT_PROPS, 84 | portfolio_props = PRODUCER_SERVICE_PORTFOLIO_PROPS, 85 | common_constructs = account_common_constructs, 86 | env = env, 87 | description= "Guidance for Connecting Data Products with Amazon DataZone - Producer Portfolio Stack - (SO9317)" 88 | ) 89 | 90 | # ---------------- Consumer Stacks ------------------------ 91 | dz_conn_c_workflows_stack = ConsumerWorkflowsStack( 92 | scope= app, 93 | construct_id = "dz-conn-c-workflows-stack", 94 | account_props = ACCOUNT_PROPS, 95 | workflows_props = CONSUMER_WORKFLOW_PROPS, 96 | common_constructs = account_common_constructs, 97 | env = env, 98 | description= "Guidance for Connecting Data Products with Amazon DataZone - Consumer Workflows Stack - (SO9317)" 99 | ) 100 | 101 | dz_conn_c_service_portfolio_stack = ConsumerServicePortfolioStack( 102 | scope= app, 103 | construct_id = "dz-conn-c-service-portfolio-stack", 104 | account_props = ACCOUNT_PROPS, 105 | portfolio_props = CONSUMER_SERVICE_PORTFOLIO_PROPS, 106 | common_constructs = account_common_constructs, 107 | env = env, 108 | description= "Guidance for Connecting Data Products with Amazon DataZone - Consumer Portfolio Stack - (SO9317)" 109 | ) 110 | 111 | app.synth() 112 | -------------------------------------------------------------------------------- /app_governance.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import aws_cdk as cdk 3 | 4 | from config.governance.g_config import GOVERNANCE_PROPS, GOVERNANCE_WORKFLOW_PROPS 5 | 6 | from src.governance.stacks.governance_common_stack import DataZoneConnectorsGovernanceCommonStack 7 | from src.governance.stacks.governance_workflows_stack import GovernanceWorkflowsStack 8 | 9 | app = cdk.App() 10 | account = cdk.Aws.ACCOUNT_ID 11 | region = cdk.Aws.REGION 12 | 13 | env = cdk.Environment( 14 | account= account, 15 | region= region 16 | ) 17 | 18 | # ---------------- Governance Stacks ------------------------ 19 | governance_common_constructs = {} 20 | 21 | dz_conn_g_common_stack = DataZoneConnectorsGovernanceCommonStack( 22 | scope= app, 23 | construct_id = "dz-conn-g-common-stack", 24 | governance_props = GOVERNANCE_PROPS, 25 | env = env, 26 | description= "Guidance for Connecting Data Products with Amazon DataZone - Governance Common Stack - (SO9317)" 27 | ) 28 | 29 | governance_common_constructs = { 30 | **governance_common_constructs, 31 | **dz_conn_g_common_stack.outputs 32 | } 33 | 34 | dz_conn_g_workflows_stack = GovernanceWorkflowsStack( 35 | scope= app, 36 | construct_id = "dz-conn-g-workflows-stack", 37 | governance_props = GOVERNANCE_PROPS, 38 | workflows_props = GOVERNANCE_WORKFLOW_PROPS, 39 | common_constructs = governance_common_constructs, 40 | env = env, 41 | description= "Guidance for Connecting Data Products with Amazon DataZone - Governance Workflows Stack - (SO9317)" 42 | ) 43 | 44 | app.synth() 45 | -------------------------------------------------------------------------------- /assets/images/reference-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-connecting-data-products-with-amazon-datazone/689fd472dd67677cedbea594d8f05e6803dee315/assets/images/reference-architecture.png -------------------------------------------------------------------------------- /base/deploy.sh: -------------------------------------------------------------------------------- 1 | UUID_ENV=$(uuidgen) 2 | UUID_ENV=${UUID_ENV:0:7} 3 | UUID_ENV=$(echo $UUID_ENV | tr '[:upper:]' '[:lower:]') 4 | BUCKET_NAME="cfn-dz-conn-base-env-${UUID_ENV}" 5 | 6 | while getopts p:a: flag 7 | do 8 | case "${flag}" in 9 | p) PROFILE_NAME=${OPTARG};; 10 | a) ACCOUNT_ID=${OPTARG};; 11 | esac 12 | done 13 | 14 | aws cloudformation deploy --stack-name dz-conn-base-env --template-file base/template.yaml --parameter-overrides "file://base/params/dz_conn_b_${ACCOUNT_ID}_params.json" --capabilities CAPABILITY_NAMED_IAM --profile $PROFILE_NAME -------------------------------------------------------------------------------- /base/params/dz_conn_b__params.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "ParameterKey": "CdkCloudFormationExecutionRoleName", 4 | "ParameterValue": "" 5 | } 6 | ] -------------------------------------------------------------------------------- /base/template.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | Description: "Includes Amazon IAM role, assumed by CloudFormation (CDK) when deploying DataZone Connectors solution, as an AWS Lake Formation administrator." 3 | 4 | Parameters: 5 | 6 | CdkCloudFormationExecutionRoleName: 7 | Type: String 8 | Description: >- 9 | Name of the CDK CloudFormation Execution Role that will be used for DataZone Connectors app deployment. 10 | 11 | Resources: 12 | 13 | CdkCloudFormationExecutionRoleLakeFormationAdminSetting: 14 | Type: AWS::LakeFormation::DataLakeSettings 15 | Properties: 16 | Admins: 17 | - DataLakePrincipalIdentifier: !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${CdkCloudFormationExecutionRoleName} 18 | TrustedResourceOwners: 19 | - !Ref "AWS::AccountId" 20 | 21 | Outputs: 22 | 23 | CdkCloudFormationExecutionRoleArn: 24 | Description: "Arn of the CDK CloudFormation role to be used to deploy DataZone Connectors app" 25 | Value: !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${CdkCloudFormationExecutionRoleName} -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "context": { 3 | "@aws-cdk/aws-lambda:recognizeLayerVersion": true, 4 | "@aws-cdk/core:checkSecretUsage": true, 5 | "@aws-cdk/core:target-partitions": [ 6 | "aws", 7 | "aws-cn" 8 | ], 9 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 10 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 11 | "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, 12 | "@aws-cdk/aws-iam:minimizePolicies": true, 13 | "@aws-cdk/core:validateSnapshotRemovalPolicy": true, 14 | "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, 15 | "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, 16 | "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, 17 | "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, 18 | "@aws-cdk/core:enablePartitionLiterals": true, 19 | "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, 20 | "@aws-cdk/aws-iam:standardizedServicePrincipals": true, 21 | "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, 22 | "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, 23 | "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, 24 | "@aws-cdk/aws-route53-patters:useCertificate": true, 25 | "@aws-cdk/customresources:installLatestAwsSdkDefault": false, 26 | "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, 27 | "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, 28 | "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, 29 | "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, 30 | "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, 31 | "@aws-cdk/aws-redshift:columnId": true, 32 | "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, 33 | "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, 34 | "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /config/account/a__config.py: -------------------------------------------------------------------------------- 1 | # ------------------ Account ------------------------ 2 | """ 3 | ACCOUNT_PROPS dict will be used to consolidate common properties for an account with producer / consumer capabilities. 4 | The dict structures includes: 5 | account_id: str - Account id 6 | region: str - Region 7 | service_portfolio: dict - Dict containing properties for account service portfolio including: 8 | access_role_names: list - Name of the roles that are able to access / manage the service portfolio. 9 | vpc: dict - Dict containing properties for account vpc including: 10 | vpc_id: str - Id of the vpc 11 | availability_zones: list - List of Availability Zones covering the vpc 12 | private_subnets: list - List of subnet ids of the vpc 13 | security_groups: list - List of security groups of the vpc 14 | s3: dict - Dict containing properties for account S3 setup including: 15 | bucket_name: str - Name of the bucket to be created by solution to store data associated to its use. 16 | """ 17 | ACCOUNT_PROPS = { 18 | 'account_id': '', 19 | 'region': '', 20 | 'service_portfolio': { 21 | 'access_role_names': [] 22 | }, 23 | 'vpc': { 24 | 'vpc_id': '', 25 | 'availability_zones': [], 26 | 'private_subnets': [], 27 | 'security_groups': [] 28 | }, 29 | 's3': { 30 | 'bucket_name': 'dz-conn-a--' 31 | } 32 | } 33 | 34 | # ------------------ Producer ------------------------ 35 | """ 36 | PRODUCER_PROPS dict will be used to consolidate common producer-related properties for an account. 37 | The dict structures includes: 38 | p_lakeformation_tag_principals: dict - Dict containing properties for lake formation tag principals. Each key represents a principal; key (assigned by you) will be used as resource physical id and value is a dict including: 39 | type_name: str - IAM principal type and name in the format '{PRINCIPAL_TYPE}/{PRINCIPAL_NAME_PATH}'. For example 'role/Admin'. 40 | lf_tag_permission: bool - If principal should have admin permission on top of the lake formation tag deployed as part of the solution. 41 | lf_tag_objects_permission: bool - If principal should have admin permission on top of objects tagged with the lake formation tag deployed as part of the solution. 42 | """ 43 | PRODUCER_PROPS = { 44 | 'p_lakeformation_tag_principals': { 45 | '': { 46 | 'type_name': '', 47 | 'lf_tag_permission': False, 48 | 'lf_tag_objects_permission': False 49 | } 50 | } 51 | } 52 | 53 | """ 54 | PRODUCER_WORKFLOW_PROPS dict will be used to consolidate producer workflow properties for an account. 55 | The dict structures includes a key (not to be modified) per workflow: 56 | p_manage_subscription_grant: dict - Dict containing properties for managing subscription grants in the producer side including: 57 | vpc_id: str - Id of the vpc where lambda function connecting to data sources will be allocated 58 | vpc_private_subnet_ids: list - List of subnet ids of the vpc where lambda function connecting to data sources will be allocated 59 | vpc_security_group_ids: list - List of security groups of the vpc that will be associated to the lambda function connecting to data sources 60 | p_manage_subscription_revoke: dict - Dict containing properties for managing subscription revocations in the producer side including: 61 | vpc_id: str - Id of the vpc where lambda function connecting to data sources will be allocated 62 | vpc_private_subnet_ids: list - List of subnet ids of the vpc where lambda function connecting to data sources will be allocated 63 | vpc_security_group_ids: list - List of security groups of the vpc that will be associated to the lambda function connecting to data sources 64 | secret_recovery_window_in_days: str - Number of days (min '7') to use as retention window when scheduling deletion of secrets 65 | """ 66 | PRODUCER_WORKFLOW_PROPS = { 67 | 'p_manage_subscription_grant': { 68 | 'vpc_id': ACCOUNT_PROPS['vpc']['vpc_id'], 69 | 'vpc_private_subnet_ids': ACCOUNT_PROPS['vpc']['private_subnets'], 70 | 'vpc_security_group_ids': ACCOUNT_PROPS['vpc']['security_groups'] 71 | }, 72 | 'p_manage_subscription_revoke': { 73 | 'vpc_id': ACCOUNT_PROPS['vpc']['vpc_id'], 74 | 'vpc_private_subnet_ids': ACCOUNT_PROPS['vpc']['private_subnets'], 75 | 'vpc_security_group_ids': ACCOUNT_PROPS['vpc']['security_groups'], 76 | 'secret_recovery_window_in_days': '7' 77 | } 78 | } 79 | 80 | """ 81 | PRODUCER_SERVICE_PORTFOLIO_PROPS dict will be used to consolidate producer service portfolio product properties for an account. 82 | The dict structures includes: 83 | p_product_default_props: dict - Dict containing default values for producer products in the service catalog portfolio of the account. Each key (not to be modified) represents a product: 84 | glue_jdbc_connector: dict - Dict containing default values for Glue JDBC Connector product including: 85 | ssl: bool - Default value for enforcing ssl on glue connection when deployed. 86 | availability_zone: str - Default value for the availability zone where glue connection will be deployed. 87 | security_group_ids: list - Default value for list of security groups to be assigned to glue connection when deployed. 88 | subnet_id: str - Default value for the subnet where glue connection will be deployed. 89 | glue_jdbc_connector_with_crawler: dict - Dict containing default values for Glue JDBC Connector with Crawler product 90 | ssl: bool - Default value for enforcing ssl on glue connection when deployed. 91 | availability_zone: str - Default value for the availability zone where glue connection will be deployed. 92 | security_group_ids: list - Default value for list of security groups to be assigned to glue connection when deployed. 93 | subnet_id: str - Default value for the subnet where glue connection will be deployed. 94 | """ 95 | PRODUCER_SERVICE_PORTFOLIO_PROPS = { 96 | 'p_product_default_props': { 97 | 'glue_jdbc_connector': { 98 | 'ssl': True, 99 | 'availability_zone': ACCOUNT_PROPS['vpc']['availability_zones'][0], 100 | 'security_group_ids': ACCOUNT_PROPS['vpc']['security_groups'], 101 | 'subnet_id': ACCOUNT_PROPS['vpc']['private_subnets'][0] 102 | }, 103 | 'glue_jdbc_connector_with_crawler': { 104 | 'ssl': True, 105 | 'availability_zone': ACCOUNT_PROPS['vpc']['availability_zones'][0], 106 | 'security_group_ids': ACCOUNT_PROPS['vpc']['security_groups'], 107 | 'subnet_id': ACCOUNT_PROPS['vpc']['private_subnets'][0] 108 | } 109 | } 110 | } 111 | 112 | # ------------------ Consumer ------------------------ 113 | """ 114 | CONSUMER_WORKFLOW_PROPS dict will be used to consolidate consumer workflow properties for an account. 115 | The dict structures includes a key (not to be modified) per workflow: 116 | c_manage_subscription_grant: dict - Dict containing properties for managing subscription grants in the consumer side. Reserved for future use (Leave as empty dict). 117 | c_manage_subscription_revoke: dict - Dict containing properties for managing subscriptions revocations in the consumer side including: 118 | secret_recovery_window_in_days: str - Number of days (min '7') to use as retention window when scheduling deletion of secrets 119 | """ 120 | CONSUMER_WORKFLOW_PROPS = { 121 | 'c_manage_subscription_grant': { }, 122 | 'c_manage_subscription_revoke': { 123 | 'secret_recovery_window_in_days': '7' 124 | } 125 | } 126 | 127 | """ 128 | CONSUMER_SERVICE_PORTFOLIO_PROPS dict will be used to consolidate consumer service portfolio product properties for an account. 129 | The dict structures includes: 130 | c_product_default_props: dict - Dict containing default values for consumer products in the service catalog portfolio of the account. Each key (not to be modified) represents a product: 131 | athena_mysql_jdbc_connector: dict - Dict containing default values for Athena MySQL JDBC Connection product including: 132 | port: str - Default value for port to be used when connecting to source database. 133 | security_group_ids: list - Default value for list of security groups to be assigned to athena connection's underlying lambda when deployed. 134 | subnet_ids: list - Default value for the list of subnets where athena connection's underlying lambda will be deployed. 135 | athena_postgres_jdbc_connector: dict - Dict containing default values for Athena PostgreSQL JDBC Connection product 136 | port: str - Default value for port to be used when connecting to source database. 137 | security_group_ids: list - Default value for list of security groups to be assigned to athena connection's underlying lambda when deployed. 138 | subnet_ids: list - Default value for the list of subnets where athena connection's underlying lambda will be deployed. 139 | athena_sqlserver_jdbc_connector: dict - Dict containing default values for Athena SQL Server JDBC Connection product 140 | port: str - Default value for port to be used when connecting to source database. 141 | security_group_ids: list - Default value for list of security groups to be assigned to athena connection's underlying lambda when deployed. 142 | subnet_ids: list - Default value for the list of subnets where athena connection's underlying lambda will be deployed. 143 | ssl: str - 'true' or 'false' depending if ssl should be used when connecting to source database 144 | athena_oracle_jdbc_connector: dict - Dict containing default values for Athena Oracle JDBC Connection product 145 | port: str - Default value for port to be used when connecting to source database. 146 | security_group_ids: list - Default value for list of security groups to be assigned to athena connection's underlying lambda when deployed. 147 | subnet_ids: list - Default value for the list of subnets where athena connection's underlying lambda will be deployed. 148 | """ 149 | CONSUMER_SERVICE_PORTFOLIO_PROPS = { 150 | 'c_product_default_props': { 151 | 'athena_mysql_jdbc_connector': { 152 | 'port': '3306', 153 | 'security_group_ids': ACCOUNT_PROPS['vpc']['security_groups'], 154 | 'subnet_ids': ACCOUNT_PROPS['vpc']['private_subnets'] 155 | }, 156 | 'athena_postgres_jdbc_connector': { 157 | 'port': '5432', 158 | 'security_group_ids': ACCOUNT_PROPS['vpc']['security_groups'], 159 | 'subnet_ids': ACCOUNT_PROPS['vpc']['private_subnets'] 160 | }, 161 | 'athena_sqlserver_jdbc_connector': { 162 | 'port': '1433', 163 | 'security_group_ids': ACCOUNT_PROPS['vpc']['security_groups'], 164 | 'subnet_ids': ACCOUNT_PROPS['vpc']['private_subnets'], 165 | 'ssl': 'true' 166 | }, 167 | 'athena_oracle_jdbc_connector': { 168 | 'port': '1521', 169 | 'security_group_ids': ACCOUNT_PROPS['vpc']['security_groups'], 170 | 'subnet_ids': ACCOUNT_PROPS['vpc']['private_subnets'] 171 | } 172 | } 173 | } -------------------------------------------------------------------------------- /config/common/global_vars.py: -------------------------------------------------------------------------------- 1 | from config.governance.g_config import GOVERNANCE_PROPS 2 | 3 | """ 4 | GLOBAL_VARIABLES dict will be used to consolidate global variables to be accessed from everywhere from within solution. 5 | This dict is intended to BE LEFT UNMODIFIED. The dict structure includes: 6 | account: dict - Dict containing global variables for account (with producer / consumer capabilities) related resources, including: 7 | a_account_numbers: list - List containing ids of all accounts with producer / consumer capabilities 8 | a_common_glue_role_name: str - Name to be used in all accounts' role for glue resources 9 | a_common_lambda_role_name: str - Name to be used in all accounts' role for lambda functions 10 | a_cross_account_assume_role_name: str - Name to be used in all accounts' role that can be assumed by governance account for cross-account access 11 | a_update_environment_roles_lambda_name: str - Name to be used in all accounts' lambda function that will update new DataZone environment roles on creation 12 | a_clean_environment_roles_lambda_name: str - Name to be used in all accounts' lambda function that will clean DataZone environment roles on deletion 13 | producer: dict - Dict containing global variables for account's producer capability related resources, including: 14 | p_add_lf_tag_environment_dbs_lambda_name: str - Name to be used in all accounts' lambda function that will tag new DataZone environments' databases in glue catalog with LakeFormation solutions tag 15 | p_manage_subscription_grant_state_machine_name: str - Name to be used in all accounts' state machine that will orchestrate subscription grant tasks on the producer side 16 | p_manage_subscription_revoke_state_machine_name: str - Name to be used in all accounts' state machine that will orchestrate subscription revoke tasks on the producer side 17 | consumer: dict - Dict containing global variables for account's consumer capability related resources, including: 18 | c_manage_subscription_grant_state_machine_name: str - Name to be used in all accounts' state machine that will orchestrate subscription grant tasks on the consumer side 19 | c_manage_subscription_revoke_state_machine_name: str - Name to be used in all accounts' state machine that will orchestrate subscription revoke tasks on the consumer side 20 | governance: dict - Dict containing global variables for governance account related resources, including: 21 | g_account_number: str- Account id of the governance account 22 | g_common_stepfunctions_role_name: str - Name of the role for state machines (step functions) in governance account 23 | g_cross_account_assume_role: str - Name of the role in governance account that can be assumed by accounts with producer / consumer capabilities for cross-account access 24 | g_p_source_subscriptions_table_name: str - Name of the DynamoDB table in governance account that will store metadata for producer source connection subscriptions details 25 | g_c_asset_subscriptions_table_name: str - Name of the DynamoDB table in governance account that will store metadata for consumer asset subscriptions details 26 | g_c_secrets_mapping_table_name: str - Name of the DynamoDB table in governance account that will store metadata for consumer secrets mapping details 27 | 28 | g_manage_subscription_grant_state_machine_name: str - Name to be used in governance account state machine that will orchestrate the complete subscription grant 29 | g_manage_subscription_revoke_state_machine_name: str - Name to be used in governance account state machine that will orchestrate the complete subscription revoke 30 | """ 31 | GLOBAL_VARIABLES = { 32 | 'account': { 33 | 'a_account_numbers': GOVERNANCE_PROPS['a_account_numbers'], 34 | 'a_common_glue_role_name': 'dz_conn_a_common_glue_role', 35 | 'a_common_lambda_role_name': 'dz_conn_a_common_lambda_role', 36 | 'a_cross_account_assume_role_name': 'dz_conn_a_cross_account_assume_role', 37 | 'a_update_environment_roles_lambda_name': 'dz_conn_a_update_environment_roles', 38 | 'a_clean_environment_roles_lambda_name': 'dz_conn_a_clean_environment_roles' 39 | }, 40 | 'producer': { 41 | 'p_add_lf_tag_environment_dbs_lambda_name': 'dz_conn_p_add_lf_tag_environment_dbs', 42 | 43 | 'p_manage_subscription_grant_state_machine_name': 'dz_conn_p_manage_subscription_grant', 44 | 'p_manage_subscription_revoke_state_machine_name': 'dz_conn_p_manage_subscription_revoke' 45 | }, 46 | 'consumer': { 47 | 'c_manage_subscription_grant_state_machine_name': 'dz_conn_c_manage_subscription_grant', 48 | 'c_manage_subscription_revoke_state_machine_name': 'dz_conn_c_manage_subscription_revoke' 49 | }, 50 | 'governance': { 51 | 'g_account_number': GOVERNANCE_PROPS['account_id'], 52 | 'g_common_stepfunctions_role_name': 'dz_conn_g_common_stepfunctions_role', 53 | 54 | 'g_cross_account_assume_role_name': 'dz_conn_g_cross_account_assume_role', 55 | 'g_p_source_subscriptions_table_name': 'dz_conn_g_p_source_subscriptions', 56 | 'g_c_asset_subscriptions_table_name': 'dz_conn_g_c_asset_subscriptions', 57 | 'g_c_secrets_mapping_table_name': 'dz_conn_g_c_secrets_mapping', 58 | 59 | 'g_manage_subscription_grant_state_machine_name': 'dz_conn_g_manage_subscription_grant', 60 | 'g_manage_subscription_revoke_state_machine_name': 'dz_conn_g_manage_subscription_revoke' 61 | } 62 | } -------------------------------------------------------------------------------- /config/governance/g_config.py: -------------------------------------------------------------------------------- 1 | # ------------------ Governance ------------------------ 2 | 3 | """ 4 | GOVERNANCE_PROPS dict will be used to consolidate common properties for governance account. 5 | The dict structures includes: 6 | account_id: str - Account id 7 | region: str - Region 8 | a_account_numbers: list - List containing ids of all accounts with producer / consumer capabilities (governed) 9 | """ 10 | GOVERNANCE_PROPS = { 11 | 'account_id': '', 12 | 'region': '', 13 | 'a_account_numbers': [], 14 | } 15 | 16 | """ 17 | GOVERNANCE_WORKFLOW_PROPS dict will be used to consolidate governance workflow properties for governance account. 18 | The dict structures includes a key (not to be modified) per workflow: 19 | g_manage_environment_active: dict - Dict containing properties for managing when a new environment is active including: 20 | g_eventbridge_rule_enabled: bool - If workflow is enabled or not, meaning will execute on event or not. 21 | g_manage_environment_delete: dict - Dict containing properties for managing when a environment is going to be deleted including: 22 | g_eventbridge_rule_enabled: bool - If workflow is enabled or not, meaning will execute on event or not. 23 | g_manage_subscription_grant: dict - Dict containing properties for managing when a new subscription is granted including: 24 | g_eventbridge_rule_enabled: bool - If workflow is enabled or not, meaning will execute on event or not. 25 | g_manage_subscription_revoke: dict - Dict containing properties for managing when a subscription is revoked including: 26 | g_eventbridge_rule_enabled: bool - If workflow is enabled or not, meaning will execute on event or not. 27 | """ 28 | GOVERNANCE_WORKFLOW_PROPS = { 29 | 'g_manage_environment_active': { 30 | 'g_eventbridge_rule_enabled': True 31 | }, 32 | 'g_manage_environment_delete': { 33 | 'g_eventbridge_rule_enabled': True 34 | }, 35 | 'g_manage_subscription_grant': { 36 | 'g_eventbridge_rule_enabled': True 37 | }, 38 | 'g_manage_subscription_revoke': { 39 | 'g_eventbridge_rule_enabled': True 40 | } 41 | } -------------------------------------------------------------------------------- /libs/python311/boto3-layer.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-connecting-data-products-with-amazon-datazone/689fd472dd67677cedbea594d8f05e6803dee315/libs/python311/boto3-layer.zip -------------------------------------------------------------------------------- /libs/python38/oracledb-layer.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-connecting-data-products-with-amazon-datazone/689fd472dd67677cedbea594d8f05e6803dee315/libs/python38/oracledb-layer.zip -------------------------------------------------------------------------------- /libs/python38/pyodbc-layer.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-solutions-library-samples/guidance-for-connecting-data-products-with-amazon-datazone/689fd472dd67677cedbea594d8f05e6803dee315/libs/python38/pyodbc-layer.zip -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aws-cdk-lib==2.104.0 2 | cdk-nag==2.27.131 3 | constructs>=10.0.0,<11.0.0 4 | -------------------------------------------------------------------------------- /source.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem The sole purpose of this script is to make the command 4 | rem 5 | rem source .venv/bin/activate 6 | rem 7 | rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. 8 | rem On Windows, this command just runs this batch file (the argument is ignored). 9 | rem 10 | rem Now we don't need to document a Windows command for activating a virtualenv. 11 | 12 | echo Executing .venv\Scripts\activate.bat for you 13 | .venv\Scripts\activate.bat 14 | -------------------------------------------------------------------------------- /src/account/code/iam/datazone_custom_environment_permission_boundary.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Sid": "CreateGlueConnection", 6 | "Effect": "Allow", 7 | "Action": [ 8 | "ec2:CreateTags", 9 | "ec2:DeleteTags" 10 | ], 11 | "Resource": [ 12 | "arn:aws:ec2:*:*:network-interface/*" 13 | ], 14 | "Condition": { 15 | "ForAllValues:StringEquals": { 16 | "aws:TagKeys": [ 17 | "aws-glue-service-resource" 18 | ] 19 | } 20 | } 21 | }, 22 | { 23 | "Sid": "GlueOperations", 24 | "Effect": "Allow", 25 | "Action": [ 26 | "glue:*DataQuality*", 27 | "glue:*Partition*", 28 | "glue:*Connection*", 29 | "glue:*Table*", 30 | "glue:*Crawl*", 31 | "glue:*Database*", 32 | "glue:*Job*", 33 | "glue:*Workflow*", 34 | "glue:*Blueprint*", 35 | "glue:ListSchemas", 36 | "glue:NotifyEvent" 37 | ], 38 | "Resource": "*", 39 | "Condition": { 40 | "Null": { 41 | "aws:ResourceTag/AmazonDataZoneEnvironment": "false" 42 | } 43 | } 44 | }, 45 | { 46 | "Sid": "PassRole", 47 | "Effect": "Allow", 48 | "Action": [ 49 | "iam:PassRole" 50 | ], 51 | "Resource": [ 52 | "arn:aws:iam::*:role/*" 53 | ], 54 | "Condition": { 55 | "StringEquals": { 56 | "iam:PassedToService": "glue.amazonaws.com" 57 | } 58 | } 59 | }, 60 | { 61 | "Sid": "SameAccountKmsOperations", 62 | "Effect": "Allow", 63 | "Action": [ 64 | "kms:DescribeKey", 65 | "kms:Decrypt", 66 | "kms:ListKeys" 67 | ], 68 | "Resource": "*", 69 | "Condition": { 70 | "StringNotEquals": { 71 | "aws:ResourceAccount": "${aws:PrincipalAccount}" 72 | } 73 | } 74 | }, 75 | { 76 | "Sid": "KmsOperationsWithResourceTag", 77 | "Effect": "Allow", 78 | "Action": [ 79 | "kms:DescribeKey", 80 | "kms:Decrypt", 81 | "kms:ListKeys", 82 | "kms:Encrypt", 83 | "kms:GenerateDataKey", 84 | "kms:Verify", 85 | "kms:Sign" 86 | ], 87 | "Resource": "*", 88 | "Condition": { 89 | "Null": { 90 | "aws:ResourceTag/AmazonDataZoneEnvironment": "false" 91 | } 92 | } 93 | }, 94 | { 95 | "Sid": "AnalyticsOperations", 96 | "Effect": "Allow", 97 | "Action": [ 98 | "datazone:*", 99 | "sqlworkbench:*" 100 | ], 101 | "Resource": "*" 102 | }, 103 | { 104 | "Sid": "QueryOperations", 105 | "Effect": "Allow", 106 | "Action": [ 107 | "athena:*Statement*", 108 | "athena:*Quer*", 109 | "athena:*Notebook*", 110 | "athena:*Database*", 111 | "athena:*Table*", 112 | "athena:*WorkGroup*", 113 | "athena:*Catalog*", 114 | "athena:ListEngineVersions", 115 | "athena:ListTagsForResource", 116 | "athena:*Execution*", 117 | "athena:*Session*", 118 | "cloudformation:*", 119 | "config:*", 120 | "ec2:CreateNetworkInterface", 121 | "ec2:DeleteNetworkInterface", 122 | "ec2:Describe*", 123 | "glue:*DataQuality*", 124 | "glue:*Partition*", 125 | "glue:*Connection*", 126 | "glue:*Table*", 127 | "glue:*Crawl*", 128 | "glue:*Database*", 129 | "glue:*Job*", 130 | "glue:*Workflow*", 131 | "glue:*Blueprint*", 132 | "glue:ListSchemas", 133 | "glue:NotifyEvent", 134 | "iam:List*", 135 | "iam:GetRole", 136 | "iam:GetRolePolicy", 137 | "lambda:InvokeFunction", 138 | "logs:*Quer*", 139 | "logs:Describe*", 140 | "logs:Get*", 141 | "logs:PutLogEvents", 142 | "logs:CreateLogStream", 143 | "logs:FilterLogEvents", 144 | "lakeformation:Get*", 145 | "lakeformation:ListPermissions", 146 | "redshift-data:ListTables", 147 | "redshift-data:DescribeTable", 148 | "redshift-data:ListSchemas", 149 | "redshift-data:ListDatabases", 150 | "redshift-data:ExecuteStatement", 151 | "redshift-data:GetStatementResult", 152 | "redshift-data:DescribeStatement", 153 | "redshift:CreateClusterUser", 154 | "redshift:DescribeClusters", 155 | "redshift:DescribeDataShares", 156 | "redshift:GetClusterCredentials", 157 | "redshift:GetClusterCredentialsWithIAM", 158 | "redshift:JoinGroup", 159 | "redshift-serverless:ListNamespaces", 160 | "redshift-serverless:ListWorkgroups", 161 | "redshift-serverless:GetNamespace", 162 | "redshift-serverless:GetWorkgroup", 163 | "redshift-serverless:GetCredentials", 164 | "secretsmanager:ListSecrets", 165 | "secretsmanager:GetSecretValue", 166 | "secretsmanager:DescribeSecret", 167 | "servicecatalog:*", 168 | "ssm:*", 169 | "tag:GetResources" 170 | ], 171 | "Resource": "*" 172 | }, 173 | { 174 | "Sid": "SecretsManagerOperationsWithTagKeys", 175 | "Effect": "Allow", 176 | "Action": [ 177 | "secretsmanager:CreateSecret", 178 | "secretsmanager:TagResource" 179 | ], 180 | "Resource": "arn:aws:secretsmanager:*:*:secret:AmazonDataZone-*", 181 | "Condition": { 182 | "StringLike": { 183 | "aws:ResourceTag/AmazonDataZoneDomain": "*", 184 | "aws:ResourceTag/AmazonDataZoneProject": "*" 185 | }, 186 | "Null": { 187 | "aws:TagKeys": "false" 188 | }, 189 | "ForAllValues:StringEquals": { 190 | "aws:TagKeys": [ 191 | "AmazonDataZoneDomain", 192 | "AmazonDataZoneProject" 193 | ] 194 | } 195 | } 196 | }, 197 | { 198 | "Sid": "DataZoneS3Buckets", 199 | "Effect": "Allow", 200 | "Action": [ 201 | "s3:AbortMultipartUpload", 202 | "s3:*Object*" 203 | ], 204 | "Resource": [ 205 | "arn:aws:s3:::*/datazone/*" 206 | ] 207 | }, 208 | { 209 | "Sid": "DataZoneS3BucketLocation", 210 | "Effect": "Allow", 211 | "Action": [ 212 | "s3:GetBucketLocation" 213 | ], 214 | "Resource": "*" 215 | }, 216 | { 217 | "Sid": "ListDataZoneS3Bucket", 218 | "Effect": "Allow", 219 | "Action": [ 220 | "s3:ListBucket" 221 | ], 222 | "Resource": [ 223 | "*" 224 | ], 225 | "Condition": { 226 | "StringLike": { 227 | "s3:prefix": [ 228 | "*/datazone/*", 229 | "datazone/*" 230 | ] 231 | } 232 | } 233 | }, 234 | { 235 | "Sid": "NotDeniedOperations", 236 | "Effect": "Deny", 237 | "NotAction": [ 238 | "datazone:*", 239 | "sqlworkbench:*", 240 | "athena:*Statement*", 241 | "athena:*Quer*", 242 | "athena:*Notebook*", 243 | "athena:*Database*", 244 | "athena:*Table*", 245 | "athena:*WorkGroup*", 246 | "athena:*Catalog*", 247 | "athena:ListEngineVersions", 248 | "athena:ListTagsForResource", 249 | "athena:*Execution*", 250 | "athena:*Session*", 251 | "ec2:CreateNetworkInterface", 252 | "cloudformation:*", 253 | "config:*", 254 | "ec2:CreateTags", 255 | "ec2:DeleteNetworkInterface", 256 | "ec2:DeleteTags", 257 | "ec2:Describe*", 258 | "glue:*DataQuality*", 259 | "glue:*Partition*", 260 | "glue:*Connection*", 261 | "glue:*Table*", 262 | "glue:*Crawl*", 263 | "glue:*Database*", 264 | "glue:*Job*", 265 | "glue:*Workflow*", 266 | "glue:*Blueprint*", 267 | "glue:ListSchemas", 268 | "glue:NotifyEvent", 269 | "iam:List*", 270 | "iam:GetRole", 271 | "iam:GetRolePolicy", 272 | "iam:PassRole", 273 | "kms:DescribeKey", 274 | "kms:Decrypt", 275 | "kms:Encrypt", 276 | "kms:GenerateDataKey", 277 | "kms:ListKeys", 278 | "kms:Verify", 279 | "kms:Sign", 280 | "lambda:InvokeFunction", 281 | "logs:*Quer*", 282 | "logs:Describe*", 283 | "logs:Get*", 284 | "logs:PutLogEvents", 285 | "logs:CreateLogStream", 286 | "logs:FilterLogEvents", 287 | "lakeformation:Get*", 288 | "lakeformation:ListPermissions", 289 | "redshift-data:ListTables", 290 | "redshift-data:DescribeTable", 291 | "redshift-data:ListSchemas", 292 | "redshift-data:ListDatabases", 293 | "redshift-data:ExecuteStatement", 294 | "redshift-data:GetStatementResult", 295 | "redshift-data:DescribeStatement", 296 | "redshift:CreateClusterUser", 297 | "redshift:DescribeClusters", 298 | "redshift:DescribeDataShares", 299 | "redshift:GetClusterCredentials", 300 | "redshift:GetClusterCredentialsWithIAM", 301 | "redshift:JoinGroup", 302 | "redshift-serverless:ListNamespaces", 303 | "redshift-serverless:ListWorkgroups", 304 | "redshift-serverless:GetNamespace", 305 | "redshift-serverless:GetWorkgroup", 306 | "redshift-serverless:GetCredentials", 307 | "s3:AbortMultipartUpload", 308 | "s3:DeleteObject", 309 | "s3:DeleteObjectVersion", 310 | "s3:GetObject", 311 | "s3:GetBucketLocation", 312 | "s3:ListBucket", 313 | "s3:PutObject", 314 | "s3:PutObjectRetention", 315 | "s3:ReplicateObject", 316 | "s3:RestoreObject", 317 | "secretsmanager:CreateSecret", 318 | "secretsmanager:ListSecrets", 319 | "secretsmanager:TagResource", 320 | "secretsmanager:GetSecretValue", 321 | "secretsmanager:DescribeSecret", 322 | "servicecatalog:*", 323 | "ssm:*", 324 | "tag:GetResources" 325 | ], 326 | "Resource": [ 327 | "*" 328 | ] 329 | } 330 | ] 331 | } -------------------------------------------------------------------------------- /src/account/code/lambda/clean_environment_roles/clean_environment_roles.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import boto3 4 | 5 | # Constant: Represents the ARN of the solution's IAM policy to attach to environment user roles 6 | A_ENVIRONMENT_POLICY_ARN = os.getenv('A_ENVIRONMENT_POLICY_ARN') 7 | 8 | # Constant: Represents the account id 9 | A_ACCOUNT_ID = os.getenv('A_ACCOUNT_ID') 10 | 11 | # Constant: Represents the arn of the default DataZone permission boundary to restore on environment roles 12 | DEFAULT_DATAZONE_PERMISSION_BOUNDARY_ARN = 'arn:aws:iam::aws:policy/AmazonDataZoneEnvironmentRolePermissionsBoundary' 13 | 14 | # Constant: Represents the arn of the AWS managed policy that allows usage of service catalog and will be removed from environment roles 15 | SERVICE_CATALOG_POLICY_ARN = 'arn:aws:iam::aws:policy/AWSServiceCatalogEndUserFullAccess' 16 | 17 | iam = boto3.client('iam') 18 | 19 | def handler(event, context): 20 | """ Function handler: Function that will clean environment roles by 1/ Replacing permission boundary with default one assigned by Amazon DataZone, 21 | 2/ Remove managed policy that extended environment role permissions and 3/ Remove additional attached policies that were not assigned originally by Amazon DataZone 22 | 23 | Parameters 24 | ---------- 25 | event: dict - Input event dict containing: 26 | EnvironmentDetails: dict - Dict containing environment details including: 27 | DomainId: str - Id of DataZone domain 28 | ProjectId: str - Id of DataZone project 29 | EnvironmentId: str - Id of DataZone environment 30 | EnvironmentResources: dict - Dict containing environment associated resources 31 | 32 | context: dict - Input context. Not used on function 33 | 34 | Returns 35 | ------- 36 | response: dict - Dict with response details including: 37 | environment_role_arn: str - Cleaned environment role arn (str) 38 | """ 39 | 40 | environment_details = event['EnvironmentDetails'] 41 | domain_id = environment_details['DomainId'] 42 | project_id = environment_details['ProjectId'] 43 | environment_id = environment_details['EnvironmentId'] 44 | environment_resources = environment_details['EnvironmentResources'] 45 | 46 | environment_role_arn = environment_resources['userRoleArn'] 47 | environment_role_name = environment_role_arn.split('/')[1] 48 | 49 | iam.put_role_permissions_boundary( 50 | RoleName= environment_role_name, 51 | PermissionsBoundary= DEFAULT_DATAZONE_PERMISSION_BOUNDARY_ARN 52 | ) 53 | 54 | try: 55 | iam.detach_role_policy( 56 | RoleName= environment_role_name, 57 | PolicyArn= A_ENVIRONMENT_POLICY_ARN 58 | ) 59 | 60 | iam.detach_role_policy( 61 | RoleName= environment_role_name, 62 | PolicyArn= SERVICE_CATALOG_POLICY_ARN 63 | ) 64 | 65 | except: pass 66 | 67 | response = { 68 | 'environment_role_arn': environment_role_arn 69 | } 70 | 71 | return response 72 | 73 | -------------------------------------------------------------------------------- /src/account/code/lambda/manage_service_portfolio_environment_roles_access/manage_service_portfolio_environment_roles_access.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | 4 | import boto3 5 | 6 | # Constant: Represents the service portfolio id that will be shared to DataZone project roles 7 | A_SERVICE_PORTFOLIO_ID = os.getenv('A_SERVICE_PORTFOLIO_ID') 8 | 9 | # Constant: Represents the resource id to be returned to CDK on request. 10 | PHYSICAL_RESOURCE_ID = 'dz-conn-a-project-roles-access' 11 | 12 | # Constant: Represents a pattern string that identifies any DataZone project role. 13 | PROJECT_ROLES_ARN_PATTERN = 'arn:aws:iam:::role/datazone_usr_*' 14 | 15 | servicecatalog = boto3.client('servicecatalog') 16 | 17 | def handler(event, context): 18 | """ Function handler: Function that will be triggered by CDK when deploying or deleting account common stack. 19 | Function will associate or disassociate all DataZone project roles from account service catalog portfolio. 20 | 21 | Parameters 22 | ---------- 23 | event: dict - Input event dict containing: 24 | RequestType: str - Type of event invoking the function from CDK 25 | 26 | context: dict - Input context. Not used on function 27 | 28 | Returns 29 | ------- 30 | response: dict - Dict with response details including: 31 | PhysicalResourceId: str - Physical resource id required by CDK 32 | """ 33 | 34 | request_type = event['RequestType'].lower() 35 | 36 | if request_type == 'create': 37 | return on_update(event) 38 | 39 | if request_type == 'update': 40 | return on_update(event) 41 | 42 | if request_type == 'delete': 43 | return on_delete(event) 44 | 45 | raise Exception(f'Invalid request type: {request_type}') 46 | 47 | 48 | def on_update(event): 49 | """ Complementary function to associate (on CDK update event) all DataZone projects roles to service catalog portfolio""" 50 | 51 | servicecatalog_response = servicecatalog.associate_principal_with_portfolio( 52 | PortfolioId= A_SERVICE_PORTFOLIO_ID, 53 | PrincipalARN= PROJECT_ROLES_ARN_PATTERN, 54 | PrincipalType= 'IAM_PATTERN' 55 | ) 56 | 57 | print(json.dumps(servicecatalog_response, indent=2)) 58 | 59 | return {'PhysicalResourceId': PHYSICAL_RESOURCE_ID} 60 | 61 | 62 | def on_delete(event): 63 | """ Complementary function to disassociate (on CDK delete event) all DataZone projects roles from service catalog portfolio""" 64 | 65 | servicecatalog_response = servicecatalog.disassociate_principal_from_portfolio( 66 | PortfolioId= A_SERVICE_PORTFOLIO_ID, 67 | PrincipalARN= PROJECT_ROLES_ARN_PATTERN, 68 | PrincipalType= 'IAM_PATTERN' 69 | ) 70 | 71 | print(json.dumps(servicecatalog_response, indent=2)) 72 | 73 | return {'PhysicalResourceId': PHYSICAL_RESOURCE_ID} 74 | -------------------------------------------------------------------------------- /src/account/code/lambda/update_environment_roles/update_environment_roles.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | 4 | import boto3 5 | 6 | # Constant: Represents the ARN of the solution's IAM policy to attach to environment user roles 7 | A_ENVIRONMENT_POLICY_ARN = os.getenv('A_ENVIRONMENT_POLICY_ARN') 8 | 9 | # Constant: Represents the arn of the custom permission boundary that will be assigned to environment roles 10 | A_PERMISSION_BOUNDARY_POLICY_ARN = os.getenv('A_PERMISSION_BOUNDARY_POLICY_ARN') 11 | 12 | # Constant: Represents the region 13 | A_REGION = os.getenv('A_REGION') 14 | 15 | # Constant: Represents the account id 16 | A_ACCOUNT_ID = os.getenv('A_ACCOUNT_ID') 17 | 18 | # Constant: Represents the arn of the AWS managed policy that allows usage of service catalog and will be added to environment roles 19 | SERVICE_CATALOG_POLICY_ARN = 'arn:aws:iam::aws:policy/AWSServiceCatalogEndUserFullAccess' 20 | 21 | iam = boto3.client('iam') 22 | 23 | def handler(event, context): 24 | """ Function handler: Function that will update environment roles by 1/ Replacing default permission boundary with a custom more permissive one, 25 | 2/ Attach solution's managed policy that extends environment role permissions and 3/ Attach additional policies that were not assigned originally by Amazon DataZone 26 | 27 | Parameters 28 | ---------- 29 | event: dict - Input event dict containing: 30 | EnvironmentDetails: dict - Dict containing environment details including: 31 | DomainId: str - Id of DataZone domain 32 | ProjectId: str - Id of DataZone project 33 | EnvironmentId: str - Id of DataZone environment 34 | EnvironmentResources: dict - Dict containing environment associated resources 35 | 36 | context: dict - Input context. Not used on function 37 | 38 | Returns 39 | ------- 40 | response: dict - Dict with response details including: 41 | environment_role_arn: str - Updated environment role arn (str) 42 | """ 43 | 44 | environment_details = event['EnvironmentDetails'] 45 | domain_id = environment_details['DomainId'] 46 | project_id = environment_details['ProjectId'] 47 | environment_id = environment_details['EnvironmentId'] 48 | environment_resources = environment_details['EnvironmentResources'] 49 | 50 | environment_role_arn = environment_resources['userRoleArn'] 51 | environment_role_name = environment_role_arn.split('/')[1] 52 | 53 | 54 | iam.put_role_permissions_boundary( 55 | RoleName= environment_role_name, 56 | PermissionsBoundary= A_PERMISSION_BOUNDARY_POLICY_ARN 57 | ) 58 | 59 | iam.attach_role_policy( 60 | RoleName= environment_role_name, 61 | PolicyArn= SERVICE_CATALOG_POLICY_ARN 62 | ) 63 | 64 | iam.attach_role_policy( 65 | RoleName= environment_role_name, 66 | PolicyArn= A_ENVIRONMENT_POLICY_ARN 67 | ) 68 | 69 | response = { 70 | 'environment_role_arn': environment_role_arn 71 | } 72 | 73 | return response 74 | 75 | -------------------------------------------------------------------------------- /src/consumer/code/lambda/copy_subscription_secret/copy_subscription_secret.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import uuid 4 | from datetime import datetime 5 | 6 | import boto3 7 | from boto3.dynamodb.types import TypeSerializer 8 | 9 | # Constant: Represents the governance account id 10 | G_ACCOUNT_ID = os.getenv('G_ACCOUNT_ID') 11 | 12 | # Constant: Represents the governance cross account role name to be used when updating metadata in DynamoDB 13 | G_CROSS_ACCOUNT_ASSUME_ROLE_NAME = os.getenv('G_CROSS_ACCOUNT_ASSUME_ROLE_NAME') 14 | 15 | # Constant: Represents the governance DynamoDB table to map producer and consumer secrets 16 | G_C_SECRETS_MAPPING_TABLE_NAME = os.getenv('G_C_SECRETS_MAPPING_TABLE_NAME') 17 | 18 | # Constant: Represents the alias of the account common kms key 19 | A_COMMON_KEY_ALIAS = os.getenv('A_COMMON_KEY_ALIAS') 20 | 21 | # Constant: Represents the account id 22 | ACCOUNT_ID = os.getenv('ACCOUNT_ID') 23 | 24 | # Constant: Represents the region 25 | REGION = os.getenv('REGION') 26 | 27 | g_cross_account_role_arn = f'arn:aws:iam::{G_ACCOUNT_ID}:role/{G_CROSS_ACCOUNT_ASSUME_ROLE_NAME}' 28 | 29 | kms = boto3.client('kms') 30 | secrets_manager = boto3.client('secretsmanager') 31 | sts = boto3.client('sts') 32 | sts_session = sts.assume_role(RoleArn=g_cross_account_role_arn, RoleSessionName='c-dynamodb-session') 33 | 34 | session_key_id = sts_session['Credentials']['AccessKeyId'] 35 | session_access_key = sts_session['Credentials']['SecretAccessKey'] 36 | session_token = sts_session['Credentials']['SessionToken'] 37 | 38 | dynamodb = boto3.client('dynamodb', aws_access_key_id=session_key_id, aws_secret_access_key=session_access_key, aws_session_token=session_token) 39 | dynamodb_serializer = TypeSerializer() 40 | 41 | def handler(event, context): 42 | """ Function handler: Function that will copy a subscription secret by 1/ Retrieving producer shared secret and copying its content into a new one local to the consumer account and 43 | 2/ Updating metadata in governance DynamoDB table that maps producer and consumer secrets. 44 | 45 | Parameters 46 | ---------- 47 | event: dict - Input event dict containing: 48 | SubscriptionDetails: dict - Dict containing details including: 49 | ConsumerProjectDetails: dict - Dict containing consumer project details including: 50 | ProjectId: str - DataZone consumer project id 51 | EnvironmentId: str - DataZone consumer environment id 52 | ProducerGrantDetails: dict - Dict containing producer grant details including: 53 | SecretArn: str - Arn of producer shared subscription secret. 54 | 55 | context: dict - Input context. Not used on function 56 | 57 | Returns 58 | ------- 59 | response: dict - Dict with response details including: 60 | secret_arn: str - Arn of the copied secret local to the consumer account 61 | secret_name: str - Name of the copied secret local to the consumer account 62 | """ 63 | subscription_details = event['SubscriptionDetails'] 64 | producer_grant_details = event['ProducerGrantDetails'] 65 | 66 | domain_id = subscription_details['DomainId'] 67 | consumer_project_details = subscription_details['ConsumerProjectDetails'] 68 | 69 | consumer_project_id = consumer_project_details['ProjectId'] 70 | consumer_environment_id = consumer_project_details['EnvironmentId'] 71 | shared_subscription_secret_arn = producer_grant_details['SecretArn'] 72 | 73 | secrets_manager_response = secrets_manager.get_secret_value( 74 | SecretId= shared_subscription_secret_arn 75 | ) 76 | 77 | shared_subscription_secret_value = json.loads(secrets_manager_response['SecretString']) 78 | 79 | project_secret_name_suffix = str(uuid.uuid4()).replace('-', '') 80 | project_secret_name = f'dz-conn-c-{consumer_project_id}-{consumer_environment_id}-{project_secret_name_suffix}' 81 | 82 | secrets_manager_response = create_secret(project_secret_name, shared_subscription_secret_value, consumer_environment_id, consumer_project_id, domain_id) 83 | project_secret_name = secrets_manager_response['Name'] 84 | project_secret_arn = secrets_manager_response['ARN'] 85 | 86 | update_secret_association_item(shared_subscription_secret_arn, project_secret_arn, project_secret_name, consumer_environment_id, consumer_project_id, domain_id) 87 | 88 | response = { 89 | 'secret_arn': project_secret_name, 90 | 'secret_name': project_secret_arn 91 | } 92 | 93 | return response 94 | 95 | 96 | def create_secret(secret_name, secret_value, environment_id, project_id, domain_id ): 97 | """ Complementary function to create a new secret local to the consumer account and associated to subscribing project""" 98 | 99 | kms_response = kms.describe_key( 100 | KeyId= f'alias/{A_COMMON_KEY_ALIAS}' 101 | ) 102 | 103 | kms_key_arn = kms_response['KeyMetadata']['Arn'] 104 | 105 | secrets_manager_response = secrets_manager.create_secret( 106 | Name=secret_name, 107 | KmsKeyId=kms_key_arn, 108 | SecretString=json.dumps(secret_value), 109 | Tags=[ 110 | { 111 | 'Key': 'AmazonDataZoneEnvironment', 112 | 'Value': environment_id 113 | }, 114 | { 115 | 'Key': 'AmazonDataZoneProject', 116 | 'Value': project_id 117 | }, 118 | { 119 | 'Key': 'AmazonDataZoneDomain', 120 | 'Value': domain_id 121 | } 122 | ] 123 | ) 124 | 125 | secrets_manager_response = json.loads(json.dumps(secrets_manager_response, default=json_datetime_encoder)) 126 | 127 | return secrets_manager_response 128 | 129 | 130 | def update_secret_association_item(shared_secret_arn, secret_arn, secret_name, environment_id, project_id, domain_id): 131 | """ Complementary function to update item with secret mapping details in respective governance DynamoDB table""" 132 | 133 | secret_association_item = { 134 | 'shared_secret_arn': shared_secret_arn, 135 | 'secret_arn': secret_arn, 136 | 'secret_name': secret_name, 137 | 'datazone_consumer_environment_id': environment_id, 138 | 'datazone_consumer_project_id': project_id, 139 | 'datazone_domain': domain_id, 140 | 'owner_account': ACCOUNT_ID, 141 | 'owner_region': REGION, 142 | 'last_updated': datetime.now().strftime("%Y-%m-%dT%H:%M:%S") 143 | } 144 | 145 | dynamodb_response = dynamodb.put_item( 146 | TableName=G_C_SECRETS_MAPPING_TABLE_NAME, 147 | Item={key: dynamodb_serializer.serialize(value) for key, value in secret_association_item.items()} 148 | ) 149 | 150 | return secret_association_item 151 | 152 | 153 | def json_datetime_encoder(obj): 154 | """ Complementary function to transform dict objects delivered by AWS API into JSONs """ 155 | if isinstance(obj, (datetime)): return obj.strftime("%Y-%m-%dT%H:%M:%S") 156 | -------------------------------------------------------------------------------- /src/consumer/code/lambda/delete_subscription_secret/delete_subscription_secret.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from datetime import datetime 4 | 5 | import boto3 6 | from boto3.dynamodb.types import TypeSerializer, TypeDeserializer 7 | 8 | # Constant: Represents the governance account id 9 | G_ACCOUNT_ID = os.getenv('G_ACCOUNT_ID') 10 | 11 | # Constant: Represents the governance cross account role name to be used when updating metadata in DynamoDB 12 | G_CROSS_ACCOUNT_ASSUME_ROLE_NAME = os.getenv('G_CROSS_ACCOUNT_ASSUME_ROLE_NAME') 13 | 14 | # Constant: Represents the governance DynamoDB table to map producer and consumer secrets 15 | G_C_SECRETS_MAPPING_TABLE_NAME = os.getenv('G_C_SECRETS_MAPPING_TABLE_NAME') 16 | 17 | # Constant: Represents the recovery window in days that will be assigned when scheduling secret deletion 18 | RECOVERY_WINDOW_IN_DAYS = os.getenv('RECOVERY_WINDOW_IN_DAYS') 19 | 20 | g_cross_account_role_arn = f'arn:aws:iam::{G_ACCOUNT_ID}:role/{G_CROSS_ACCOUNT_ASSUME_ROLE_NAME}' 21 | 22 | secrets_manager = boto3.client('secretsmanager') 23 | sts = boto3.client('sts') 24 | sts_session = sts.assume_role(RoleArn=g_cross_account_role_arn, RoleSessionName='p-dynamodb-session') 25 | 26 | session_key_id = sts_session['Credentials']['AccessKeyId'] 27 | session_access_key = sts_session['Credentials']['SecretAccessKey'] 28 | session_token = sts_session['Credentials']['SessionToken'] 29 | 30 | dynamodb = boto3.client('dynamodb', aws_access_key_id=session_key_id, aws_secret_access_key=session_access_key, aws_session_token=session_token) 31 | dynamodb_serializer = TypeSerializer() 32 | dynamodb_deserializer = TypeDeserializer() 33 | 34 | def handler(event, context): 35 | """ Function handler: Function that will delete a subscription secret by 1/ Scheduling its deletion and 36 | 2/ Deleting metadata item in the governance DynamoDB table that maps associated producer and consumer secrets. 37 | 38 | Parameters 39 | ---------- 40 | event: dict - Input event dict containing: 41 | ProducerRevokeDetails: dict - Dict containing producer subscription revoke details including: 42 | SecretArn: str - Arn of producer shared subscription secret associated to the local consumer (to be deleted) secret. 43 | 44 | context: dict - Input context. Not used on function 45 | 46 | Returns 47 | ------- 48 | response: dict - Dict with response details including: 49 | secret_name: str - Name of the deleted secret local to the consumer account 50 | secret_arn: str - Arn of the deleted secret local to the consumer account 51 | secret_deleted: str - 'True' flag to confirm secret deletion 52 | secret_deletion_date: str - Date of secret deletion 53 | secret_recovery_window_in_days: str - Secret recovery window in days 54 | """ 55 | producer_revoke_details = event['ProducerRevokeDetails'] 56 | 57 | shared_secret_arn = producer_revoke_details['SecretArn'] 58 | secret_association_item = get_secret_association_item(shared_secret_arn) 59 | 60 | secret_name = secret_association_item['secret_name'] 61 | 62 | secrets_manager_response = delete_secret(secret_name) 63 | delete_secret_association_item(shared_secret_arn) 64 | 65 | response = { 66 | 'secret_name': secrets_manager_response['Name'], 67 | 'secret_arn': secrets_manager_response['ARN'], 68 | 'secret_deleted': 'true', 69 | 'secret_deletion_date': secrets_manager_response['DeletionDate'], 70 | 'secret_recovery_window_in_days': RECOVERY_WINDOW_IN_DAYS 71 | } 72 | 73 | return response 74 | 75 | 76 | def get_secret_association_item(shared_secret_arn): 77 | """ Complementary function to get item with secret mapping details in respective governance DynamoDB table""" 78 | 79 | dynamodb_response = dynamodb.get_item( 80 | TableName=G_C_SECRETS_MAPPING_TABLE_NAME, 81 | Key={ 'shared_secret_arn': dynamodb_serializer.serialize(shared_secret_arn) } 82 | ) 83 | 84 | secret_association_item = {key: dynamodb_deserializer.deserialize(value) for key, value in dynamodb_response['Item'].items()} 85 | 86 | return secret_association_item 87 | 88 | 89 | def delete_secret_association_item(shared_secret_arn): 90 | """ Complementary function to delete item with secret mapping details in respective governance DynamoDB table""" 91 | 92 | dynamodb_response = dynamodb.delete_item( 93 | TableName=G_C_SECRETS_MAPPING_TABLE_NAME, 94 | Key={ 'shared_secret_arn': dynamodb_serializer.serialize(shared_secret_arn) } 95 | ) 96 | 97 | 98 | def delete_secret(secret_name): 99 | """ Complementary function to schedule deletion of a local secret""" 100 | 101 | secrets_manager_response = secrets_manager.delete_secret( 102 | SecretId=secret_name, 103 | RecoveryWindowInDays= int(RECOVERY_WINDOW_IN_DAYS) 104 | ) 105 | 106 | secrets_manager_response = json.loads(json.dumps(secrets_manager_response, default=json_datetime_encoder)) 107 | 108 | return secrets_manager_response 109 | 110 | 111 | def json_datetime_encoder(obj): 112 | """ Complementary function to transform dict objects delivered by AWS API into JSONs """ 113 | if isinstance(obj, (datetime)): return obj.strftime("%Y-%m-%dT%H:%M:%S") -------------------------------------------------------------------------------- /src/consumer/code/lambda/remove_subscription_records/remove_subscription_records.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | 4 | import boto3 5 | from boto3.dynamodb.types import TypeDeserializer, TypeSerializer 6 | 7 | # Constant: Represents the governance account id 8 | G_ACCOUNT_ID = os.getenv('G_ACCOUNT_ID') 9 | 10 | # Constant: Represents the governance cross account role name to be used when updating metadata in DynamoDB 11 | G_CROSS_ACCOUNT_ASSUME_ROLE_NAME = os.getenv('G_CROSS_ACCOUNT_ASSUME_ROLE_NAME') 12 | 13 | # Constant: Represents the governance DynamoDB table to track consumer subscriptions (assets) 14 | G_C_ASSET_SUBSCRIPTIONS_TABLE_NAME = os.getenv('G_C_ASSET_SUBSCRIPTIONS_TABLE_NAME') 15 | 16 | g_cross_account_role_arn = f'arn:aws:iam::{G_ACCOUNT_ID}:role/{G_CROSS_ACCOUNT_ASSUME_ROLE_NAME}' 17 | 18 | sts = boto3.client('sts') 19 | sts_session = sts.assume_role(RoleArn=g_cross_account_role_arn, RoleSessionName='p-dynamodb-session') 20 | 21 | session_key_id = sts_session['Credentials']['AccessKeyId'] 22 | session_access_key = sts_session['Credentials']['SecretAccessKey'] 23 | session_token = sts_session['Credentials']['SessionToken'] 24 | 25 | dynamodb = boto3.client('dynamodb', aws_access_key_id=session_key_id, aws_secret_access_key=session_access_key, aws_session_token=session_token) 26 | dynamodb_deserializer = TypeDeserializer() 27 | dynamodb_serializer = TypeSerializer() 28 | 29 | def handler(event, context): 30 | """ Function handler: Function that will delete subscription asset metadata in governance DynamoDB table 31 | 32 | Parameters 33 | ---------- 34 | event: dict - Input event dict containing: 35 | SubscriptionDetails: dict - Dict containing subscription details including: 36 | ConsumerProjectDetails: dict - Dict containing consumer details including: 37 | EnvironmentId: str - Id of the DataZone consumer environment 38 | AssetDetails: dict - Dict containing asset details including: 39 | Id: str - Id of the data asset that consumer is subscribing to 40 | 41 | context: dict - Input context. Not used on function 42 | 43 | Returns 44 | ------- 45 | response: dict - Dict with response details: 46 | datazone_consumer_environment_id: str - Id of DataZone environment that subscribed to the asset 47 | datazone_consumer_project_id: str - Id of DataZone project that subscribed to the asset 48 | datazone_domain_id: str - Id of DataZone domain 49 | datazone_asset_id: str - Id of the asset that the consumer subscribed to. 50 | datazone_asset_revision: str - Revision of the asset that the consumer subscribed to. 51 | datazone_asset_type: str - Type of the asset that the consumer subscribed to. 52 | datazone_listing_id: str - Id of the listing associated to the asset that the consumer subscribed to. 53 | datazone_listing_revision: str - Revision of the listing associated to the asset that the consumer subscribed to. 54 | datazone_listing_name: str - Name of the listing associated to the asset that the consumer subscribed to. 55 | secret_arn: str - ARN of the secret (local to the consumer account) that can be used to access the subscribed asset 56 | secret_name: str - Name of the secret (local to the consumer account) that can be used to access the subscribed asset 57 | owner_account: str - Id of the account that owns the item 58 | owner_region: str - Region that owns the item 59 | last_updated: str - Datetime of last update performed on the item 60 | """ 61 | subscription_details = event['SubscriptionDetails'] 62 | 63 | consumer_project_details = subscription_details['ConsumerProjectDetails'] 64 | consumer_asset_details = subscription_details['AssetDetails'] 65 | 66 | environment_id = consumer_project_details['EnvironmentId'] 67 | asset_id = consumer_asset_details['Id'] 68 | 69 | asset_subscription_item = get_asset_subscription_item(environment_id, asset_id) 70 | delete_asset_subscription_item(environment_id, asset_id) 71 | 72 | return asset_subscription_item 73 | 74 | 75 | def get_asset_subscription_item(environment_id, asset_id): 76 | """ Complementary function to get item with asset subscription details from respective governance DynamoDB table """ 77 | 78 | dynamodb_response = dynamodb.get_item( 79 | TableName=G_C_ASSET_SUBSCRIPTIONS_TABLE_NAME, 80 | Key={ 81 | 'datazone_consumer_environment_id': dynamodb_serializer.serialize(environment_id), 82 | 'datazone_asset_id': dynamodb_serializer.serialize(asset_id) 83 | } 84 | ) 85 | 86 | asset_subscription_item = {key: dynamodb_deserializer.deserialize(value) for key, value in dynamodb_response['Item'].items()} 87 | 88 | return asset_subscription_item 89 | 90 | 91 | def delete_asset_subscription_item(environment_id, asset_id): 92 | """ Complementary function to delete item with asset subscription details in respective governance DynamoDB table""" 93 | 94 | dynamodb_response = dynamodb.delete_item( 95 | TableName=G_C_ASSET_SUBSCRIPTIONS_TABLE_NAME, 96 | Key={ 97 | 'datazone_consumer_environment_id': dynamodb_serializer.serialize(environment_id), 98 | 'datazone_asset_id': dynamodb_serializer.serialize(asset_id) 99 | } 100 | ) 101 | 102 | 103 | def json_datetime_encoder(obj): 104 | """ Complementary function to transform dict objects delivered by AWS API into JSONs """ 105 | if isinstance(obj, (datetime)): return obj.strftime("%Y-%m-%dT%H:%M:%S") -------------------------------------------------------------------------------- /src/consumer/code/lambda/update_subscription_records/update_subscription_records.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | 4 | import boto3 5 | from boto3.dynamodb.types import TypeDeserializer, TypeSerializer 6 | 7 | # Constant: Represents the governance account id 8 | G_ACCOUNT_ID = os.getenv('G_ACCOUNT_ID') 9 | 10 | # Constant: Represents the governance cross account role name to be used when updating metadata in DynamoDB 11 | G_CROSS_ACCOUNT_ASSUME_ROLE_NAME = os.getenv('G_CROSS_ACCOUNT_ASSUME_ROLE_NAME') 12 | 13 | # Constant: Represents the governance DynamoDB table to map producer and consumer secrets 14 | G_C_SECRETS_MAPPING_TABLE_NAME = os.getenv('G_C_SECRETS_MAPPING_TABLE_NAME') 15 | 16 | # Constant: Represents the governance DynamoDB table to track consumer subscriptions (assets) 17 | G_C_ASSET_SUBSCRIPTIONS_TABLE_NAME = os.getenv('G_C_ASSET_SUBSCRIPTIONS_TABLE_NAME') 18 | 19 | # Constant: Represents the account id 20 | ACCOUNT_ID = os.getenv('ACCOUNT_ID') 21 | 22 | # Constant: Represents the region 23 | REGION = os.getenv('REGION') 24 | 25 | g_cross_account_role_arn = f'arn:aws:iam::{G_ACCOUNT_ID}:role/{G_CROSS_ACCOUNT_ASSUME_ROLE_NAME}' 26 | 27 | sts = boto3.client('sts') 28 | sts_session = sts.assume_role(RoleArn=g_cross_account_role_arn, RoleSessionName='p-dynamodb-session') 29 | 30 | session_key_id = sts_session['Credentials']['AccessKeyId'] 31 | session_access_key = sts_session['Credentials']['SecretAccessKey'] 32 | session_token = sts_session['Credentials']['SessionToken'] 33 | 34 | dynamodb = boto3.client('dynamodb', aws_access_key_id=session_key_id, aws_secret_access_key=session_access_key, aws_session_token=session_token) 35 | dynamodb_deserializer = TypeDeserializer() 36 | dynamodb_serializer = TypeSerializer() 37 | 38 | def handler(event, context): 39 | """ Function handler: Function that will update subscription asset metadata in governance DynamoDB table 40 | 41 | Parameters 42 | ---------- 43 | event: dict - Input event dict containing: 44 | SubscriptionDetails: dict - Dict containing subscription details including: 45 | DomainId: str - DataZone domain id 46 | ConsumerProjectDetails: dict - Dict containing consumer details including: 47 | ProjectId: str - DataZone consumer project id 48 | EnvironmentId: str - DataZone consumer environment id 49 | AssetDetails: dict - Dict containing asset details including: 50 | Id: str - Id of the data asset that consumer is subscribing to 51 | Revision: str - Revision of the data asset that consumer is subscribing to 52 | Type: str - Type of the data asset that consumer is subscribing to 53 | ListingDetails: dict - Dict containing listing details including: 54 | Id: str - Id of the listing associated to the data asset that consumer is subscribing to 55 | Revision: str - Revision of the listing associated to the data asset that consumer is subscribing to 56 | Name: str - Name of the listing associated to the data asset that consumer is subscribing to 57 | ProducerGrantDetails: dict - Dict containing producer grant details including: 58 | SecretArn: str - Arn of producer shared subscription secret. 59 | 60 | context: dict - Input context. Not used on function 61 | 62 | Returns 63 | ------- 64 | asset_subscription_item: dict - Dict with asset subscription item details: 65 | datazone_consumer_environment_id: str - Id of DataZone environment that subscribed to the asset 66 | datazone_consumer_project_id: str - Id of DataZone project that subscribed to the asset 67 | datazone_domain_id: str - Id of DataZone domain 68 | datazone_asset_id: str - Id of the asset that the consumer subscribed to. 69 | datazone_asset_revision: str - Revision of the asset that the consumer subscribed to. 70 | datazone_asset_type: str - Type of the asset that the consumer subscribed to. 71 | datazone_listing_id: str - Id of the listing associated to the asset that the consumer subscribed to. 72 | datazone_listing_revision: str - Revision of the listing associated to the asset that the consumer subscribed to. 73 | datazone_listing_name: str - Name of the listing associated to the asset that the consumer subscribed to. 74 | secret_arn: str - ARN of the secret (local to the consumer account) that can be used to access the subscribed asset 75 | secret_name: str - Name of the secret (local to the consumer account) that can be used to access the subscribed asset 76 | owner_account: str - Id of the account that owns the item 77 | owner_region: str - Region that owns the item 78 | last_updated: str - Datetime of last update performed on the item 79 | """ 80 | subscription_details = event['SubscriptionDetails'] 81 | producer_grant_details = event['ProducerGrantDetails'] 82 | 83 | domain_id = subscription_details['DomainId'] 84 | consumer_project_details = subscription_details['ConsumerProjectDetails'] 85 | asset_details = subscription_details['AssetDetails'] 86 | listing_details = subscription_details['ListingDetails'] 87 | 88 | consumer_environment_id = consumer_project_details['EnvironmentId'] 89 | consumer_project_id = consumer_project_details['ProjectId'] 90 | asset_id = asset_details['Id'] 91 | asset_revision = asset_details['Revision'] 92 | asset_type = asset_details['Type'] 93 | listing_id = listing_details['Id'] 94 | listing_revision = listing_details['Revision'] 95 | listing_name = listing_details['Name'] 96 | 97 | shared_secret_arn = producer_grant_details['SecretArn'] 98 | secret_association_item = get_secret_association_item(shared_secret_arn) 99 | secret_arn = secret_association_item['secret_arn'] 100 | secret_name = secret_association_item['secret_name'] 101 | 102 | asset_subscription_item = update_asset_subscription_item( 103 | consumer_environment_id, consumer_project_id, domain_id, asset_id, asset_revision, asset_type, 104 | listing_id, listing_revision, listing_name, secret_arn, secret_name 105 | ) 106 | 107 | return asset_subscription_item 108 | 109 | 110 | def get_secret_association_item(shared_secret_arn): 111 | """ Complementary function to get item with secret mapping details in respective governance DynamoDB table""" 112 | 113 | dynamodb_response = dynamodb.get_item( 114 | TableName= G_C_SECRETS_MAPPING_TABLE_NAME, 115 | Key= { 'shared_secret_arn': dynamodb_serializer.serialize(shared_secret_arn) } 116 | ) 117 | 118 | secret_association_item = None 119 | if 'Item' in dynamodb_response: 120 | secret_association_item = {key: dynamodb_deserializer.deserialize(value) for key, value in dynamodb_response['Item'].items()} 121 | 122 | return secret_association_item 123 | 124 | 125 | def update_asset_subscription_item(environment_id, project_id, domain_id, asset_id, asset_revision, asset_type, listing_id, listing_revision, listing_name, secret_arn, secret_name): 126 | """ Complementary function to update item with asset subscription details in respective governance DynamoDB table""" 127 | 128 | asset_subscription_item = { 129 | 'datazone_consumer_environment_id': environment_id, 130 | 'datazone_consumer_project_id': project_id, 131 | 'datazone_domain_id': domain_id, 132 | 'datazone_asset_id': asset_id, 133 | 'datazone_asset_revision': asset_revision, 134 | 'datazone_asset_type': asset_type, 135 | 'datazone_listing_id': listing_id, 136 | 'datazone_listing_revision': listing_revision, 137 | 'datazone_listing_name': listing_name, 138 | 'secret_arn': secret_arn, 139 | 'secret_name': secret_name, 140 | 'owner_account': ACCOUNT_ID, 141 | 'owner_region': REGION, 142 | 'last_updated': datetime.now().strftime("%Y-%m-%dT%H:%M:%S") 143 | 144 | } 145 | 146 | dynamodb_response = dynamodb.put_item( 147 | TableName=G_C_ASSET_SUBSCRIPTIONS_TABLE_NAME, 148 | Item={key: dynamodb_serializer.serialize(value) for key, value in asset_subscription_item.items()} 149 | ) 150 | 151 | return asset_subscription_item 152 | 153 | 154 | def json_datetime_encoder(obj): 155 | """ Complementary function to transform dict objects delivered by AWS API into JSONs """ 156 | if isinstance(obj, (datetime)): return obj.strftime("%Y-%m-%dT%H:%M:%S") 157 | -------------------------------------------------------------------------------- /src/consumer/code/stepfunctions/consumer_manage_subscription_grant_workflow.asl.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "State machine to orchestrate activities to manage dataset subscription grants on consumer side", 3 | "StartAt": "New secret?", 4 | "States": { 5 | "New secret?": { 6 | "Type": "Choice", 7 | "Default": "Secret already shared", 8 | "Choices": [ 9 | { 10 | "Next": "Copy subscription secret", 11 | "StringEquals": "true", 12 | "Variable": "$.ProducerGrantDetails.NewSubscriptionSecret" 13 | } 14 | ] 15 | }, 16 | "Secret already shared": { 17 | "Type": "Pass", 18 | "Next": "Update subscription records", 19 | "ResultPath": null 20 | }, 21 | "Copy subscription secret": { 22 | "Type": "Task", 23 | "Next": "Update subscription records", 24 | "Parameters": { 25 | "FunctionName": "${c_copy_subscription_secret_lambda_arn}", 26 | "Payload.$": "$" 27 | }, 28 | "Resource": "arn:aws:states:::lambda:invoke", 29 | "ResultPath": "$.CopySubscriptionSecretDetails", 30 | "ResultSelector": { 31 | "SecretArn.$": "$.Payload.secret_arn", 32 | "SecretName.$": "$.Payload.secret_name" 33 | } 34 | }, 35 | "Update subscription records": { 36 | "Type": "Task", 37 | "End": true, 38 | "Parameters": { 39 | "FunctionName": "${c_update_subscription_records_lambda_arn}", 40 | "Payload.$": "$" 41 | }, 42 | "Resource": "arn:aws:states:::lambda:invoke", 43 | "ResultPath": "$.UpdateSubscriptionRecordsDetails", 44 | "ResultSelector": { 45 | "DataZoneConsumerEnvironmentId.$": "$.Payload.datazone_consumer_environment_id", 46 | "DataZoneConsumerProjectId.$": "$.Payload.datazone_consumer_project_id", 47 | "DataZoneDomainId.$": "$.Payload.datazone_domain_id", 48 | "DataZoneAssetId.$": "$.Payload.datazone_asset_id", 49 | "DataZoneAssetRevision.$": "$.Payload.datazone_asset_revision", 50 | "DataZoneAssetType.$": "$.Payload.datazone_asset_type", 51 | "DataZoneListingId.$": "$.Payload.datazone_listing_id", 52 | "DataZoneListingRevision.$": "$.Payload.datazone_listing_revision", 53 | "DataZoneListingName.$": "$.Payload.datazone_listing_name", 54 | "SecretArn.$": "$.Payload.secret_arn", 55 | "SecretName.$": "$.Payload.secret_name", 56 | "OwnerAccount.$": "$.Payload.owner_account", 57 | "OwnerRegion.$": "$.Payload.owner_region", 58 | "LastUpdated.$": "$.Payload.last_updated" 59 | } 60 | } 61 | } 62 | } -------------------------------------------------------------------------------- /src/consumer/code/stepfunctions/consumer_manage_subscription_revoke_workflow.asl.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "State machine to orchestrate activities to manage dataset subscription revocations on consumer side", 3 | "StartAt": "Remove subscription records", 4 | "States": { 5 | "Remove subscription records": { 6 | "Type": "Task", 7 | "Next": "Delete / keep subscription secret?", 8 | "Parameters": { 9 | "FunctionName": "${c_remove_subscription_records_lambda_arn}", 10 | "Payload.$": "$" 11 | }, 12 | "Resource": "arn:aws:states:::lambda:invoke", 13 | "ResultPath": "$.RemoveSubscriptionRecordsDetails", 14 | "ResultSelector": { 15 | "DataZoneConsumerEnvironmentId.$": "$.Payload.datazone_consumer_environment_id", 16 | "DataZoneConsumerProjectId.$": "$.Payload.datazone_consumer_project_id", 17 | "DataZoneDomainId.$": "$.Payload.datazone_domain_id", 18 | "DataZoneAssetId.$": "$.Payload.datazone_asset_id", 19 | "DataZoneAssetRevision.$": "$.Payload.datazone_asset_revision", 20 | "DataZoneAssetType.$": "$.Payload.datazone_asset_type", 21 | "DataZoneListingId.$": "$.Payload.datazone_listing_id", 22 | "DataZoneListingRevision.$": "$.Payload.datazone_listing_revision", 23 | "DataZoneListingName.$": "$.Payload.datazone_listing_name", 24 | "SecretArn.$": "$.Payload.secret_arn", 25 | "SecretName.$": "$.Payload.secret_name", 26 | "OwnerAccount.$": "$.Payload.owner_account", 27 | "OwnerRegion.$": "$.Payload.owner_region", 28 | "LastUpdated.$": "$.Payload.last_updated" 29 | } 30 | }, 31 | "Delete / keep subscription secret?": { 32 | "Type": "Choice", 33 | "Default": "Keep subscription secret", 34 | "Choices": [ 35 | { 36 | "Next": "Delete subscription secret", 37 | "StringEquals": "true", 38 | "Variable": "$.ProducerRevokeDetails.SecretDeleted" 39 | } 40 | ] 41 | }, 42 | "Keep subscription secret": { 43 | "Type": "Pass", 44 | "End": true, 45 | "Parameters": { 46 | "SecretArn.$": "$.RemoveSubscriptionRecordsDetails.SecretArn", 47 | "SecretName.$": "$.RemoveSubscriptionRecordsDetails.SecretName", 48 | "SecretDeleted": "false", 49 | "SecretDeletionDate": "None", 50 | "SecretRecoveryWindowInDays": "None" 51 | }, 52 | "ResultPath": "$.DeleteSubscriptionSecretDetails" 53 | }, 54 | "Delete subscription secret": { 55 | "Type": "Task", 56 | "End": true, 57 | "Parameters": { 58 | "FunctionName": "${c_delete_subscription_secret_lambda_arn}", 59 | "Payload.$": "$" 60 | }, 61 | "Resource": "arn:aws:states:::lambda:invoke", 62 | "ResultPath": "$.DeleteSubscriptionSecretDetails", 63 | "ResultSelector": { 64 | "SecretArn.$": "$.Payload.secret_arn", 65 | "SecretName.$": "$.Payload.secret_name", 66 | "SecretDeleted.$": "$.Payload.secret_deleted", 67 | "SecretDeletionDate.$": "$.Payload.secret_deletion_date", 68 | "SecretRecoveryWindowInDays.$": "$.Payload.secret_recovery_window_in_days" 69 | } 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /src/consumer/constructs/consumer_subscription_grant_workflow.py: -------------------------------------------------------------------------------- 1 | from config.common.global_vars import GLOBAL_VARIABLES 2 | 3 | from aws_cdk import ( 4 | Environment, 5 | RemovalPolicy, 6 | aws_lambda as lambda_, 7 | aws_stepfunctions as stepfunctions, 8 | aws_logs as logs 9 | ) 10 | 11 | from os import path; 12 | 13 | from constructs import Construct 14 | 15 | class ConsumerManageSubscriptionGrantWorkflowConstruct(Construct): 16 | """ Class to represent the workflow that will execute in the consumer account after a Amazon DataZone subscription is approved. 17 | The workflow will copy the secret shared by the producer account into a local one with access only for project owning the subscription. Metadata will be updated in dynamodb tables hosted on governance account. 18 | Actions involving governance account resources will be done via cross-account access. 19 | """ 20 | 21 | def __init__(self, scope: Construct, construct_id: str, account_props: dict, workflow_props: dict, common_constructs: dict, env: Environment, **kwargs) -> None: 22 | """ Class Constructor. Will create a workflow (state machine and belonging lambda functions) based on properties specified as parameter. 23 | 24 | Parameters 25 | ---------- 26 | account_props : dict 27 | dict with common properties for account. 28 | For more details check config/account/a__config.py documentation and examples. 29 | 30 | workflow_props : dict 31 | dict with required properties for workflow creation. 32 | For more details check config/account/a__config.py documentation and examples. 33 | 34 | common_constructs: dic 35 | dict with constructs common to the account. Created in and output of account common stack. 36 | 37 | env: Environment 38 | Environment object with region and account details 39 | """ 40 | 41 | super().__init__(scope, construct_id, **kwargs) 42 | account_id, region = account_props['account_id'], account_props['region'] 43 | 44 | # ---------------- Lambda ------------------------ 45 | c_copy_subscription_secret_lambda = lambda_.Function( 46 | scope= self, 47 | id= 'c_copy_subscription_secret_lambda', 48 | function_name= 'dz_conn_c_copy_subscription_secret', 49 | runtime= lambda_.Runtime.PYTHON_3_11, 50 | code=lambda_.Code.from_asset(path.join('src/consumer/code/lambda', "copy_subscription_secret")), 51 | handler= "copy_subscription_secret.handler", 52 | role= common_constructs['a_common_lambda_role'], 53 | environment= { 54 | 'G_ACCOUNT_ID': GLOBAL_VARIABLES['governance']['g_account_number'], 55 | 'G_CROSS_ACCOUNT_ASSUME_ROLE_NAME': GLOBAL_VARIABLES['governance']['g_cross_account_assume_role_name'], 56 | 'G_C_SECRETS_MAPPING_TABLE_NAME': GLOBAL_VARIABLES['governance']['g_c_secrets_mapping_table_name'], 57 | 'A_COMMON_KEY_ALIAS': common_constructs['a_common_key_alias'], 58 | 'ACCOUNT_ID': account_id, 59 | 'REGION': region 60 | } 61 | ) 62 | 63 | c_update_subscription_records_lambda = lambda_.Function( 64 | scope= self, 65 | id= 'c_update_subscription_records_lambda', 66 | function_name= 'dz_conn_c_update_subscription_records', 67 | runtime= lambda_.Runtime.PYTHON_3_11, 68 | code=lambda_.Code.from_asset(path.join('src/consumer/code/lambda', "update_subscription_records")), 69 | handler= "update_subscription_records.handler", 70 | role= common_constructs['a_common_lambda_role'], 71 | environment= { 72 | 'G_ACCOUNT_ID': GLOBAL_VARIABLES['governance']['g_account_number'], 73 | 'G_CROSS_ACCOUNT_ASSUME_ROLE_NAME': GLOBAL_VARIABLES['governance']['g_cross_account_assume_role_name'], 74 | 'G_C_SECRETS_MAPPING_TABLE_NAME': GLOBAL_VARIABLES['governance']['g_c_secrets_mapping_table_name'], 75 | 'G_C_ASSET_SUBSCRIPTIONS_TABLE_NAME': GLOBAL_VARIABLES['governance']['g_c_asset_subscriptions_table_name'], 76 | 'ACCOUNT_ID': account_id, 77 | 'REGION': region 78 | } 79 | ) 80 | 81 | # ---------------- Step Functions ------------------------ 82 | c_manage_subscription_grant_state_machine_name = GLOBAL_VARIABLES['consumer']['c_manage_subscription_grant_state_machine_name'] 83 | 84 | c_manage_subscription_grant_state_machine_logs = logs.LogGroup( 85 | scope= self, 86 | id= 'c_manage_subscription_grant_state_machine_logs', 87 | log_group_name=f'/aws/step-functions/{c_manage_subscription_grant_state_machine_name}', 88 | removal_policy=RemovalPolicy.DESTROY 89 | ) 90 | 91 | c_manage_subscription_grant_state_machine = stepfunctions.StateMachine( 92 | scope= self, 93 | id= 'c_manage_subscription_grant_state_machine', 94 | state_machine_name= GLOBAL_VARIABLES['consumer']['c_manage_subscription_grant_state_machine_name'], 95 | definition_body=stepfunctions.DefinitionBody.from_file('src/consumer/code/stepfunctions/consumer_manage_subscription_grant_workflow.asl.json'), 96 | definition_substitutions= { 97 | 'c_copy_subscription_secret_lambda_arn': c_copy_subscription_secret_lambda.function_arn, 98 | 'c_update_subscription_records_lambda_arn': c_update_subscription_records_lambda.function_arn 99 | }, 100 | role= common_constructs['a_common_sf_role'], 101 | logs= stepfunctions.LogOptions( 102 | destination=c_manage_subscription_grant_state_machine_logs, 103 | level=stepfunctions.LogLevel.ALL 104 | ), 105 | tracing_enabled=True 106 | ) 107 | 108 | -------------------------------------------------------------------------------- /src/consumer/constructs/consumer_subscription_revoke_workflow.py: -------------------------------------------------------------------------------- 1 | from config.common.global_vars import GLOBAL_VARIABLES 2 | 3 | from aws_cdk import ( 4 | Environment, 5 | RemovalPolicy, 6 | aws_lambda as lambda_, 7 | aws_stepfunctions as stepfunctions, 8 | aws_logs as logs 9 | ) 10 | 11 | from os import path; 12 | 13 | from constructs import Construct 14 | 15 | class ConsumerManageSubscriptionRevokeWorkflowConstruct(Construct): 16 | """ Class to represent the workflow that will execute in the consumer account after a Amazon DataZone subscription is revoked. 17 | The workflow will delete the local secret if not additional grants are supported by it. Metadata will be updated in dynamodb tables hosted on governance account. 18 | Actions involving governance account resources will be done via cross-account access. 19 | """ 20 | 21 | def __init__(self, scope: Construct, construct_id: str, account_props: dict, workflow_props: dict, common_constructs: dict, env: Environment, **kwargs) -> None: 22 | """ Class Constructor. Will create a workflow (state machine and belonging lambda functions) based on properties specified as parameter. 23 | 24 | Parameters 25 | ---------- 26 | account_props : dict 27 | dict with common properties for account. 28 | For more details check config/account/a__config.py documentation and examples. 29 | 30 | workflow_props : dict 31 | dict with required properties for workflow creation. 32 | For more details check config/account/a__config.py documentation and examples. 33 | 34 | common_constructs: dic 35 | dict with constructs common to the account. Created in and output of account common stack. 36 | 37 | env: Environment 38 | Environment object with region and account details 39 | """ 40 | 41 | super().__init__(scope, construct_id, **kwargs) 42 | account_id, region = account_props['account_id'], account_props['region'] 43 | 44 | # ---------------- Lambda ------------------------ 45 | c_delete_subscription_secret_lambda = lambda_.Function( 46 | scope= self, 47 | id= 'c_delete_subscription_secret_lambda', 48 | function_name= 'dz_conn_c_delete_subscription_secret', 49 | runtime= lambda_.Runtime.PYTHON_3_11, 50 | code=lambda_.Code.from_asset(path.join('src/consumer/code/lambda', "delete_subscription_secret")), 51 | handler= "delete_subscription_secret.handler", 52 | role= common_constructs['a_common_lambda_role'], 53 | environment= { 54 | 'G_ACCOUNT_ID': GLOBAL_VARIABLES['governance']['g_account_number'], 55 | 'G_CROSS_ACCOUNT_ASSUME_ROLE_NAME': GLOBAL_VARIABLES['governance']['g_cross_account_assume_role_name'], 56 | 'G_C_SECRETS_MAPPING_TABLE_NAME': GLOBAL_VARIABLES['governance']['g_c_secrets_mapping_table_name'], 57 | 'RECOVERY_WINDOW_IN_DAYS': workflow_props['secret_recovery_window_in_days'] 58 | } 59 | ) 60 | 61 | c_remove_subscription_records_lambda = lambda_.Function( 62 | scope= self, 63 | id= 'c_remove_subscription_records_lambda', 64 | function_name= 'dz_conn_c_remove_subscription_records', 65 | runtime= lambda_.Runtime.PYTHON_3_11, 66 | code=lambda_.Code.from_asset(path.join('src/consumer/code/lambda', "remove_subscription_records")), 67 | handler= "remove_subscription_records.handler", 68 | role= common_constructs['a_common_lambda_role'], 69 | environment= { 70 | 'G_ACCOUNT_ID': GLOBAL_VARIABLES['governance']['g_account_number'], 71 | 'G_CROSS_ACCOUNT_ASSUME_ROLE_NAME': GLOBAL_VARIABLES['governance']['g_cross_account_assume_role_name'], 72 | 'G_C_ASSET_SUBSCRIPTIONS_TABLE_NAME': GLOBAL_VARIABLES['governance']['g_c_asset_subscriptions_table_name'] 73 | } 74 | ) 75 | 76 | # ---------------- Step Functions ------------------------ 77 | c_manage_subscription_revoke_state_machine_name = GLOBAL_VARIABLES['consumer']['c_manage_subscription_revoke_state_machine_name'] 78 | 79 | c_manage_subscription_revoke_state_machine_logs = logs.LogGroup( 80 | scope= self, 81 | id= 'c_manage_subscription_revoke_state_machine_logs', 82 | log_group_name=f'/aws/step-functions/{c_manage_subscription_revoke_state_machine_name}', 83 | removal_policy=RemovalPolicy.DESTROY 84 | ) 85 | 86 | c_manage_subscription_revoke_state_machine = stepfunctions.StateMachine( 87 | scope= self, 88 | id= 'c_manage_subscription_revoke_state_machine', 89 | state_machine_name= GLOBAL_VARIABLES['consumer']['c_manage_subscription_revoke_state_machine_name'], 90 | definition_body=stepfunctions.DefinitionBody.from_file('src/consumer/code/stepfunctions/consumer_manage_subscription_revoke_workflow.asl.json'), 91 | definition_substitutions= { 92 | 'c_delete_subscription_secret_lambda_arn': c_delete_subscription_secret_lambda.function_arn, 93 | 'c_remove_subscription_records_lambda_arn': c_remove_subscription_records_lambda.function_arn 94 | }, 95 | role= common_constructs['a_common_sf_role'], 96 | logs= stepfunctions.LogOptions( 97 | destination=c_manage_subscription_revoke_state_machine_logs, 98 | level=stepfunctions.LogLevel.ALL 99 | ), 100 | tracing_enabled=True 101 | ) 102 | 103 | -------------------------------------------------------------------------------- /src/consumer/stacks/consumer_workflows_stack.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import ( 2 | Stack, 3 | Environment 4 | ) 5 | 6 | from constructs import Construct 7 | from src.consumer.constructs.consumer_subscription_grant_workflow import ConsumerManageSubscriptionGrantWorkflowConstruct 8 | from src.consumer.constructs.consumer_subscription_revoke_workflow import ConsumerManageSubscriptionRevokeWorkflowConstruct 9 | 10 | class ConsumerWorkflowsStack(Stack): 11 | """ Class to represents the stack containing all consumer workflows in account.""" 12 | 13 | def __init__(self, scope: Construct, construct_id: str, account_props: dict, workflows_props: list, common_constructs: dict, env: Environment, **kwargs) -> None: 14 | """ Class Constructor. Will deploy one of each consumer workflow constructs based on properties specified as parameter. 15 | 16 | Parameters 17 | ---------- 18 | account_props : dict 19 | dict with common properties for account. 20 | For more details check config/account/a__config.py documentation and examples. 21 | 22 | workflows_props : dict 23 | dict with required properties for all workflows creation. 24 | For more details check config/account/a__config.py documentation and examples. 25 | 26 | common_constructs: dic 27 | dict with constructs common to the account. Created in and output of account common stack. 28 | 29 | env: Environment 30 | Environment object with region and account details 31 | """ 32 | 33 | super().__init__(scope, construct_id, **kwargs) 34 | account_id, region = account_props['account_id'], account_props['region'] 35 | 36 | c_manage_subscription_grant_workflow_props = workflows_props['c_manage_subscription_grant'] 37 | 38 | ConsumerManageSubscriptionGrantWorkflowConstruct( 39 | scope = self, 40 | construct_id = 'dz-conn-c-manage-subscription-grant-workflow-construct', 41 | account_props = account_props, 42 | workflow_props = c_manage_subscription_grant_workflow_props, 43 | common_constructs = common_constructs, 44 | env = env 45 | ) 46 | 47 | c_manage_subscription_revoke_workflow_props = workflows_props['c_manage_subscription_revoke'] 48 | 49 | ConsumerManageSubscriptionRevokeWorkflowConstruct( 50 | scope = self, 51 | construct_id = 'dz-conn-c-manage-subscription-revoke-workflow-construct', 52 | account_props = account_props, 53 | workflow_props = c_manage_subscription_revoke_workflow_props, 54 | common_constructs = common_constructs, 55 | env = env 56 | ) 57 | -------------------------------------------------------------------------------- /src/governance/code/lambda/get_environment_details/get_environment_details.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import boto3 4 | from datetime import datetime 5 | 6 | datazone = boto3.client('datazone') 7 | 8 | def handler(event, context): 9 | """ Function handler: Function that will retrieve environment's details. 1/ Will retrieve environment metadata from Amazon DataZone, then 10 | 2/ will retrieve environment blueprint metadata from Amazon DataZone. 11 | 12 | Parameters 13 | ---------- 14 | event: dict - Input event dict containing: 15 | EventDetails: dict - Dict containing details including: 16 | metadata.domain: str - Id of DataZone domain 17 | data.environmentId: str - Id of DataZone environment 18 | 19 | context: dict - Input context. Not used on function 20 | 21 | Returns 22 | ------- 23 | environment_details: dict - Dict with Amazon DataZone environment details: 24 | AccountId: Id of the AWS account hosting the environment 25 | Region: Region hosting the environment 26 | DomainId: Id of the Amazon DataZone domain 27 | ProjectId: Id of the Amazon DataZone project owning the environment 28 | EnvironmentId: str - Id of the Amazon DataZone environment. 29 | EnvironmentName: str - Name of the Amazon DataZone environment. 30 | EnvironmentBlueprintId: str - Id of the Amazon DataZone environment blueprint. 31 | EnvironmentBlueprintName: str - Name of the Amazon DataZone environment blueprint. 32 | EnvironmentResources: dict - Dictionary with all provisioned resources for Amazon DataZone environment 33 | """ 34 | 35 | event_details = event['EventDetails'] 36 | 37 | domain_id = event_details['metadata']['domain'] 38 | environment_id = event_details['data']['environmentId'] 39 | delete_status_overwrite = True if 'delete' in event_details['data'] else False 40 | 41 | datazone_response = datazone.get_environment(domainIdentifier=domain_id, identifier=environment_id) 42 | account_id = datazone_response['awsAccountId'] 43 | region = datazone_response['awsAccountRegion'] 44 | project_id = datazone_response['projectId'] 45 | environment_name = datazone_response['name'] 46 | environment_status = datazone_response['status'] if not delete_status_overwrite else 'DELETING' 47 | environment_blueprint_id = datazone_response['environmentBlueprintId'] 48 | 49 | environment_resources = {} 50 | if 'provisionedResources' in datazone_response: 51 | environment_resources = { 52 | resource['name']: resource['value'] for resource in datazone_response['provisionedResources'] 53 | } 54 | 55 | datazone_response = datazone.get_environment_blueprint(domainIdentifier=domain_id, identifier=environment_blueprint_id) 56 | environment_blueprint_name = datazone_response['name'] 57 | 58 | environment_details = { 59 | 'AccountId': account_id, 60 | 'Region': region, 61 | 'DomainId': domain_id, 62 | 'ProjectId': project_id, 63 | 'EnvironmentId': environment_id, 64 | 'EnvironmentName': environment_name, 65 | 'EnvironmentStatus': environment_status, 66 | 'EnvironmentBlueprintId': environment_blueprint_id, 67 | 'EnvironmentBlueprintName': environment_blueprint_name, 68 | 'EnvironmentResources': environment_resources 69 | } 70 | 71 | return environment_details 72 | 73 | def json_datetime_encoder(obj): 74 | """ Complementary function to transform dict objects delivered by AWS API into JSONs """ 75 | if isinstance(obj, (datetime)): return obj.strftime("%Y-%m-%dT%H:%M:%S") 76 | -------------------------------------------------------------------------------- /src/governance/code/lambda/get_subscription_details/get_subscription_details.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import boto3 4 | from datetime import datetime 5 | 6 | datazone = boto3.client('datazone') 7 | 8 | def json_datetime_encoder(obj): 9 | """ Complementary function to transform dict objects delivered by AWS API into JSONs """ 10 | if isinstance(obj, (datetime)): return obj.strftime("%Y-%m-%dT%H:%M:%S") 11 | 12 | def handler(event, context): 13 | """ Function handler: Function that will retrieve subscription's details. 1/ Will retrieve listing metadata from Amazon DataZone 14 | 2/ Will retrieve producer project details from Amazon DataZone 3/ Will retrieve consumer project and environment details from Amazon DataZone 15 | 4/ Will build response base on producer, consumer, listing and asset details. 16 | 17 | Parameters 18 | ---------- 19 | event: dict - Input event dict containing: 20 | EventDetails: dict - Dict containing details including: 21 | metadata.domain: str - Id of DataZone domain 22 | data.asset.listingId: str - Id of the DataZone listing associated to subscription 23 | data.asset.listingVersion: str - Revision of the DataZone listing associated to subscription 24 | data.projectId: str - Id of the Amazon DataZone consumer project 25 | data.subscriptionTarget.environmentId: str - Id of the Amazon DataZone consumer environment 26 | 27 | context: dict - Input context. Not used on function 28 | 29 | Returns 30 | ------- 31 | subscription_details: dict - Dict with subscription details including: 32 | DomainId: str - Id of the Amazon DataZone domain. 33 | ProducerProjectDetails: dict - Dict with producer project details. 34 | ProjectId: str - Id of the Amazon DataZone producer project 35 | ProjectName: str - Name of the Amazon DataZone producer project 36 | ConsumerProjectDetails: dict - Dict with producer project details. 37 | AccountId: str - Consumer environment AWS account id. 38 | Region: str - Consumer environment region. 39 | ProjectId: str - Id of the Amazon DataZone consumer project 40 | ProjectName: str - Name of the Amazon DataZone consumer project 41 | EnvironmentId: str - Id of the Amazon DataZone consumer environment 42 | EnvironmentName: str - Name of the Amazon DataZone consumer environment 43 | EnvironmentProfileId: str - Id of the Amazon DataZone consumer environment profile 44 | EnvironmentProfileName: str - Name of the Amazon DataZone consumer environment profile 45 | ListingDetails: dict - Dict with listing details. 46 | Id: str - Id of the Amazon DataZone listing 47 | Name: str - Name of the Amazon DataZone listing 48 | Revision: str - Revision of the Amazon DataZone listing 49 | AssetDetails: dict - Dict with data asset details. 50 | Id: str - Id of the Amazon DataZone data asset 51 | Revision: str - Revision of the Amazon DataZone data asset 52 | Type: str - Type of the Amazon DataZone data asset. 53 | GlueTableDetails: dict - Dict included when AssetType is 'GlueTableAssetType'. Dict includes glue table details: 54 | AccountId: str - data asset AWS account id. 55 | Region: str - data asset region. 56 | DatabaseName: str - data asset glue database name. 57 | TableName: str - data asset glue table name. 58 | TableArn: str - data asset glue table arn. 59 | SourceClassification: str - data asset glue table source classification 60 | """ 61 | 62 | event_details = event['EventDetails'] 63 | domain_id = event_details['metadata']['domain'] 64 | listing_id = event_details['data']['asset']['listingId'] 65 | listing_revision = event_details['data']['asset']['listingVersion'] 66 | consumer_project_id = event_details['data']['projectId'] 67 | consumer_environment_id = event_details['data']['subscriptionTarget']['environmentId'] 68 | 69 | datazone_response = datazone.get_listing(domainIdentifier=domain_id, identifier=listing_id, listingRevision=listing_revision) 70 | listing_details = datazone_response 71 | 72 | data_asset_details = listing_details['item']['assetListing'] 73 | data_asset_type = data_asset_details['assetType'] 74 | data_asset_forms = json.loads(data_asset_details['forms']) 75 | producer_project_id = data_asset_details['owningProjectId'] 76 | 77 | subscription_details = { 78 | 'DomainId': domain_id, 79 | 'ProducerProjectDetails': get_project_details(domain_id, producer_project_id), 80 | 'ConsumerProjectDetails': { 81 | **get_project_details(domain_id, consumer_project_id), 82 | **get_environments_details(domain_id, consumer_environment_id) 83 | }, 84 | 'ListingDetails': { 85 | 'Id': listing_details['id'], 86 | 'Name': listing_details['name'], 87 | 'Revision': listing_details['listingRevision'], 88 | }, 89 | 'AssetDetails': { 90 | 'Type': data_asset_type, 91 | 'Id': data_asset_details['assetId'], 92 | 'Revision': data_asset_details['assetRevision'] 93 | } 94 | } 95 | 96 | if data_asset_type == 'GlueTableAssetType': 97 | glue_table_form = data_asset_forms['GlueTableForm'] 98 | 99 | subscription_details['AssetDetails']['GlueTableDetails'] = { 100 | 'AccountId': glue_table_form['catalogId'], 101 | 'Region': glue_table_form['region'], 102 | 'DatabaseName': glue_table_form['tableArn'].split('/')[1], 103 | 'TableName': glue_table_form['tableName'], 104 | 'TableArn': glue_table_form['tableArn'], 105 | 'SourceClassification': glue_table_form['sourceClassification'] 106 | } 107 | 108 | return subscription_details 109 | 110 | 111 | def get_project_details(domain_id, project_id): 112 | """ Complementary function to get Amazon DataZone project details """ 113 | datazone_response = datazone.get_project(domainIdentifier=domain_id, identifier=project_id) 114 | project_details = { 115 | 'ProjectId': project_id, 116 | 'ProjectName': datazone_response['name'] 117 | } 118 | return project_details 119 | 120 | 121 | def get_environments_details(domain_id, environment_id): 122 | """ Complementary function to get Amazon DataZone environment details """ 123 | environment_full_details = datazone.get_environment(domainIdentifier=domain_id, identifier=environment_id) 124 | 125 | environment_profile_id = environment_full_details['environmentProfileId'] 126 | environment_full_profile_details = datazone.get_environment_profile(domainIdentifier=domain_id, identifier=environment_profile_id) 127 | 128 | environment_details = { 129 | 'AccountId': environment_full_details['awsAccountId'], 130 | 'Region': environment_full_details['awsAccountRegion'], 131 | 'EnvironmentId': environment_id, 132 | 'EnvironmentName': environment_full_details['name'], 133 | 'EnvironmentProfileId': environment_profile_id, 134 | 'EnvironmentProfileName': environment_full_profile_details['name'], 135 | } 136 | 137 | return environment_details 138 | 139 | -------------------------------------------------------------------------------- /src/governance/code/lambda/start_subscription_workflow/start_subscription_workflow.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import boto3 4 | from datetime import datetime 5 | 6 | # Constant: Represents the default data lake datazone blueprint name 7 | DATA_LAKE_BLUEPRINT_NAME = 'DefaultDataLake' 8 | 9 | # Constant: Represents the approved status for a datazone subscription 10 | APPROVED_STATUS = 'APPROVED' 11 | 12 | # Constant: Represents the revoked status for a datazone subscription 13 | REVOKED_STATUS = 'REVOKED' 14 | 15 | # Constant: Represents the cancelled status for a datazone subscription 16 | CANCELLED_STATUS = 'CANCELLED' 17 | 18 | # Constant: Arn of the grant subscription workflow state machine 19 | G_SUBSCRIPTION_GRANT_WORKFLOW_ARN = os.getenv('G_SUBSCRIPTION_GRANT_WORKFLOW_ARN') 20 | 21 | # Constant: Arn of the revoke subscription workflow state machine 22 | G_SUBSCRIPTION_REVOKE_WORKFLOW_ARN = os.getenv('G_SUBSCRIPTION_REVOKE_WORKFLOW_ARN') 23 | 24 | datazone = boto3.client('datazone') 25 | 26 | step_functions = boto3.client('stepfunctions') 27 | 28 | def handler(event, context): 29 | """ Function handler: Function that will start either the subscription grant workflow or the subscription revoke workflow with corresponding 30 | metadata, depending to triggering event sent to to EventBridge by DataZone. 31 | 1/ Will retrieve listing metadata from Amazon DataZone. 2/ Will retrieve consumer environment list from Amazon DataZone 32 | 3/ For each environment will retrieve its details as well as its bluebrint details. 33 | 4/ For each environment will start a grant or revoke workflow (with proper event structure and metadata) if the environment is associated to the default data lake blueprint. 34 | 35 | Parameters 36 | ---------- 37 | event: dict - Input event dict containing: 38 | EventDetails: dict - Dict containing details including: 39 | metadata.domain: str - Id of DataZone domain 40 | data.subscribedListing.id: str - Id of the DataZone listing associated to subscription 41 | data.subscribedListing.version: str - Revision of the DataZone listing associated to subscription 42 | data.subscribedPrincipal.id: str - Id of the Amazon DataZone consumer project 43 | data.status: str - Status of the Amazon DataZone subscription 44 | 45 | context: dict - Input context. Not used on function 46 | 47 | Returns 48 | ------- 49 | start_events: list - List of dicts, one for each event that was sent to start a subscription workflow (grant or revoke). Each dict with structure: 50 | EventDetails: str - Dict with event details. 51 | metadata: dict - Dict with event metadata details. 52 | typeName: str - Name of the DataZone event type associated to a subscription grant / revoke event. 53 | domain: str - Id of DataZone domain 54 | data: dict - Dict with event data details. 55 | projectId: str - Id of the Amazon DataZone consumer project 56 | asset: dict - Dict with asset details. 57 | listingId: str - Id of the Amazon DataZone listing 58 | listingVersion: str - Revision of the Amazon DataZone listing 59 | typeName: str - Type of the Amazon DataZone listing 60 | subscriptionTarget: dict - Dict with subscription target details. 61 | environmentId: str - Id of the Amazon DataZone consumer environment 62 | Type: str - Type of the Amazon DataZone subscription target. 63 | """ 64 | print(event) 65 | 66 | event_details = event['EventDetails'] 67 | domain_id = event_details['metadata']['domain'] 68 | listing_id = event_details['data']['subscribedListing']['id'] 69 | listing_revision = event_details['data']['subscribedListing']['version'] 70 | consumer_project_id = event_details['data']['subscribedPrincipal']['id'] 71 | subscription_status = event_details['data']['status'] 72 | 73 | listing_details = datazone.get_listing(domainIdentifier=domain_id, identifier=listing_id, listingRevision=listing_revision) 74 | asset_type = listing_details['item']['assetListing']['assetType'] 75 | 76 | datazone_response = datazone.list_environments(domainIdentifier=domain_id, projectIdentifier=consumer_project_id) 77 | consumer_environments = datazone_response['items'] 78 | 79 | start_events = [] 80 | for environment in consumer_environments: 81 | consumer_environment_id = environment['id'] 82 | 83 | environment_details = datazone.get_environment(domainIdentifier=domain_id, identifier=consumer_environment_id) 84 | environment_blueprint_id = environment_details['environmentBlueprintId'] 85 | 86 | environment_blueprint_details = datazone.get_environment_blueprint(domainIdentifier=domain_id, identifier=environment_blueprint_id) 87 | environment_blueprint_name = environment_blueprint_details['name'] 88 | 89 | if environment_blueprint_name == DATA_LAKE_BLUEPRINT_NAME: 90 | 91 | start_subscription_event = { 92 | 'EventDetails': { 93 | "metadata": { 94 | "typeName": "SubscriptionGrantEntityType", 95 | "domain": domain_id 96 | }, 97 | "data": { 98 | "asset": { 99 | "listingId": listing_id, 100 | "listingVersion": listing_revision, 101 | "typeName": asset_type 102 | }, 103 | "projectId": consumer_project_id, 104 | "subscriptionTarget": { 105 | "environmentId": consumer_environment_id, 106 | "typeName": "GlueSubscriptionTargetType" 107 | } 108 | } 109 | } 110 | } 111 | 112 | if subscription_status == APPROVED_STATUS: 113 | step_functions.start_execution(stateMachineArn=G_SUBSCRIPTION_GRANT_WORKFLOW_ARN, input=json.dumps(start_subscription_event)) 114 | 115 | elif subscription_status in [CANCELLED_STATUS, REVOKED_STATUS]: 116 | step_functions.start_execution(stateMachineArn=G_SUBSCRIPTION_REVOKE_WORKFLOW_ARN, input=json.dumps(start_subscription_event)) 117 | 118 | start_events.append(start_subscription_event) 119 | 120 | return start_events 121 | 122 | 123 | 124 | 125 | 126 | 127 | -------------------------------------------------------------------------------- /src/governance/code/stepfunctions/governance_manage_environment_active_workflow.asl.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "State machine to orchestrate activities to manage environment active event in environment's account", 3 | "StartAt": "Get Environment Details", 4 | "States": { 5 | "Get Environment Details": { 6 | "Type": "Task", 7 | "Next": "Which environment profile?", 8 | "Parameters": { 9 | "FunctionName": "${g_get_environment_details_lambda_arn}", 10 | "Payload.$": "$" 11 | }, 12 | "Resource": "arn:aws:states:::lambda:invoke", 13 | "ResultPath": "$.EnvironmentDetails", 14 | "ResultSelector": { 15 | "AccountId.$": "$.Payload.AccountId", 16 | "Region.$": "$.Payload.Region", 17 | "DomainId.$": "$.Payload.DomainId", 18 | "ProjectId.$": "$.Payload.ProjectId", 19 | "EnvironmentId.$": "$.Payload.EnvironmentId", 20 | "EnvironmentName.$": "$.Payload.EnvironmentName", 21 | "EnvironmentStatus.$": "$.Payload.EnvironmentStatus", 22 | "EnvironmentBlueprintId.$": "$.Payload.EnvironmentBlueprintId", 23 | "EnvironmentBlueprintName.$": "$.Payload.EnvironmentBlueprintName", 24 | "EnvironmentResources.$": "$.Payload.EnvironmentResources" 25 | } 26 | }, 27 | "Which environment profile?": { 28 | "Type": "Choice", 29 | "Default": "Unsupported environment profile", 30 | "Choices": [ 31 | { 32 | "Next": "Get cross-account resource ARNs", 33 | "StringEquals": "DefaultDataLake", 34 | "Variable": "$.EnvironmentDetails.EnvironmentBlueprintName" 35 | } 36 | ] 37 | }, 38 | "Unsupported environment profile": { 39 | "Type": "Pass", 40 | "End": true, 41 | "ResultPath": null 42 | }, 43 | "Get cross-account resource ARNs": { 44 | "Type": "Pass", 45 | "Next": "Update Environment Roles", 46 | "Parameters": { 47 | "UpdateEnvironmentRolesLambdaArn.$": "States.Format('arn:aws:lambda:{}:{}:function:${a_update_environment_roles_lambda_name}', $.EnvironmentDetails.Region, $.EnvironmentDetails.AccountId)", 48 | "AddLFTagEnvironmentDBsLambdaArn.$": "States.Format('arn:aws:lambda:{}:{}:function:${p_add_lf_tag_environment_dbs_lambda_name}', $.EnvironmentDetails.Region, $.EnvironmentDetails.AccountId)", 49 | "AccountAssumeRoleArn.$": "States.Format('arn:aws:iam::{}:role/${a_cross_account_assume_role_name}', $.EnvironmentDetails.AccountId)" 50 | }, 51 | "ResultPath": "$.CrossAccountResources" 52 | }, 53 | "Update Environment Roles": { 54 | "Type": "Task", 55 | "Next": "Add LF-Tag to Environment DBs", 56 | "Parameters": { 57 | "FunctionName.$": "$.CrossAccountResources.UpdateEnvironmentRolesLambdaArn", 58 | "Payload.$": "$" 59 | }, 60 | "Resource": "arn:aws:states:::lambda:invoke", 61 | "Credentials": { 62 | "RoleArn.$": "$.CrossAccountResources.AccountAssumeRoleArn" 63 | }, 64 | "ResultPath": "$.AccountUpdateEnvironmentRolesDetails", 65 | "ResultSelector": { 66 | "EnvironmentRoleArn.$": "$.Payload.environment_role_arn" 67 | } 68 | }, 69 | "Add LF-Tag to Environment DBs": { 70 | "Type": "Task", 71 | "End": true, 72 | "Parameters": { 73 | "FunctionName.$": "$.CrossAccountResources.AddLFTagEnvironmentDBsLambdaArn", 74 | "Payload.$": "$" 75 | }, 76 | "Resource": "arn:aws:states:::lambda:invoke", 77 | "Credentials": { 78 | "RoleArn.$": "$.CrossAccountResources.AccountAssumeRoleArn" 79 | }, 80 | "ResultPath": "$.AccountAddLFTagEnvironmentDBsDetails", 81 | "ResultSelector": { 82 | "EnvironmentDBNames.$": "$.Payload.environment_db_names", 83 | "LFTagKey.$": "$.Payload.lakeformation_tag_key", 84 | "LFTagValue.$": "$.Payload.lakeformation_tag_value" 85 | } 86 | } 87 | } 88 | } -------------------------------------------------------------------------------- /src/governance/code/stepfunctions/governance_manage_environment_delete_workflow.asl.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "State machine to orchestrate activities to manage environment delete event in environments's account", 3 | "StartAt": "Get Environment Details", 4 | "States": { 5 | "Get Environment Details": { 6 | "Type": "Task", 7 | "Next": "Which status?", 8 | "Parameters": { 9 | "FunctionName": "${g_get_environment_details_lambda_arn}", 10 | "Payload.$": "$" 11 | }, 12 | "Resource": "arn:aws:states:::lambda:invoke", 13 | "ResultPath": "$.EnvironmentDetails", 14 | "ResultSelector": { 15 | "AccountId.$": "$.Payload.AccountId", 16 | "Region.$": "$.Payload.Region", 17 | "DomainId.$": "$.Payload.DomainId", 18 | "ProjectId.$": "$.Payload.ProjectId", 19 | "EnvironmentId.$": "$.Payload.EnvironmentId", 20 | "EnvironmentName.$": "$.Payload.EnvironmentName", 21 | "EnvironmentStatus.$": "$.Payload.EnvironmentStatus", 22 | "EnvironmentBlueprintId.$": "$.Payload.EnvironmentBlueprintId", 23 | "EnvironmentBlueprintName.$": "$.Payload.EnvironmentBlueprintName", 24 | "EnvironmentResources.$": "$.Payload.EnvironmentResources" 25 | } 26 | }, 27 | "Which status?": { 28 | "Type": "Choice", 29 | "Default": "Not deleting", 30 | "Choices": [ 31 | { 32 | "Next": "Which environment profile?", 33 | "StringEquals": "DELETING", 34 | "Variable": "$.EnvironmentDetails.EnvironmentStatus" 35 | } 36 | ] 37 | }, 38 | "Not deleting": { 39 | "Type": "Pass", 40 | "End": true, 41 | "ResultPath": null 42 | }, 43 | "Which environment profile?": { 44 | "Type": "Choice", 45 | "Default": "Unsupported environment profile", 46 | "Choices": [ 47 | { 48 | "Next": "Get cross-account resource ARNs", 49 | "StringEquals": "DefaultDataLake", 50 | "Variable": "$.EnvironmentDetails.EnvironmentBlueprintName" 51 | } 52 | ] 53 | }, 54 | "Unsupported environment profile": { 55 | "Type": "Pass", 56 | "End": true, 57 | "ResultPath": null 58 | }, 59 | "Get cross-account resource ARNs": { 60 | "Type": "Pass", 61 | "Next": "Clean Environment Roles", 62 | "Parameters": { 63 | "CleanEnvironmentRolesLambdaArn.$": "States.Format('arn:aws:lambda:{}:{}:function:${a_clean_environment_roles_lambda_name}', $.EnvironmentDetails.Region, $.EnvironmentDetails.AccountId)", 64 | "AccountAssumeRoleArn.$": "States.Format('arn:aws:iam::{}:role/${a_cross_account_assume_role_name}', $.EnvironmentDetails.AccountId)" 65 | }, 66 | "ResultPath": "$.CrossAccountResources" 67 | }, 68 | "Clean Environment Roles": { 69 | "Type": "Task", 70 | "End": true, 71 | "Parameters": { 72 | "FunctionName.$": "$.CrossAccountResources.CleanEnvironmentRolesLambdaArn", 73 | "Payload.$": "$" 74 | }, 75 | "Resource": "arn:aws:states:::lambda:invoke", 76 | "Credentials": { 77 | "RoleArn.$": "$.CrossAccountResources.AccountAssumeRoleArn" 78 | }, 79 | "ResultPath": "$.AccountCleanEnvironmentRolesDetails", 80 | "ResultSelector": { 81 | "EnvironmentRoleArn.$": "$.Payload.environment_role_arn" 82 | } 83 | } 84 | } 85 | } -------------------------------------------------------------------------------- /src/governance/code/stepfunctions/governance_manage_subscription_grant_workflow.asl.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "State machine to orchestrate activities to manage dataset subscription grants on consumer and producer side", 3 | "StartAt": "Get Subscription Details", 4 | "States": { 5 | "Get Subscription Details": { 6 | "Type": "Task", 7 | "Next": "Which asset type?", 8 | "Parameters": { 9 | "FunctionName": "${g_get_subscription_details_lambda_arn}", 10 | "Payload.$": "$" 11 | }, 12 | "Resource": "arn:aws:states:::lambda:invoke", 13 | "ResultPath": "$.SubscriptionDetails", 14 | "ResultSelector": { 15 | "DomainId.$": "$.Payload.DomainId", 16 | "ProducerProjectDetails.$": "$.Payload.ProducerProjectDetails", 17 | "ConsumerProjectDetails.$": "$.Payload.ConsumerProjectDetails", 18 | "ListingDetails.$": "$.Payload.ListingDetails", 19 | "AssetDetails.$": "$.Payload.AssetDetails" 20 | } 21 | }, 22 | "Which asset type?": { 23 | "Type": "Choice", 24 | "Default": "Unsupported asset type", 25 | "Choices": [ 26 | { 27 | "Next": "Get cross-account resource ARNs", 28 | "StringEquals": "GlueTableAssetType", 29 | "Variable": "$.SubscriptionDetails.AssetDetails.Type" 30 | } 31 | ] 32 | }, 33 | "Unsupported asset type": { 34 | "Type": "Pass", 35 | "End": true, 36 | "ResultPath": null 37 | }, 38 | "Get cross-account resource ARNs": { 39 | "Type": "Pass", 40 | "Next": "Manage Subscription Grant - Producer", 41 | "Parameters": { 42 | "ProducerStateMachineArn.$": "States.Format('arn:aws:states:{}:{}:stateMachine:${p_manage_subscription_grant_state_machine_name}', $.SubscriptionDetails.AssetDetails.GlueTableDetails.Region, $.SubscriptionDetails.AssetDetails.GlueTableDetails.AccountId)", 43 | "ProducerAssumeRoleArn.$": "States.Format('arn:aws:iam::{}:role/${a_cross_account_assume_role_name}', $.SubscriptionDetails.AssetDetails.GlueTableDetails.AccountId)", 44 | "ConsumerStateMachineArn.$": "States.Format('arn:aws:states:{}:{}:stateMachine:${c_manage_subscription_grant_state_machine_name}', $.SubscriptionDetails.ConsumerProjectDetails.Region, $.SubscriptionDetails.ConsumerProjectDetails.AccountId)", 45 | "ConsumerAssumeRoleArn.$": "States.Format('arn:aws:iam::{}:role/${a_cross_account_assume_role_name}', $.SubscriptionDetails.ConsumerProjectDetails.AccountId)" 46 | }, 47 | "ResultPath": "$.CrossAccountResources" 48 | }, 49 | "Manage Subscription Grant - Producer": { 50 | "Type": "Task", 51 | "Next": "Manage Subscription Grant - Consumer", 52 | "Parameters": { 53 | "Input": { 54 | "SubscriptionDetails.$": "$.SubscriptionDetails" 55 | }, 56 | "StateMachineArn.$": "$.CrossAccountResources.ProducerStateMachineArn" 57 | }, 58 | "Resource": "arn:aws:states:::states:startExecution.sync:2", 59 | "Credentials": { 60 | "RoleArn.$": "$.CrossAccountResources.ProducerAssumeRoleArn" 61 | }, 62 | "ResultPath": "$.ProducerGrantDetails", 63 | "ResultSelector": { 64 | "SecretArn.$": "$.Output.ShareSubscriptionSecretDetails.SecretArn", 65 | "SecretName.$": "$.Output.ShareSubscriptionSecretDetails.SecretName", 66 | "SubscriptionConsumerRoles.$": "$.Output.ShareSubscriptionSecretDetails.SubscriptionConsumerRoles", 67 | "NewSubscriptionSecret.$": "$.Output.ShareSubscriptionSecretDetails.NewSubscriptionSecret" 68 | } 69 | }, 70 | "Manage Subscription Grant - Consumer": { 71 | "Type": "Task", 72 | "End": true, 73 | "Parameters": { 74 | "Input": { 75 | "SubscriptionDetails.$": "$.SubscriptionDetails", 76 | "ProducerGrantDetails.$": "$.ProducerGrantDetails" 77 | }, 78 | "StateMachineArn.$": "$.CrossAccountResources.ConsumerStateMachineArn" 79 | }, 80 | "Resource": "arn:aws:states:::states:startExecution.sync:2", 81 | "Credentials": { 82 | "RoleArn.$": "$.CrossAccountResources.ConsumerAssumeRoleArn" 83 | }, 84 | "ResultPath": "$.ConsumerGrantDetails", 85 | "ResultSelector": { 86 | "DataZoneConsumerEnvironmentId.$": "$.Output.UpdateSubscriptionRecordsDetails.DataZoneConsumerEnvironmentId", 87 | "DataZoneAssetId.$": "$.Output.UpdateSubscriptionRecordsDetails.DataZoneAssetId", 88 | "SecretArn.$": "$.Output.UpdateSubscriptionRecordsDetails.SecretArn", 89 | "SecretName.$": "$.Output.UpdateSubscriptionRecordsDetails.SecretName" 90 | } 91 | } 92 | } 93 | } -------------------------------------------------------------------------------- /src/governance/code/stepfunctions/governance_manage_subscription_revoke_workflow.asl.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "State machine to orchestrate activities to manage dataset subscription revocations on consumer and producer side", 3 | "StartAt": "Get Subscription Details", 4 | "States": { 5 | "Get Subscription Details": { 6 | "Type": "Task", 7 | "Next": "Which asset type?", 8 | "Parameters": { 9 | "FunctionName": "${g_get_subscription_details_lambda_arn}", 10 | "Payload.$": "$" 11 | }, 12 | "Resource": "arn:aws:states:::lambda:invoke", 13 | "ResultPath": "$.SubscriptionDetails", 14 | "ResultSelector": { 15 | "DomainId.$": "$.Payload.DomainId", 16 | "ProducerProjectDetails.$": "$.Payload.ProducerProjectDetails", 17 | "ConsumerProjectDetails.$": "$.Payload.ConsumerProjectDetails", 18 | "ListingDetails.$": "$.Payload.ListingDetails", 19 | "AssetDetails.$": "$.Payload.AssetDetails" 20 | } 21 | }, 22 | "Which asset type?": { 23 | "Type": "Choice", 24 | "Default": "Unsupported asset type", 25 | "Choices": [ 26 | { 27 | "Next": "Get cross-account resource ARNs", 28 | "StringEquals": "GlueTableAssetType", 29 | "Variable": "$.SubscriptionDetails.AssetDetails.Type" 30 | } 31 | ] 32 | }, 33 | "Unsupported asset type": { 34 | "Type": "Fail", 35 | "Error": "Unsupported asset type" 36 | }, 37 | "Get cross-account resource ARNs": { 38 | "Type": "Pass", 39 | "Next": "Manage Subscription Revoke - Producer", 40 | "Parameters": { 41 | "ProducerStateMachineArn.$": "States.Format('arn:aws:states:{}:{}:stateMachine:${p_manage_subscription_revoke_state_machine_name}', $.SubscriptionDetails.AssetDetails.GlueTableDetails.Region, $.SubscriptionDetails.AssetDetails.GlueTableDetails.AccountId)", 42 | "ProducerAssumeRoleArn.$": "States.Format('arn:aws:iam::{}:role/${a_cross_account_assume_role_name}', $.SubscriptionDetails.AssetDetails.GlueTableDetails.AccountId)", 43 | "ConsumerStateMachineArn.$": "States.Format('arn:aws:states:{}:{}:stateMachine:${c_manage_subscription_revoke_state_machine_name}', $.SubscriptionDetails.ConsumerProjectDetails.Region, $.SubscriptionDetails.ConsumerProjectDetails.AccountId)", 44 | "ConsumerAssumeRoleArn.$": "States.Format('arn:aws:iam::{}:role/${a_cross_account_assume_role_name}', $.SubscriptionDetails.ConsumerProjectDetails.AccountId)" 45 | }, 46 | "ResultPath": "$.CrossAccountResources" 47 | }, 48 | "Manage Subscription Revoke - Producer": { 49 | "Type": "Task", 50 | "Next": "Manage Subscription Revoke - Consumer", 51 | "Parameters": { 52 | "Input": { 53 | "SubscriptionDetails.$": "$.SubscriptionDetails" 54 | }, 55 | "StateMachineArn.$": "$.CrossAccountResources.ProducerStateMachineArn" 56 | }, 57 | "Resource": "arn:aws:states:::states:startExecution.sync:2", 58 | "Credentials": { 59 | "RoleArn.$": "$.CrossAccountResources.ProducerAssumeRoleArn" 60 | }, 61 | "ResultPath": "$.ProducerRevokeDetails", 62 | "ResultSelector": { 63 | "SecretArn.$": "$.Output.RevokeSubscriptionDetails.SecretArn", 64 | "SecretName.$": "$.Output.RevokeSubscriptionDetails.SecretName", 65 | "SecretDeleted.$": "$.Output.DeleteKeepSubscriptionSecretDetails.SecretDeleted", 66 | "SecretDeletionDate.$": "$.Output.DeleteKeepSubscriptionSecretDetails.SecretDeletionDate", 67 | "SecretRecoveryWindowInDays.$": "$.Output.DeleteKeepSubscriptionSecretDetails.SecretRecoveryWindowInDays" 68 | } 69 | }, 70 | "Manage Subscription Revoke - Consumer": { 71 | "Type": "Task", 72 | "End": true, 73 | "Parameters": { 74 | "Input": { 75 | "SubscriptionDetails.$": "$.SubscriptionDetails", 76 | "ProducerRevokeDetails.$": "$.ProducerRevokeDetails" 77 | }, 78 | "StateMachineArn.$": "$.CrossAccountResources.ConsumerStateMachineArn" 79 | }, 80 | "Resource": "arn:aws:states:::states:startExecution.sync:2", 81 | "Credentials": { 82 | "RoleArn.$": "$.CrossAccountResources.ConsumerAssumeRoleArn" 83 | }, 84 | "ResultPath": "$.ConsumerRevokeDetails", 85 | "ResultSelector": { 86 | "DataZoneConsumerEnvironmentId.$": "$.Output.RemoveSubscriptionRecordsDetails.DataZoneConsumerEnvironmentId", 87 | "DataZoneAssetId.$": "$.Output.RemoveSubscriptionRecordsDetails.DataZoneAssetId", 88 | "SecretArn.$": "$.Output.DeleteSubscriptionSecretDetails.SecretArn", 89 | "SecretName.$": "$.Output.DeleteSubscriptionSecretDetails.SecretName", 90 | "SecretDeleted.$": "$.Output.DeleteSubscriptionSecretDetails.SecretDeleted", 91 | "SecretDeletionDate.$": "$.Output.DeleteSubscriptionSecretDetails.SecretDeletionDate", 92 | "SecretRecoveryWindowInDays.$": "$.Output.DeleteSubscriptionSecretDetails.SecretRecoveryWindowInDays" 93 | } 94 | } 95 | } 96 | } -------------------------------------------------------------------------------- /src/governance/constructs/governance_environment_active_workflow.py: -------------------------------------------------------------------------------- 1 | from config.common.global_vars import GLOBAL_VARIABLES 2 | 3 | from aws_cdk import ( 4 | Environment, 5 | RemovalPolicy, 6 | aws_stepfunctions as stepfunctions, 7 | aws_events as events, 8 | aws_events_targets as event_targets, 9 | aws_iam as iam, 10 | aws_logs as logs 11 | ) 12 | 13 | from os import path 14 | 15 | from constructs import Construct 16 | 17 | class GovernanceManageEnvironmentActiveWorkflowConstruct(Construct): 18 | """ Class to represent the workflow that will execute after a Amazon DataZone environment is turned active. 19 | The workflow will update environment roles' permission boundaries and policies. Additionally will assign LF-tags to project Glue catalog databases. 20 | Actions will be performed in account owning the Amazon DataZone environment through cross-account access. 21 | """ 22 | 23 | def __init__(self, scope: Construct, construct_id: str, governance_props: dict, workflow_props: dict, common_constructs: dict, env: Environment, **kwargs) -> None: 24 | """ Class Constructor. Will create a workflow (state machine and event rule/target) based on properties specified as parameter 25 | Lambdas to be invoked by the workflow are provisioned in account common stack. 26 | 27 | Parameters 28 | ---------- 29 | governance_props : dict 30 | dict with common properties for governance account. 31 | For more details check config/governance/g_config.py documentation and examples. 32 | 33 | workflow_props : dict 34 | dict with required properties for workflow creation. 35 | For more details check config/governance/g_config.py documentation and examples. 36 | 37 | common_constructs: dic 38 | dict with constructs common to the governance account. Created in and output of governance common stack. 39 | 40 | env: Environment 41 | Environment object with region and account details 42 | """ 43 | 44 | super().__init__(scope, construct_id, **kwargs) 45 | account_id, region = governance_props['account_id'], governance_props['region'] 46 | 47 | # ---------------- Step Functions ------------------------ 48 | g_manage_environment_active_state_machine_name = 'dz_conn_g_manage_environment_active' 49 | 50 | g_manage_environment_active_state_machine_logs = logs.LogGroup( 51 | scope= self, 52 | id= 'g_manage_environment_active_state_machine_logs', 53 | log_group_name=f'/aws/step-functions/{g_manage_environment_active_state_machine_name}', 54 | removal_policy=RemovalPolicy.DESTROY 55 | ) 56 | 57 | g_manage_environment_active_state_machine = stepfunctions.StateMachine( 58 | scope= self, 59 | id= 'g_manage_environment_active_state_machine', 60 | state_machine_name= g_manage_environment_active_state_machine_name, 61 | definition_body=stepfunctions.DefinitionBody.from_file('src/governance/code/stepfunctions/governance_manage_environment_active_workflow.asl.json'), 62 | definition_substitutions= { 63 | 'g_get_environment_details_lambda_arn': common_constructs['g_get_environment_details_lambda'].function_arn, 64 | 'a_update_environment_roles_lambda_name': GLOBAL_VARIABLES['account']['a_update_environment_roles_lambda_name'], 65 | 'p_add_lf_tag_environment_dbs_lambda_name': GLOBAL_VARIABLES['producer']['p_add_lf_tag_environment_dbs_lambda_name'], 66 | 'a_cross_account_assume_role_name': GLOBAL_VARIABLES['account']['a_cross_account_assume_role_name'] 67 | }, 68 | role=common_constructs['g_common_sf_role'], 69 | logs= stepfunctions.LogOptions( 70 | destination=g_manage_environment_active_state_machine_logs, 71 | level=stepfunctions.LogLevel.ALL 72 | ), 73 | tracing_enabled=True 74 | ) 75 | 76 | # --------------- EventBridge --------------------------- 77 | g_common_eventbridge_role = iam.Role.from_role_name( 78 | scope= self, 79 | id= 'g_common_eventbridge_role_name', 80 | role_name= common_constructs['g_common_eventbridge_role_name'] 81 | ) 82 | 83 | g_manage_environment_active_rule = events.Rule( 84 | scope= self, 85 | id= 'g_manage_environment_active_rule', 86 | rule_name= 'dz_conn_g_manage_environment_active_rule', 87 | enabled=workflow_props['g_eventbridge_rule_enabled'], 88 | event_pattern=events.EventPattern( 89 | source=['aws.datazone'], 90 | detail_type=['Environment Deployment Completed'] 91 | ) 92 | ) 93 | 94 | g_manage_environment_active_rule_target = event_targets.SfnStateMachine( 95 | machine= g_manage_environment_active_state_machine, 96 | role=g_common_eventbridge_role, 97 | input=events.RuleTargetInput.from_object( 98 | { 'EventDetails': events.EventField.from_path('$.detail') } 99 | ) 100 | ) 101 | 102 | g_manage_environment_active_rule.add_target(g_manage_environment_active_rule_target) 103 | 104 | -------------------------------------------------------------------------------- /src/governance/constructs/governance_environment_delete_workflow.py: -------------------------------------------------------------------------------- 1 | from config.common.global_vars import GLOBAL_VARIABLES 2 | 3 | from aws_cdk import ( 4 | Environment, 5 | RemovalPolicy, 6 | aws_stepfunctions as stepfunctions, 7 | aws_events as events, 8 | aws_events_targets as event_targets, 9 | aws_iam as iam, 10 | aws_logs as logs 11 | ) 12 | 13 | from constructs import Construct 14 | 15 | class GovernanceManageEnvironmentDeleteWorkflowConstruct(Construct): 16 | """ Class to represent the workflow that will execute after a Amazon DataZone environment is triggered for deletion. 17 | The workflow will clean environment roles' permission boundaries and policies to their original state. 18 | Actions will be performed in account owning the Amazon DataZone environment through cross-account access. 19 | """ 20 | 21 | def __init__(self, scope: Construct, construct_id: str, governance_props: dict, workflow_props: dict, common_constructs: dict, env: Environment, **kwargs) -> None: 22 | """ Class Constructor. Will create a workflow (state machine and event rule/target) based on properties specified as parameter 23 | Lambdas to be invoked by the workflow are provisioned in account common stack. 24 | 25 | Parameters 26 | ---------- 27 | governance_props : dict 28 | dict with common properties for governance account. 29 | For more details check config/governance/g_config.py documentation and examples. 30 | 31 | workflow_props : dict 32 | dict with required properties for workflow creation. 33 | For more details check config/governance/g_config.py documentation and examples. 34 | 35 | common_constructs: dic 36 | dict with constructs common to the governance account. Created in and output of governance common stack. 37 | 38 | env: Environment 39 | Environment object with region and account details 40 | """ 41 | 42 | super().__init__(scope, construct_id, **kwargs) 43 | account_id, region = governance_props['account_id'], governance_props['region'] 44 | 45 | # ---------------- Step Functions ------------------------ 46 | g_manage_environment_delete_state_machine_name = 'dz_conn_g_manage_environment_delete' 47 | 48 | g_manage_environment_delete_state_machine_logs = logs.LogGroup( 49 | scope= self, 50 | id= 'g_manage_environment_active_state_machine_logs', 51 | log_group_name=f'/aws/step-functions/{g_manage_environment_delete_state_machine_name}', 52 | removal_policy=RemovalPolicy.DESTROY 53 | ) 54 | 55 | g_manage_environment_delete_state_machine = stepfunctions.StateMachine( 56 | scope= self, 57 | id= 'g_manage_environment_delete_state_machine', 58 | state_machine_name= g_manage_environment_delete_state_machine_name, 59 | definition_body=stepfunctions.DefinitionBody.from_file('src/governance/code/stepfunctions/governance_manage_environment_delete_workflow.asl.json'), 60 | definition_substitutions= { 61 | 'g_get_environment_details_lambda_arn': common_constructs['g_get_environment_details_lambda'].function_arn, 62 | 'a_clean_environment_roles_lambda_name': GLOBAL_VARIABLES['account']['a_clean_environment_roles_lambda_name'], 63 | 'a_cross_account_assume_role_name': GLOBAL_VARIABLES['account']['a_cross_account_assume_role_name'], 64 | }, 65 | role=common_constructs['g_common_sf_role'], 66 | logs= stepfunctions.LogOptions( 67 | destination=g_manage_environment_delete_state_machine_logs, 68 | level=stepfunctions.LogLevel.ALL 69 | ), 70 | tracing_enabled=True 71 | ) 72 | 73 | # --------------- EventBridge --------------------------- 74 | g_common_eventbridge_role = iam.Role.from_role_name( 75 | scope= self, 76 | id= 'g_common_eventbridge_role_name', 77 | role_name= common_constructs['g_common_eventbridge_role_name'] 78 | ) 79 | 80 | g_manage_environment_delete_rule = events.Rule( 81 | scope= self, 82 | id= 'g_manage_environment_delete_rule', 83 | rule_name= 'dz_conn_g_manage_environment_delete_rule', 84 | enabled=workflow_props['g_eventbridge_rule_enabled'], 85 | event_pattern=events.EventPattern( 86 | source=['aws.datazone'], 87 | detail_type=['Environment Deletion Started'] 88 | ) 89 | ) 90 | 91 | g_manage_environment_delete_rule_target = event_targets.SfnStateMachine( 92 | machine= g_manage_environment_delete_state_machine, 93 | role=g_common_eventbridge_role, 94 | input=events.RuleTargetInput.from_object( 95 | { 'EventDetails': events.EventField.from_path('$.detail') } 96 | ) 97 | ) 98 | 99 | g_manage_environment_delete_rule.add_target(g_manage_environment_delete_rule_target) 100 | 101 | -------------------------------------------------------------------------------- /src/governance/constructs/governance_subscription_grant_workflow.py: -------------------------------------------------------------------------------- 1 | from config.common.global_vars import GLOBAL_VARIABLES 2 | 3 | from aws_cdk import ( 4 | Environment, 5 | RemovalPolicy, 6 | aws_stepfunctions as stepfunctions, 7 | aws_events as events, 8 | aws_events_targets as event_targets, 9 | aws_iam as iam, 10 | aws_logs as logs 11 | ) 12 | 13 | from os import path 14 | 15 | from constructs import Construct 16 | 17 | class GovernanceManageSubscriptionGrantWorkflowConstruct(Construct): 18 | """ Class to represent the workflow that will execute after a Amazon DataZone subscription is approved. 19 | The workflow will orchestrate sub-workflows in producer (starting) and consumer (following) accounts to grant subscription access to JDBC sources. 20 | Actions will be performed in both producer and consumer accounts through cross-account access. 21 | """ 22 | 23 | def __init__(self, scope: Construct, construct_id: str, governance_props: dict, workflow_props: dict, common_constructs: dict, env: Environment, **kwargs) -> None: 24 | """ Class Constructor. Will create a workflow (state machine and event rule/target) based on properties specified as parameter 25 | State machines (sub-workflows) to be invoked by the workflow are provisioned in account common stack. 26 | 27 | Parameters 28 | ---------- 29 | governance_props : dict 30 | dict with common properties for governance account. 31 | For more details check config/governance/g_config.py documentation and examples. 32 | 33 | workflow_props : dict 34 | dict with required properties for workflow creation. 35 | For more details check config/governance/g_config.py documentation and examples. 36 | 37 | common_constructs: dic 38 | dict with constructs common to the governance account. Created in and output of governance common stack. 39 | 40 | env: Environment 41 | Environment object with region and account details 42 | """ 43 | 44 | super().__init__(scope, construct_id, **kwargs) 45 | account_id, region = governance_props['account_id'], governance_props['region'] 46 | 47 | # ---------------- Step Functions ------------------------ 48 | g_manage_subscription_grant_state_machine_name = GLOBAL_VARIABLES['governance']['g_manage_subscription_grant_state_machine_name'] 49 | 50 | g_manage_subscription_grant_state_machine_logs = logs.LogGroup( 51 | scope= self, 52 | id= 'g_manage_subscription_grant_state_machine_logs', 53 | log_group_name=f'/aws/step-functions/{g_manage_subscription_grant_state_machine_name}', 54 | removal_policy=RemovalPolicy.DESTROY 55 | ) 56 | 57 | g_manage_subscription_grant_state_machine = stepfunctions.StateMachine( 58 | scope= self, 59 | id= 'g_manage_subscription_grant_state_machine', 60 | state_machine_name= g_manage_subscription_grant_state_machine_name, 61 | definition_body=stepfunctions.DefinitionBody.from_file('src/governance/code/stepfunctions/governance_manage_subscription_grant_workflow.asl.json'), 62 | definition_substitutions= { 63 | 'g_get_subscription_details_lambda_arn': common_constructs['g_get_subscription_details_lambda'].function_arn, 64 | 'p_manage_subscription_grant_state_machine_name': GLOBAL_VARIABLES['producer']['p_manage_subscription_grant_state_machine_name'], 65 | 'c_manage_subscription_grant_state_machine_name': GLOBAL_VARIABLES['consumer']['c_manage_subscription_grant_state_machine_name'], 66 | 'a_cross_account_assume_role_name': GLOBAL_VARIABLES['account']['a_cross_account_assume_role_name'], 67 | }, 68 | role=common_constructs['g_common_sf_role'], 69 | logs= stepfunctions.LogOptions( 70 | destination=g_manage_subscription_grant_state_machine_logs, 71 | level=stepfunctions.LogLevel.ALL 72 | ), 73 | tracing_enabled=True 74 | ) 75 | 76 | # --------------- EventBridge --------------------------- 77 | g_manage_subscription_grant_rule = events.Rule( 78 | scope= self, 79 | id= 'g_manage_subscription_grant_rule', 80 | rule_name= 'dz_conn_g_manage_subscription_grant_rule', 81 | enabled=workflow_props['g_eventbridge_rule_enabled'], 82 | event_pattern=events.EventPattern( 83 | source=['aws.datazone'], 84 | detail_type=['Subscription Created'] 85 | ) 86 | ) 87 | 88 | g_manage_subscription_grant_rule_target = event_targets.LambdaFunction( 89 | handler= common_constructs['g_start_subscription_workflow_lambda'], 90 | event=events.RuleTargetInput.from_object( 91 | { 'EventDetails': events.EventField.from_path('$.detail') } 92 | ) 93 | ) 94 | 95 | g_manage_subscription_grant_rule.add_target(g_manage_subscription_grant_rule_target) 96 | 97 | -------------------------------------------------------------------------------- /src/governance/constructs/governance_subscription_revoke_workflow.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import ( 2 | Environment, 3 | RemovalPolicy, 4 | aws_stepfunctions as stepfunctions, 5 | aws_events as events, 6 | aws_events_targets as event_targets, 7 | aws_iam as iam, 8 | aws_logs as logs 9 | ) 10 | 11 | from config.common.global_vars import GLOBAL_VARIABLES 12 | 13 | from constructs import Construct 14 | 15 | class GovernanceManageSubscriptionRevokeWorkflowConstruct(Construct): 16 | """ Class to represent the workflow that will execute after a Amazon DataZone subscription is revoked. 17 | The workflow will orchestrate sub-workflows in producer (starting) and consumer (following) accounts to revoke subscription access to JDBC sources. 18 | Actions will be performed in both producer and consumer accounts through cross-account access. 19 | """ 20 | 21 | def __init__(self, scope: Construct, construct_id: str, governance_props: dict, workflow_props: dict, common_constructs: dict, env: Environment, **kwargs) -> None: 22 | """ Class Constructor. Will create a workflow (state machine and event rule/target) based on properties specified as parameter 23 | State machines (sub-workflows) to be invoked by the workflow are provisioned in account common stack. 24 | 25 | Parameters 26 | ---------- 27 | governance_props : dict 28 | dict with common properties for governance account. 29 | For more details check config/governance/g_config.py documentation and examples. 30 | 31 | workflow_props : dict 32 | dict with required properties for workflow creation. 33 | For more details check config/governance/g_config.py documentation and examples. 34 | 35 | common_constructs: dic 36 | dict with constructs common to the governance account. Created in and output of governance common stack. 37 | 38 | env: Environment 39 | Environment object with region and account details 40 | """ 41 | 42 | super().__init__(scope, construct_id, **kwargs) 43 | account_id, region = governance_props['account_id'], governance_props['region'] 44 | 45 | # ---------------- Step Functions ------------------------ 46 | g_manage_subscription_revoke_state_machine_name = GLOBAL_VARIABLES['governance']['g_manage_subscription_revoke_state_machine_name'] 47 | 48 | g_manage_subscription_revoke_state_machine_logs = logs.LogGroup( 49 | scope= self, 50 | id= 'g_manage_subscription_revoke_state_machine_logs', 51 | log_group_name=f'/aws/step-functions/{g_manage_subscription_revoke_state_machine_name}', 52 | removal_policy=RemovalPolicy.DESTROY 53 | ) 54 | 55 | g_manage_subscription_revoke_state_machine = stepfunctions.StateMachine( 56 | scope= self, 57 | id= 'g_manage_subscription_revoke_state_machine', 58 | state_machine_name= g_manage_subscription_revoke_state_machine_name, 59 | definition_body=stepfunctions.DefinitionBody.from_file('src/governance/code/stepfunctions/governance_manage_subscription_revoke_workflow.asl.json'), 60 | definition_substitutions= { 61 | 'g_get_subscription_details_lambda_arn': common_constructs['g_get_subscription_details_lambda'].function_arn, 62 | 'p_manage_subscription_revoke_state_machine_name': GLOBAL_VARIABLES['producer']['p_manage_subscription_revoke_state_machine_name'], 63 | 'c_manage_subscription_revoke_state_machine_name': GLOBAL_VARIABLES['consumer']['c_manage_subscription_revoke_state_machine_name'], 64 | 'a_cross_account_assume_role_name': GLOBAL_VARIABLES['account']['a_cross_account_assume_role_name'], 65 | }, 66 | role=common_constructs['g_common_sf_role'], 67 | logs= stepfunctions.LogOptions( 68 | destination=g_manage_subscription_revoke_state_machine_logs, 69 | level=stepfunctions.LogLevel.ALL 70 | ), 71 | tracing_enabled=True 72 | ) 73 | 74 | # --------------- EventBridge --------------------------- 75 | g_manage_subscription_revoke_rule = events.Rule( 76 | scope= self, 77 | id= 'g_manage_subscription_revoke_rule', 78 | rule_name= 'dz_conn_g_manage_subscription_revoke_rule', 79 | enabled=workflow_props['g_eventbridge_rule_enabled'], 80 | event_pattern=events.EventPattern( 81 | source=['aws.datazone'], 82 | detail_type=['Subscription Revoked', 'Subscription Cancelled'] 83 | ) 84 | ) 85 | 86 | g_manage_subscription_revoke_rule_target = event_targets.LambdaFunction( 87 | handler= common_constructs['g_start_subscription_workflow_lambda'], 88 | event=events.RuleTargetInput.from_object( 89 | { 'EventDetails': events.EventField.from_path('$.detail') } 90 | ) 91 | ) 92 | 93 | g_manage_subscription_revoke_rule.add_target(g_manage_subscription_revoke_rule_target) 94 | 95 | -------------------------------------------------------------------------------- /src/governance/stacks/governance_workflows_stack.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import ( 2 | Stack, 3 | Environment 4 | ) 5 | 6 | from constructs import Construct 7 | from src.governance.constructs.governance_environment_active_workflow import GovernanceManageEnvironmentActiveWorkflowConstruct 8 | from src.governance.constructs.governance_environment_delete_workflow import GovernanceManageEnvironmentDeleteWorkflowConstruct 9 | from src.governance.constructs.governance_subscription_grant_workflow import GovernanceManageSubscriptionGrantWorkflowConstruct 10 | from src.governance.constructs.governance_subscription_revoke_workflow import GovernanceManageSubscriptionRevokeWorkflowConstruct 11 | 12 | class GovernanceWorkflowsStack(Stack): 13 | """ Class to represents the stack containing all workflows in governance account.""" 14 | 15 | def __init__(self, scope: Construct, construct_id: str, governance_props: dict, workflows_props: list, common_constructs: dict, env: Environment, **kwargs) -> None: 16 | """ Class Constructor. Will deploy one of each governance workflow constructs based on properties specified as parameter. 17 | 18 | Parameters 19 | ---------- 20 | governance_props : dict 21 | dict with common properties for governance account. 22 | For more details check config/governance/g_config.py documentation and examples. 23 | 24 | workflows_props : dict 25 | dict with required properties for all workflows creation. 26 | For more details check config/governance/g_config.py documentation and examples. 27 | 28 | common_constructs: dic 29 | dict with constructs common to the governance account. Created in and output of governance common stack. 30 | 31 | env: Environment 32 | Environment object with region and account details 33 | """ 34 | 35 | super().__init__(scope, construct_id, **kwargs) 36 | account_id, region = governance_props['account_id'], governance_props['region'] 37 | 38 | g_manage_environment_active_workflow_props = workflows_props['g_manage_environment_active'] 39 | 40 | GovernanceManageEnvironmentActiveWorkflowConstruct( 41 | scope = self, 42 | construct_id = 'dz-conn-g-manage-environment-active-workflow-construct', 43 | governance_props = governance_props, 44 | workflow_props = g_manage_environment_active_workflow_props, 45 | common_constructs = common_constructs, 46 | env = env 47 | ) 48 | 49 | g_manage_environment_delete_workflow_props = workflows_props['g_manage_environment_delete'] 50 | 51 | GovernanceManageEnvironmentDeleteWorkflowConstruct( 52 | scope = self, 53 | construct_id = 'dz-conn-g-manage-environment-delete-workflow-construct', 54 | governance_props = governance_props, 55 | workflow_props = g_manage_environment_delete_workflow_props, 56 | common_constructs = common_constructs, 57 | env = env 58 | ) 59 | 60 | g_manage_subscription_grant_workflow_props = workflows_props['g_manage_subscription_grant'] 61 | 62 | GovernanceManageSubscriptionGrantWorkflowConstruct( 63 | scope = self, 64 | construct_id = 'dz-conn-g-manage-subscription-grant-workflow-construct', 65 | governance_props = governance_props, 66 | workflow_props = g_manage_subscription_grant_workflow_props, 67 | common_constructs = common_constructs, 68 | env = env 69 | ) 70 | 71 | g_manage_subscription_revoke_workflow_props = workflows_props['g_manage_subscription_revoke'] 72 | 73 | GovernanceManageSubscriptionRevokeWorkflowConstruct( 74 | scope = self, 75 | construct_id = 'dz-conn-g-manage-subscription-revoke-workflow-construct', 76 | governance_props = governance_props, 77 | workflow_props = g_manage_subscription_revoke_workflow_props, 78 | common_constructs = common_constructs, 79 | env = env 80 | ) 81 | -------------------------------------------------------------------------------- /src/producer/code/lambda/add_lf_tag_environment_dbs/add_lf_tag_environment_dbs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from datetime import datetime 4 | import boto3 5 | 6 | # Constant: Lake Formation tag key to add to environment databases in glue catalog 7 | P_LAKEFORMATION_TAG_KEY = os.getenv('P_LAKEFORMATION_TAG_KEY') 8 | 9 | # Constant: Lake Formation tag value to add to environment databases in glue catalog 10 | P_LAKEFORMATION_TAG_VALUE = os.getenv('P_LAKEFORMATION_TAG_VALUE') 11 | 12 | # Constant: List of keys pointing to glue databases inside environment resource details 13 | P_ENVIRONMENT_DBS_KEYS = ['glueProducerDBName', 'glueConsumerDBName'] 14 | 15 | lakeformation = boto3.client('lakeformation') 16 | 17 | def handler(event, context): 18 | """ Function handler: Function that will add custom lake formation tag to all environment databases in glue catalog 19 | to allow access to solution's resources on permission grants. 20 | 21 | Parameters 22 | ---------- 23 | event: dict - Input event dict containing: 24 | EnvironmentDetails: dict - Dict containing environment details including: 25 | EnvironmentResources: dict - Dict containing environment resource details. 26 | 27 | context: dict - Input context. Not used on function 28 | 29 | Returns 30 | ------- 31 | response: dict - Dict with response details including: 32 | environment_db_names: list - List with names of the glue databases that where tagged 33 | lakeformation_tag_key: str - Key of the tag added to environment databases 34 | lakeformation_tag_value: str - 'Value of the tag added to environment databases 35 | """ 36 | environment_details = event['EnvironmentDetails'] 37 | environment_resources = environment_details['EnvironmentResources'] 38 | 39 | environment_db_names = [] 40 | for glue_db_key in P_ENVIRONMENT_DBS_KEYS: 41 | 42 | if glue_db_key in environment_resources: 43 | 44 | glue_db_name = environment_resources[glue_db_key] 45 | lakeformation_response = lakeformation.add_lf_tags_to_resource( 46 | Resource= { 47 | 'Database': { 48 | 'Name': glue_db_name 49 | } 50 | }, 51 | LFTags= [ 52 | { 53 | 'TagKey': P_LAKEFORMATION_TAG_KEY, 54 | 'TagValues': [P_LAKEFORMATION_TAG_VALUE] 55 | }, 56 | ] 57 | ) 58 | 59 | lakeformation_response = json.loads(json.dumps(lakeformation_response, default=json_datetime_encoder)) 60 | environment_db_names.append(glue_db_name) 61 | 62 | response = { 63 | 'environment_db_names': environment_db_names, 64 | 'lakeformation_tag_key': P_LAKEFORMATION_TAG_KEY, 65 | 'lakeformation_tag_value': P_LAKEFORMATION_TAG_VALUE 66 | } 67 | 68 | return response 69 | 70 | 71 | def json_datetime_encoder(obj): 72 | """ Complementary function to transform dict objects delivered by AWS API into JSONs """ 73 | if isinstance(obj, (datetime)): return obj.strftime("%Y-%m-%dT%H:%M:%S") -------------------------------------------------------------------------------- /src/producer/code/lambda/delete_keep_subscription_secret/delete_keep_subscription_secret.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import boto3 4 | 5 | from datetime import datetime 6 | 7 | # Constant: Represents the recovery window in days that will be assigned when scheduling secret deletion 8 | RECOVERY_WINDOW_IN_DAYS = os.getenv('RECOVERY_WINDOW_IN_DAYS') 9 | 10 | secrets_manager = boto3.client('secretsmanager') 11 | 12 | def handler(event, context): 13 | """ Function handler: Function that will delete or keep a subscription secret by 1/ Scheduling its deletion. 14 | Deletion will depend if project user was deleted previously or not. 15 | 16 | Parameters 17 | ---------- 18 | event: dict - Input event dict containing: 19 | RevokeSubscriptionDetails: dict - Dict containing revoke subscription details including: 20 | SecretName: str - Name of producer local subscription secret. 21 | DeleteSecret: str - If subscription user was deleted so that associated secret is deleted as well. 22 | 23 | context: dict - Input context. Not used on function 24 | 25 | Returns 26 | ------- 27 | response: dict - Dict with response details including: 28 | secret_name: str - Name of the deleted secret local to the producer account 29 | secret_arn: str - Arn of the deleted secret local to the producer account 30 | secret_deleted: str - 'true' or 'false' depending if secret was deleted or not 31 | secret_deletion_date: str - Date of secret deletion 32 | secret_recovery_window_in_days: str - Secret recovery window in days 33 | """ 34 | 35 | revoke_subscription_details = event['RevokeSubscriptionDetails'] 36 | revoke_subscription_secret_name = revoke_subscription_details['SecretName'] 37 | revoke_subscription_delete_secret = revoke_subscription_details['DeleteSecret'] 38 | 39 | if revoke_subscription_delete_secret: 40 | secrets_manager_response = delete_secret(revoke_subscription_secret_name) 41 | else: 42 | secrets_manager_response = secrets_manager.describe_secret( 43 | SecretId= revoke_subscription_secret_name 44 | ) 45 | 46 | response = { 47 | 'secret_name': secrets_manager_response['Name'], 48 | 'secret_arn': secrets_manager_response['ARN'], 49 | 'secret_deleted': 'true' if revoke_subscription_delete_secret else 'false', 50 | 'secret_deletion_date': secrets_manager_response['DeletionDate'] if revoke_subscription_delete_secret else 'None', 51 | 'secret_recovery_window_in_days': RECOVERY_WINDOW_IN_DAYS if revoke_subscription_delete_secret else 'None' 52 | } 53 | 54 | return response 55 | 56 | 57 | def delete_secret(secret_name): 58 | """ Complementary function to schedule deletion of a local secret""" 59 | 60 | secrets_manager_response = secrets_manager.delete_secret( 61 | SecretId=secret_name, 62 | RecoveryWindowInDays= int(RECOVERY_WINDOW_IN_DAYS) 63 | ) 64 | 65 | secrets_manager_response = json.loads(json.dumps(secrets_manager_response, default=json_datetime_encoder)) 66 | 67 | return secrets_manager_response 68 | 69 | 70 | def json_datetime_encoder(obj): 71 | """ Complementary function to transform dict objects delivered by AWS API into JSONs """ 72 | if isinstance(obj, (datetime)): return obj.strftime("%Y-%m-%dT%H:%M:%S") -------------------------------------------------------------------------------- /src/producer/code/lambda/get_connection_details/get_connection_details.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import boto3 4 | from datetime import datetime 5 | 6 | # Constant: Represents the account id 7 | ACCOUNT_ID = os.getenv('ACCOUNT_ID') 8 | 9 | # Constant: Represents the region 10 | REGION = os.getenv('REGION') 11 | 12 | glue = boto3.client('glue') 13 | 14 | def handler(event, context): 15 | """ Function handler: Function that will retrieve subscription's data asset source glue connection details. 1/ Will retrieve data asset metadata from glue data catalog, then 16 | 2/ will retrieve connection details, including credentials secret name/arn and source data asset name 17 | 18 | Parameters 19 | ---------- 20 | event: dict - Input event dict containing: 21 | SubscriptionDetails: dict - Dict containing subscription details including: 22 | AssetDetails.GlueTableDetails.DatabaseName: str - Name of the glue data catalog source database where subscribed asset is located 23 | AssetDetails.GlueTableDetails.TableName: str - Name of the table in the glue catalog that the subscription is pointing to 24 | 25 | context: dict - Input context. Not used on function 26 | 27 | Returns 28 | ------- 29 | glue_connection_details: dict - Dict with glue connection details: 30 | ConnectionType: str - Type of the glue connection associated to the subscribed asset. 31 | ConnectionArn: str - ARN of the glue connection associated to the subscribed asset. 32 | ConnectionName: str - Name of the glue connection associated to the subscribed asset. 33 | ConnectionProperties: dict - Dict with connection properties details including: 34 | JDBC_ENFORCE_SSL: str - 'True' or 'False' depending if SSL is enforced by connection 35 | JDBC_CONNECTION_URL: str - Connection URL to connect to source 36 | SECRET_ID: str - ARN of the secret with credential used by glue connection to connect to source database 37 | ConnectionAssetName: str - Name of the asset on source database mapped to subscribed table in Glue catalog 38 | ConnectionCrawlerName: str - Name of the crawler the retrieved subscribed asset 39 | """ 40 | subscription_details = event['SubscriptionDetails'] 41 | 42 | glue_database_name = subscription_details['AssetDetails']['GlueTableDetails']['DatabaseName'] 43 | glue_table_name = subscription_details['AssetDetails']['GlueTableDetails']['TableName'] 44 | 45 | glue_response = glue.get_table( 46 | DatabaseName=glue_database_name, 47 | Name=glue_table_name 48 | ) 49 | 50 | glue_response = json.loads(json.dumps(glue_response, default=json_datetime_encoder)) 51 | glue_table_details = glue_response['Table'] 52 | glue_table_database_asset_name = glue_table_details['StorageDescriptor']['Location'] 53 | glue_crawler_name = glue_table_details['Parameters']['UPDATED_BY_CRAWLER'] 54 | glue_connection_name = glue_table_details['Parameters']['connectionName'] 55 | glue_connection_arn = f'arn:aws:glue:{REGION}:{ACCOUNT_ID}:connection/{glue_connection_name}' 56 | 57 | glue_response = glue.get_connection( 58 | Name=glue_connection_name 59 | ) 60 | 61 | glue_response = json.loads(json.dumps(glue_response, default=json_datetime_encoder)) 62 | glue_connection_type = glue_response['Connection']['ConnectionType'] 63 | glue_connection_properties = glue_response['Connection']['ConnectionProperties'] 64 | 65 | glue_connection_details = { 66 | 'ConnectionType': glue_connection_type, 67 | 'ConnectionArn': glue_connection_arn, 68 | 'ConnectionName': glue_connection_name, 69 | 'ConnectionProperties': glue_connection_properties, 70 | 'ConnectionAssetName': glue_table_database_asset_name, 71 | 'ConnectionCrawlerName': glue_crawler_name 72 | } 73 | 74 | return glue_connection_details 75 | 76 | 77 | def json_datetime_encoder(obj): 78 | """ Complementary function to transform dict objects delivered by AWS API into JSONs """ 79 | if isinstance(obj, (datetime)): return obj.strftime("%Y-%m-%dT%H:%M:%S") -------------------------------------------------------------------------------- /src/producer/code/lambda/share_subscription_secret/share_subscription_secret.py: -------------------------------------------------------------------------------- 1 | import json 2 | import boto3 3 | import os 4 | 5 | # Constant: Represents the account id 6 | ACCOUNT_ID = os.getenv('ACCOUNT_ID') 7 | 8 | # Constant: Name of the role in consumer account that will be able to retrieve shared secret 9 | C_ROLE_NAME = os.getenv('C_ROLE_NAME') 10 | 11 | secrets_manager = boto3.client('secretsmanager') 12 | 13 | def handler(event, context): 14 | """ Function handler: Function that will share subscription secret by 1/ Adding a resource policy that allows access by consumer environment account (specific target role) 15 | when secret is new. When is an already shared secret it won't do anything. 16 | 17 | Parameters 18 | ---------- 19 | event: dict - Input event dict containing: 20 | SubscriptionDetails: dict - Dict containing details including: 21 | ConsumerProjectDetails.AccountId: str - Account id of the DataZone environment subscribing to the data asset 22 | GrantSubscriptionDetails: dict - Dict containing grant subscription details including: 23 | SecretName: str - Name of subscription secret to be shared. 24 | NewSubscriptionSecret: bool - If subscription secret was newly created or not (reused and already shared). 25 | 26 | context: dict - Input context. Not used on function 27 | 28 | Returns 29 | ------- 30 | response: dict - Dict with response details including: 31 | secret_arn: str - Arn of the copied secret local to the consumer account 32 | secret_name: str - Name of the copied secret local to the consumer account 33 | new_subscription_secret: bool - If subscription secret was newly created or not (reused and already shared). 34 | subscription_consumer_role: str - ARN of the role that can access the secret on the consumer account 35 | """ 36 | subscription_details = event['SubscriptionDetails'] 37 | grant_subscription_details = event['GrantSubscriptionDetails'] 38 | 39 | consumer_project_details = subscription_details['ConsumerProjectDetails'] 40 | consumer_account_id = consumer_project_details['AccountId'] 41 | subscription_consumer_roles = [f'arn:aws:iam::{consumer_account_id}:role/{C_ROLE_NAME}'] 42 | 43 | subscription_secret_name = grant_subscription_details['SecretName'] 44 | new_subscription_secret = grant_subscription_details['NewSubscriptionSecret'] 45 | 46 | subscription_secret_resource_policy = { 47 | "Version": "2012-10-17", 48 | "Statement": [ 49 | { 50 | "Effect": "Allow", 51 | "Principal": { 52 | "AWS": f"arn:aws:iam::{ACCOUNT_ID}:root" 53 | }, 54 | "Action": "secretsmanager:*", 55 | "Resource": "*" 56 | }, 57 | { 58 | "Effect": "Allow", 59 | "Principal": { 60 | "AWS": subscription_consumer_roles 61 | }, 62 | "Action": "secretsmanager:GetSecretValue", 63 | "Resource": "*" 64 | } 65 | ] 66 | } 67 | 68 | secrets_manager_response = secrets_manager.put_resource_policy( 69 | SecretId= subscription_secret_name, 70 | ResourcePolicy= json.dumps(subscription_secret_resource_policy) 71 | ) 72 | 73 | response = { 74 | 'secret_name': secrets_manager_response['Name'], 75 | 'secret_arn': secrets_manager_response['ARN'], 76 | 'new_subscription_secret': new_subscription_secret, 77 | 'subscription_consumer_roles': subscription_consumer_roles 78 | } 79 | 80 | return response 81 | -------------------------------------------------------------------------------- /src/producer/code/stepfunctions/producer_manage_subscription_grant_workflow.asl.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "State machine to orchestrate activities to manage dataset subscription grants on the producer side", 3 | "StartAt": "Get connection details", 4 | "States": { 5 | "Get connection details": { 6 | "Type": "Task", 7 | "Next": "Which connection type?", 8 | "Parameters": { 9 | "FunctionName": "${p_get_connection_details_lambda_arn}", 10 | "Payload.$": "$" 11 | }, 12 | "Resource": "arn:aws:states:::lambda:invoke", 13 | "ResultPath": "$.ConnectionDetails", 14 | "ResultSelector": { 15 | "ConnectionType.$": "$.Payload.ConnectionType", 16 | "ConnectionArn.$": "$.Payload.ConnectionArn", 17 | "ConnectionName.$": "$.Payload.ConnectionName", 18 | "ConnectionProperties.$": "$.Payload.ConnectionProperties", 19 | "ConnectionAssetName.$": "$.Payload.ConnectionAssetName", 20 | "ConnectionCrawlerName.$": "$.Payload.ConnectionCrawlerName" 21 | } 22 | }, 23 | "Which connection type?": { 24 | "Type": "Choice", 25 | "Default": "Unsupported connection type", 26 | "Choices": [ 27 | { 28 | "Next": "Grant JDBC subscription", 29 | "StringEquals": "JDBC", 30 | "Variable": "$.ConnectionDetails.ConnectionType" 31 | } 32 | ] 33 | }, 34 | "Unsupported connection type": { 35 | "Type": "Fail", 36 | "Error": "Unsupported connection type" 37 | }, 38 | "Grant JDBC subscription": { 39 | "Type": "Task", 40 | "Next": "Share subscription secret", 41 | "Parameters": { 42 | "FunctionName": "${p_grant_jdbc_subscription_lambda_arn}", 43 | "Payload.$": "$" 44 | }, 45 | "Resource": "arn:aws:states:::lambda:invoke", 46 | "ResultPath": "$.GrantSubscriptionDetails", 47 | "ResultSelector": { 48 | "ConnectionArn.$": "$.Payload.glue_connection_arn", 49 | "DatazoneConsumerEnvironmentId.$": "$.Payload.datazone_consumer_environment_id", 50 | "DatazoneConsumerProjectId.$": "$.Payload.datazone_consumer_project_id", 51 | "DatazoneDomainId.$": "$.Payload.datazone_domain_id", 52 | "SecretArn.$": "$.Payload.secret_arn", 53 | "SecretName.$": "$.Payload.secret_name", 54 | "DataAssets.$": "$.Payload.data_assets", 55 | "OwnerAccount.$": "$.Payload.owner_account", 56 | "OwnerRegion.$": "$.Payload.owner_region", 57 | "LastUpdated.$": "$.Payload.last_updated", 58 | "NewSubscriptionSecret.$": "$.Payload.new_subscription_secret" 59 | } 60 | }, 61 | "Share subscription secret": { 62 | "Type": "Task", 63 | "End": true, 64 | "Parameters": { 65 | "FunctionName": "${p_share_subscription_secret_lambda_arn}", 66 | "Payload.$": "$" 67 | }, 68 | "Resource": "arn:aws:states:::lambda:invoke", 69 | "ResultPath": "$.ShareSubscriptionSecretDetails", 70 | "ResultSelector": { 71 | "SecretName.$": "$.Payload.secret_name", 72 | "SecretArn.$": "$.Payload.secret_arn", 73 | "NewSubscriptionSecret.$": "$.Payload.new_subscription_secret", 74 | "SubscriptionConsumerRoles.$": "$.Payload.subscription_consumer_roles" 75 | } 76 | } 77 | } 78 | } -------------------------------------------------------------------------------- /src/producer/code/stepfunctions/producer_manage_subscription_revoke_workflow.asl.json: -------------------------------------------------------------------------------- 1 | { 2 | "Comment": "State machine to orchestrate activities to manage dataset subscription revocations on the producer side", 3 | "StartAt": "Get connection details", 4 | "States": { 5 | "Get connection details": { 6 | "Type": "Task", 7 | "Next": "Which connection type?", 8 | "Parameters": { 9 | "FunctionName": "${p_get_connection_details_lambda_arn}", 10 | "Payload.$": "$" 11 | }, 12 | "Resource": "arn:aws:states:::lambda:invoke", 13 | "ResultPath": "$.ConnectionDetails", 14 | "ResultSelector": { 15 | "ConnectionType.$": "$.Payload.ConnectionType", 16 | "ConnectionArn.$": "$.Payload.ConnectionArn", 17 | "ConnectionName.$": "$.Payload.ConnectionName", 18 | "ConnectionProperties.$": "$.Payload.ConnectionProperties", 19 | "ConnectionAssetName.$": "$.Payload.ConnectionAssetName", 20 | "ConnectionCrawlerName.$": "$.Payload.ConnectionCrawlerName" 21 | } 22 | }, 23 | "Which connection type?": { 24 | "Type": "Choice", 25 | "Default": "Unsupported connection type", 26 | "Choices": [ 27 | { 28 | "Next": "Revoke JDBC subscription", 29 | "StringEquals": "JDBC", 30 | "Variable": "$.ConnectionDetails.ConnectionType" 31 | } 32 | ] 33 | }, 34 | "Unsupported connection type": { 35 | "Type": "Fail", 36 | "Error": "Unsupported connection type" 37 | }, 38 | "Revoke JDBC subscription": { 39 | "Type": "Task", 40 | "Next": "Delete / keep subscription secret", 41 | "Parameters": { 42 | "FunctionName": "${p_revoke_jdbc_subscription_lambda_arn}", 43 | "Payload.$": "$" 44 | }, 45 | "Resource": "arn:aws:states:::lambda:invoke", 46 | "ResultPath": "$.RevokeSubscriptionDetails", 47 | "ResultSelector": { 48 | "ConnectionArn.$": "$.Payload.glue_connection_arn", 49 | "DatazoneConsumerEnvironmentId.$": "$.Payload.datazone_consumer_environment_id", 50 | "DatazoneConsumerProjectId.$": "$.Payload.datazone_consumer_project_id", 51 | "DatazoneDomainId.$": "$.Payload.datazone_domain_id", 52 | "SecretArn.$": "$.Payload.secret_arn", 53 | "SecretName.$": "$.Payload.secret_name", 54 | "DataAssets.$": "$.Payload.data_assets", 55 | "OwnerAccount.$": "$.Payload.owner_account", 56 | "LastUpdated.$": "$.Payload.last_updated", 57 | "DeleteSecret.$": "$.Payload.delete_secret" 58 | } 59 | }, 60 | "Delete / keep subscription secret": { 61 | "Type": "Task", 62 | "End": true, 63 | "Parameters": { 64 | "FunctionName": "${p_delete_keep_subscription_secret_lambda_arn}", 65 | "Payload.$": "$" 66 | }, 67 | "Resource": "arn:aws:states:::lambda:invoke", 68 | "ResultPath": "$.DeleteKeepSubscriptionSecretDetails", 69 | "ResultSelector": { 70 | "SecretName.$": "$.Payload.secret_name", 71 | "SecretArn.$": "$.Payload.secret_arn", 72 | "SecretDeleted.$": "$.Payload.secret_deleted", 73 | "SecretDeletionDate.$": "$.Payload.secret_deletion_date", 74 | "SecretRecoveryWindowInDays.$": "$.Payload.secret_recovery_window_in_days" 75 | } 76 | } 77 | } 78 | } -------------------------------------------------------------------------------- /src/producer/constructs/glue_connection.py: -------------------------------------------------------------------------------- 1 | from config.common.global_vars import GLOBAL_VARIABLES 2 | 3 | from aws_cdk import ( 4 | Environment, 5 | Tags, 6 | aws_glue as glue 7 | ) 8 | 9 | from constructs import Construct 10 | 11 | class GlueJDBCConnectorConstruct(Construct): 12 | """ Class to represent a glue connection for JDBC sources.""" 13 | 14 | def __init__(self, scope: Construct, construct_id: str, account_props: dict, connection_props: dict, env: Environment, **kwargs) -> None: 15 | """ Class Constructor. Will create a JDBC glue connection. 16 | Resources will be created based on properties specified as parameter. 17 | 18 | Parameters 19 | ---------- 20 | account_props : dict 21 | dict with common properties for account. 22 | For more details check config/account/a__config.py documentation and examples. 23 | 24 | connection_props : dict 25 | dict with required properties for workflow creation, including: 26 | datazone_environment_id: str - Id of the Amazon DataZone environment that will be associated to the glue connection 27 | glue_connection: dict - Dict containing connection specific properties including: 28 | name: str - Name to assign to the glue connection. 29 | engine: str - Engine of the JDBC source 30 | host: str - Host url of the JDBC source 31 | port: str - Port of the JDBC source 32 | db_name: str - Database name of the JDBC source 33 | ssl: str - 'True' or 'False' depending if ssl needs to be enforced by connection 34 | secret_arn_suffix: str - Secret arn suffix with credentials of admin user of the JDBC source. Note that is not the name but the id as specified in ARN. 35 | availability_zone: str - Availability zone where glue connection will be connecting from 36 | security_group_ids: list - List of security group ids to be associated to the glue connection 37 | subnet_id: str - Subnet id where glue connection will be connecting from 38 | 39 | env: Environment 40 | Environment object with region and account details 41 | """ 42 | 43 | super().__init__(scope, construct_id, **kwargs) 44 | account_id, region = account_props['account_id'], account_props['region'] 45 | 46 | datazone_domain_id = connection_props['datazone_domain_id'] 47 | datazone_project_id = connection_props['datazone_project_id'] 48 | datazone_environment_id = connection_props['datazone_environment_id'] 49 | glue_connection_props = connection_props['glue_connection'] 50 | glue_connection_engine = glue_connection_props['engine'] 51 | glue_connection_host = glue_connection_props['host'] 52 | glue_connection_port = glue_connection_props['port'] 53 | glue_connection_db_name = glue_connection_props['db_name'] 54 | glue_connection_ssl = glue_connection_props['ssl'] 55 | 56 | 57 | glue_connection_jdbc_url = f'jdbc:{glue_connection_engine}://{glue_connection_host}:{glue_connection_port}/{glue_connection_db_name}' 58 | glue_connection_secret_arn_suffix = glue_connection_props['secret_arn_suffix'] 59 | glue_connection_secret_arn = f'arn:aws:secretsmanager:{region}:{account_id}:secret:{glue_connection_secret_arn_suffix}' 60 | 61 | self.glue_connection = glue.CfnConnection( 62 | scope= self, 63 | id= 'glue_connection', 64 | catalog_id= account_id, 65 | connection_input=glue.CfnConnection.ConnectionInputProperty( 66 | name= glue_connection_props["name"], 67 | connection_type= 'JDBC', 68 | 69 | connection_properties= { 70 | 'JDBC_CONNECTION_URL': glue_connection_jdbc_url, 71 | 'SECRET_ID': glue_connection_secret_arn, 72 | 'JDBC_ENFORCE_SSL': glue_connection_ssl 73 | }, 74 | 75 | physical_connection_requirements= glue.CfnConnection.PhysicalConnectionRequirementsProperty( 76 | availability_zone= glue_connection_props["availability_zone"], 77 | security_group_id_list= glue_connection_props["security_group_ids"].split(','), 78 | subnet_id= glue_connection_props["subnet_id"] 79 | ) 80 | ) 81 | ) 82 | 83 | Tags.of(self.glue_connection).add('AmazonDataZoneDomain', datazone_domain_id) 84 | Tags.of(self.glue_connection).add('AmazonDataZoneProject', datazone_project_id) 85 | Tags.of(self.glue_connection).add('AmazonDataZoneEnvironment', datazone_environment_id) 86 | 87 | 88 | class GlueJDBCConnectorWithCrawlersConstruct(GlueJDBCConnectorConstruct): 89 | """ Class to represent a glue connection for JDBC sources with a set of Glue crawlers associated to it. 90 | Extends glue connection construct. 91 | """ 92 | 93 | def __init__(self, scope: Construct, construct_id: str, account_props: dict, connection_props: dict, env: Environment, **kwargs) -> None: 94 | """ Class Constructor. Will create a JDBC glue connection. 95 | Resources will be created based on properties specified as parameter. 96 | 97 | Parameters 98 | ---------- 99 | account_props : dict 100 | dict with common properties for account. 101 | For more details check config/account/a__config.py documentation and examples. 102 | 103 | connection_props : dict 104 | dict with required properties for workflow creation, including: 105 | datazone_environment_id: str - Id of the Amazon DataZone environment that will be associated to the glue connection 106 | glue_connection: dict - Dict containing connection specific properties (refer to super class for more details) 107 | glue_crawlers: list - List containing a list of dicts, each with properties of a crawler to associate to the connection, including: 108 | name: str - Name to assign to the glue crawler. 109 | catalog_database_name: str - Name of the glue data catalog database where schemas will be written 110 | table_prefix: str - Prefix to include on table schema names when written in glue catalog 111 | source_db_path: str - Path within source database from which assets will be crawled 112 | 113 | env: Environment 114 | Environment object with region and account details 115 | """ 116 | 117 | super().__init__(scope, construct_id, account_props, connection_props, env, **kwargs) 118 | account_id = account_props['account_id'] 119 | 120 | datazone_domain_id = connection_props['datazone_domain_id'] 121 | datazone_project_id = connection_props['datazone_project_id'] 122 | datazone_environment_id = connection_props['datazone_environment_id'] 123 | glue_connection_props = connection_props['glue_connection'] 124 | glue_crawlers_props = connection_props['glue_crawlers'] if 'glue_crawlers' in connection_props else None 125 | 126 | a_common_glue_role_name = GLOBAL_VARIABLES['account']['a_common_glue_role_name'] 127 | a_common_glue_role_arn = f'arn:aws:iam::{account_id}:role/{a_common_glue_role_name}' 128 | 129 | glue_crawlers = [] 130 | if glue_crawlers_props: 131 | for glue_crawler_id, glue_crawler_props in glue_crawlers_props.items(): 132 | 133 | glue_crawler = glue.CfnCrawler( 134 | scope= self, 135 | id= glue_crawler_id, 136 | name= glue_crawler_props["name"], 137 | role= a_common_glue_role_arn, 138 | database_name= glue_crawler_props["catalog_database_name"], 139 | table_prefix= glue_crawler_props["table_prefix"], 140 | targets= glue.CfnCrawler.TargetsProperty( 141 | jdbc_targets= [ 142 | glue.CfnCrawler.JdbcTargetProperty( 143 | connection_name= glue_connection_props['name'], 144 | path=glue_crawler_props["source_db_path"] 145 | ) 146 | ] 147 | ), 148 | tags= { 149 | 'AmazonDataZoneDomain': datazone_domain_id, 150 | 'AmazonDataZoneProject': datazone_project_id, 151 | 'AmazonDataZoneEnvironment': datazone_environment_id 152 | } 153 | ) 154 | 155 | glue_crawler.node.add_dependency(self.glue_connection) 156 | glue_crawlers.append(glue_crawler) -------------------------------------------------------------------------------- /src/producer/constructs/producer_subscription_grant_workflow.py: -------------------------------------------------------------------------------- 1 | from config.common.global_vars import GLOBAL_VARIABLES 2 | 3 | from aws_cdk import ( 4 | Environment, 5 | Fn, 6 | RemovalPolicy, 7 | aws_lambda as lambda_, 8 | aws_stepfunctions as stepfunctions, 9 | aws_ec2 as ec2, 10 | aws_logs as logs 11 | ) 12 | 13 | from os import path; 14 | 15 | from constructs import Construct 16 | 17 | class ProducerManageSubscriptionGrantWorkflowConstruct(Construct): 18 | """ Class to represent the workflow that will execute in the producer account after a Amazon DataZone subscription is approved. 19 | The workflow will grant access to specified dataset in JDBC source to a new/existing user associated to the subscribing project. Then it will create a secret (if non existent) with project user credentials and share it (only with workflow access) to consumer account. 20 | Metadata will be updated in dynamodb tables hosted on governance account. 21 | Actions involving governance account resources will be done via cross-account access. 22 | """ 23 | 24 | def __init__(self, scope: Construct, construct_id: str, account_props: dict, workflow_props: dict, common_constructs: dict, env: Environment, **kwargs) -> None: 25 | """ Class Constructor. Will create a workflow (state machine and belonging lambda functions) based on properties specified as parameter. 26 | 27 | Parameters 28 | ---------- 29 | account_props : dict 30 | dict with common properties for account. 31 | For more details check config/account/a__config.py documentation and examples. 32 | 33 | workflow_props : dict 34 | dict with required properties for workflow creation. 35 | For more details check config/account/a__config.py documentation and examples. 36 | 37 | common_constructs: dic 38 | dict with constructs common to the account. Created in and output of account common stack. 39 | 40 | env: Environment 41 | Environment object with region and account details 42 | """ 43 | 44 | super().__init__(scope, construct_id, **kwargs) 45 | account_id, region = account_props['account_id'], account_props['region'] 46 | 47 | # ---------------- VPC ------------------------ 48 | p_lambda_vpc = ec2.Vpc.from_vpc_attributes( 49 | scope= self, 50 | id= 'p_lambda_vpc', 51 | vpc_id= workflow_props['vpc_id'], 52 | availability_zones = Fn.get_azs(), 53 | private_subnet_ids = workflow_props['vpc_private_subnet_ids'] 54 | ) 55 | 56 | p_lambda_security_group_ids = workflow_props['vpc_security_group_ids'] 57 | p_lambda_security_groups = [] 58 | 59 | for index, p_lambda_security_group_id in enumerate(p_lambda_security_group_ids): 60 | p_lambda_security_group = ec2.SecurityGroup.from_security_group_id( 61 | scope= self, 62 | id= f'p_lambda_security_group_{(index + 1):02d}', 63 | security_group_id= p_lambda_security_group_id, 64 | ) 65 | 66 | p_lambda_security_groups.append(p_lambda_security_group) 67 | 68 | # ---------------- Lambda ------------------------ 69 | p_get_connection_details_lambda = lambda_.Function( 70 | scope= self, 71 | id= 'p_get_connection_details_lambda', 72 | function_name= 'dz_conn_p_get_connection_details', 73 | runtime= lambda_.Runtime.PYTHON_3_11, 74 | code=lambda_.Code.from_asset(path.join('src/producer/code/lambda', "get_connection_details")), 75 | handler= "get_connection_details.handler", 76 | role= common_constructs['a_common_lambda_role'], 77 | environment= { 78 | 'ACCOUNT_ID': account_id, 79 | 'REGION': region 80 | } 81 | ) 82 | 83 | p_grant_jdbc_subscription_lambda = lambda_.Function( 84 | scope= self, 85 | id= 'p_grant_jdbc_subscription_lambda', 86 | function_name= 'dz_conn_p_grant_jdbc_subscription', 87 | runtime= lambda_.Runtime.PYTHON_3_8, 88 | code=lambda_.Code.from_asset(path.join('src/producer/code/lambda', "grant_jdbc_subscription")), 89 | handler= "grant_jdbc_subscription.handler", 90 | layers= [ 91 | common_constructs['p_aws_sdk_pandas_layer'], 92 | common_constructs['p_pyodbc_layer'], 93 | common_constructs['p_oracledb_layer'] 94 | ], 95 | role= common_constructs['a_common_lambda_role'], 96 | vpc= p_lambda_vpc, 97 | security_groups= p_lambda_security_groups, 98 | environment= { 99 | 'G_ACCOUNT_ID': GLOBAL_VARIABLES['governance']['g_account_number'], 100 | 'G_CROSS_ACCOUNT_ASSUME_ROLE_NAME': GLOBAL_VARIABLES['governance']['g_cross_account_assume_role_name'], 101 | 'G_P_SOURCE_SUBSCRIPTIONS_TABLE_NAME': GLOBAL_VARIABLES['governance']['g_p_source_subscriptions_table_name'], 102 | 'A_COMMON_KEY_ALIAS': common_constructs['a_common_key_alias'], 103 | 'ACCOUNT_ID': account_id, 104 | 'REGION': region 105 | } 106 | ) 107 | 108 | p_share_subscription_secret_lambda = lambda_.Function( 109 | scope= self, 110 | id= 'p_share_subscription_secret_lambda', 111 | function_name= 'dz_conn_p_share_subscription_secret', 112 | runtime= lambda_.Runtime.PYTHON_3_11, 113 | code=lambda_.Code.from_asset(path.join('src/producer/code/lambda', "share_subscription_secret")), 114 | handler= "share_subscription_secret.handler", 115 | role= common_constructs['a_common_lambda_role'], 116 | environment= { 117 | 'ACCOUNT_ID': account_id, 118 | 'C_ROLE_NAME': GLOBAL_VARIABLES['account']['a_common_lambda_role_name'] 119 | } 120 | ) 121 | 122 | # ---------------- Step Functions ------------------------ 123 | p_manage_subscription_grant_state_machine_name = GLOBAL_VARIABLES['producer']['p_manage_subscription_grant_state_machine_name'] 124 | 125 | p_manage_subscription_grant_state_machine_logs = logs.LogGroup( 126 | scope= self, 127 | id= 'p_manage_subscription_grant_state_machine_logs', 128 | log_group_name=f'/aws/step-functions/{p_manage_subscription_grant_state_machine_name}', 129 | removal_policy=RemovalPolicy.DESTROY 130 | ) 131 | 132 | p_manage_subscription_grant_state_machine = stepfunctions.StateMachine( 133 | scope= self, 134 | id= 'p_manage_subscription_grant_state_machine', 135 | state_machine_name= p_manage_subscription_grant_state_machine_name, 136 | definition_body=stepfunctions.DefinitionBody.from_file('src/producer/code/stepfunctions/producer_manage_subscription_grant_workflow.asl.json'), 137 | definition_substitutions= { 138 | 'p_get_connection_details_lambda_arn': p_get_connection_details_lambda.function_arn, 139 | 'p_grant_jdbc_subscription_lambda_arn': p_grant_jdbc_subscription_lambda.function_arn, 140 | 'p_share_subscription_secret_lambda_arn': p_share_subscription_secret_lambda.function_arn 141 | }, 142 | role= common_constructs['a_common_sf_role'], 143 | logs= stepfunctions.LogOptions( 144 | destination=p_manage_subscription_grant_state_machine_logs, 145 | level=stepfunctions.LogLevel.ALL 146 | ), 147 | tracing_enabled=True 148 | ) 149 | 150 | -------------------------------------------------------------------------------- /src/producer/constructs/producer_subscription_revoke_workflow.py: -------------------------------------------------------------------------------- 1 | from config.common.global_vars import GLOBAL_VARIABLES 2 | 3 | from aws_cdk import ( 4 | Environment, 5 | Fn, 6 | RemovalPolicy, 7 | aws_lambda as lambda_, 8 | aws_stepfunctions as stepfunctions, 9 | aws_ec2 as ec2, 10 | aws_logs as logs 11 | ) 12 | 13 | from os import path; 14 | 15 | from constructs import Construct 16 | 17 | class ProducerManageSubscriptionRevokeWorkflowConstruct(Construct): 18 | """ Class to represent the workflow that will execute in the producer account after a Amazon DataZone subscription is revoked. 19 | The workflow will remove access to specified dataset in JDBC source to a existing user associated to the subscribing project. Then it will delete the shared secret if no additional assets are associated to it. 20 | Metadata will be updated in dynamodb tables hosted on governance account. 21 | Actions involving governance account resources will be done via cross-account access. 22 | """ 23 | 24 | def __init__(self, scope: Construct, construct_id: str, account_props: dict, workflow_props: dict, common_constructs: dict, env: Environment, **kwargs) -> None: 25 | """ Class Constructor. Will create a workflow (state machine and belonging lambda functions) based on properties specified as parameter. 26 | 27 | Parameters 28 | ---------- 29 | account_props : dict 30 | dict with common properties for account. 31 | For more details check config/account/a__config.py documentation and examples. 32 | 33 | workflow_props : dict 34 | dict with required properties for workflow creation. 35 | For more details check config/account/a__config.py documentation and examples. 36 | 37 | common_constructs: dic 38 | dict with constructs common to the account. Created in and output of account common stack. 39 | 40 | env: Environment 41 | Environment object with region and account details 42 | """ 43 | 44 | super().__init__(scope, construct_id, **kwargs) 45 | account_id, region = account_props['account_id'], account_props['region'] 46 | 47 | # ---------------- VPC ------------------------ 48 | p_lambda_vpc = ec2.Vpc.from_vpc_attributes( 49 | scope= self, 50 | id= 'p_lambda_vpc', 51 | vpc_id= workflow_props['vpc_id'], 52 | availability_zones = Fn.get_azs(), 53 | private_subnet_ids = workflow_props['vpc_private_subnet_ids'] 54 | ) 55 | 56 | p_lambda_security_group_ids = workflow_props['vpc_security_group_ids'] 57 | p_lambda_security_groups = [] 58 | 59 | for index, p_lambda_security_group_id in enumerate(p_lambda_security_group_ids): 60 | p_lambda_security_group = ec2.SecurityGroup.from_security_group_id( 61 | scope= self, 62 | id= f'p_lambda_security_group_{(index + 1):02d}', 63 | security_group_id= p_lambda_security_group_id, 64 | ) 65 | 66 | p_lambda_security_groups.append(p_lambda_security_group) 67 | 68 | # ---------------- Lambda ------------------------ 69 | p_get_connection_details_lambda = lambda_.Function.from_function_name( 70 | scope= self, 71 | id= 'p_get_connection_details_lambda', 72 | function_name= 'dz_conn_p_get_connection_details' 73 | ) 74 | 75 | p_revoke_jdbc_subscription_lambda = lambda_.Function( 76 | scope= self, 77 | id= 'p_revoke_jdbc_subscription_lambda', 78 | function_name= 'dz_conn_p_revoke_jdbc_subscription', 79 | runtime= lambda_.Runtime.PYTHON_3_8, 80 | code=lambda_.Code.from_asset(path.join('src/producer/code/lambda', "revoke_jdbc_subscription")), 81 | handler= "revoke_jdbc_subscription.handler", 82 | layers= [ 83 | common_constructs['p_aws_sdk_pandas_layer'], 84 | common_constructs['p_pyodbc_layer'], 85 | common_constructs['p_oracledb_layer'] 86 | ], 87 | role= common_constructs['a_common_lambda_role'], 88 | vpc= p_lambda_vpc, 89 | security_groups= p_lambda_security_groups, 90 | environment= { 91 | 'G_ACCOUNT_ID': GLOBAL_VARIABLES['governance']['g_account_number'], 92 | 'G_CROSS_ACCOUNT_ASSUME_ROLE_NAME': GLOBAL_VARIABLES['governance']['g_cross_account_assume_role_name'], 93 | 'G_P_SOURCE_SUBSCRIPTIONS_TABLE_NAME': GLOBAL_VARIABLES['governance']['g_p_source_subscriptions_table_name'], 94 | 'ACCOUNT_ID': account_id 95 | } 96 | ) 97 | 98 | p_delete_keep_subscription_secret_lambda = lambda_.Function( 99 | scope= self, 100 | id= 'p_delete_keep_subscription_secret_lambda', 101 | function_name= 'dz_conn_p_delete_keep_subscription_secret', 102 | runtime= lambda_.Runtime.PYTHON_3_11, 103 | code=lambda_.Code.from_asset(path.join('src/producer/code/lambda', "delete_keep_subscription_secret")), 104 | handler= "delete_keep_subscription_secret.handler", 105 | environment= { 106 | 'RECOVERY_WINDOW_IN_DAYS': workflow_props['secret_recovery_window_in_days'] 107 | }, 108 | role= common_constructs['a_common_lambda_role'] 109 | ) 110 | 111 | # ---------------- Step Functions ------------------------ 112 | p_manage_subscription_revoke_state_machine_name = GLOBAL_VARIABLES['producer']['p_manage_subscription_revoke_state_machine_name'] 113 | 114 | p_manage_subscription_revoke_state_machine_logs = logs.LogGroup( 115 | scope= self, 116 | id= 'p_manage_subscription_revoke_state_machine_logs', 117 | log_group_name=f'/aws/step-functions/{p_manage_subscription_revoke_state_machine_name}', 118 | removal_policy=RemovalPolicy.DESTROY 119 | ) 120 | 121 | p_manage_subscription_revoke_state_machine = stepfunctions.StateMachine( 122 | scope= self, 123 | id= 'p_manage_subscription_revoke_state_machine', 124 | state_machine_name= GLOBAL_VARIABLES['producer']['p_manage_subscription_revoke_state_machine_name'], 125 | definition_body=stepfunctions.DefinitionBody.from_file('src/producer/code/stepfunctions/producer_manage_subscription_revoke_workflow.asl.json'), 126 | definition_substitutions= { 127 | 'p_get_connection_details_lambda_arn': p_get_connection_details_lambda.function_arn, 128 | 'p_revoke_jdbc_subscription_lambda_arn': p_revoke_jdbc_subscription_lambda.function_arn, 129 | 'p_delete_keep_subscription_secret_lambda_arn': p_delete_keep_subscription_secret_lambda.function_arn 130 | }, 131 | role= common_constructs['a_common_sf_role'], 132 | logs= stepfunctions.LogOptions( 133 | destination=p_manage_subscription_revoke_state_machine_logs, 134 | level=stepfunctions.LogLevel.ALL 135 | ), 136 | tracing_enabled=True 137 | ) 138 | 139 | -------------------------------------------------------------------------------- /src/producer/stacks/producer_common_stack.py: -------------------------------------------------------------------------------- 1 | from config.common.global_vars import GLOBAL_VARIABLES 2 | 3 | from os import path; 4 | 5 | from aws_cdk import ( 6 | Environment, 7 | Stack, 8 | aws_iam as iam, 9 | aws_sam as sam, 10 | aws_lambda as lambda_, 11 | aws_lakeformation as lakeformation, 12 | ) 13 | 14 | from constructs import Construct 15 | 16 | AWSSDKPANDAS_LAYER_ARNS = { 17 | 'af-south-1': 'arn:aws:lambda:af-south-1:336392948345:layer:AWSSDKPandas-Python38:16', 18 | 'ap-northeast-1': 'arn:aws:lambda:ap-northeast-1:336392948345:layer:AWSSDKPandas-Python38:16', 19 | 'ap-northeast-2': 'arn:aws:lambda:ap-northeast-2:336392948345:layer:AWSSDKPandas-Python38:18', 20 | 'ap-northeast-3': 'arn:aws:lambda:ap-northeast-3:336392948345:layer:AWSSDKPandas-Python38:18', 21 | 'ap-south-1': 'arn:aws:lambda:ap-south-1:336392948345:layer:AWSSDKPandas-Python38:16', 22 | 'ap-southeast-1': 'arn:aws:lambda:ap-southeast-1:336392948345:layer:AWSSDKPandas-Python38:16', 23 | 'ap-southeast-2': 'arn:aws:lambda:ap-southeast-2:336392948345:layer:AWSSDKPandas-Python38:16', 24 | 'ca-central-1': 'arn:aws:lambda:ca-central-1:336392948345:layer:AWSSDKPandas-Python38:18', 25 | 'eu-central-1': 'arn:aws:lambda:eu-central-1:336392948345:layer:AWSSDKPandas-Python38:16', 26 | 'eu-north-1': 'arn:aws:lambda:eu-north-1:336392948345:layer:AWSSDKPandas-Python38:18', 27 | 'eu-west-1': 'arn:aws:lambda:eu-west-1:336392948345:layer:AWSSDKPandas-Python38:16', 28 | 'eu-west-2': 'arn:aws:lambda:eu-west-2:336392948345:layer:AWSSDKPandas-Python38:16', 29 | 'eu-west-3': 'arn:aws:lambda:eu-west-3:336392948345:layer:AWSSDKPandas-Python38:18', 30 | 'sa-east-1': 'arn:aws:lambda:sa-east-1:336392948345:layer:AWSSDKPandas-Python38:18', 31 | 'us-east-1': 'arn:aws:lambda:us-east-1:336392948345:layer:AWSSDKPandas-Python38:16', 32 | 'us-east-2': 'arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python38:16', 33 | 'us-west-1': 'arn:aws:lambda:us-west-1:336392948345:layer:AWSSDKPandas-Python38:19', 34 | 'us-west-2': 'arn:aws:lambda:us-west-2:336392948345:layer:AWSSDKPandas-Python38:17', 35 | 'ap-east-1': 'arn:aws:lambda:ap-east-1:839552336658:layer:AWSSDKPandas-Python38:11', 36 | 'ap-south-2': 'arn:aws:lambda:ap-south-2:246107603503:layer:AWSSDKPandas-Python38:14', 37 | 'ap-southeast-3': 'arn:aws:lambda:ap-southeast-3:258944054355:layer:AWSSDKPandas-Python38:11', 38 | 'ap-southeast-4': 'arn:aws:lambda:ap-southeast-4:945386623051:layer:AWSSDKPandas-Python38:10', 39 | 'eu-central-2': 'arn:aws:lambda:eu-central-2:956415814219:layer:AWSSDKPandas-Python38:10', 40 | 'eu-south-1': 'arn:aws:lambda:eu-south-1:774444163449:layer:AWSSDKPandas-Python38:11', 41 | 'eu-south-2': 'arn:aws:lambda:eu-south-2:982086096842:layer:AWSSDKPandas-Python38:10', 42 | 'il-central-1': 'arn:aws:lambda:il-central-1:263840725265:layer:AWSSDKPandas-Python38:8', 43 | 'me-central-1': 'arn:aws:lambda:me-central-1:593833071574:layer:AWSSDKPandas-Python38:10', 44 | 'me-south-1': 'arn:aws:lambda:me-south-1:938046470361:layer:AWSSDKPandas-Python38:11', 45 | 'cn-north-1': 'arn:aws-cn:lambda:cn-north-1:406640652441:layer:AWSSDKPandas-Python38:9', 46 | 'cn-northwest-1': 'arn:aws-cn:lambda:cn-northwest-1:406640652441:layer:AWSSDKPandas-Python38:7' 47 | } 48 | 49 | class DataZoneConnectorsProducerCommonStack(Stack): 50 | """ Class to represents the stack containing all producer-specific common resources in account.""" 51 | 52 | def __init__(self, scope: Construct, construct_id: str, account_props: dict, producer_props: dict, common_constructs: dict, env: Environment, **kwargs) -> None: 53 | """ Class Constructor. Will deploy producer-specific common resources in account based on properties specified as parameter. 54 | 55 | Parameters 56 | ---------- 57 | account_props : dict 58 | dict with common properties for account. 59 | For more details check config/account/a__config.py documentation and examples. 60 | 61 | producer_props : dict 62 | dict with producer-specific common properties for account. 63 | For more details check config/account/a__config.py documentation and examples. 64 | 65 | common_constructs: dic 66 | dict with constructs common to the account. Created in and output of account common stack. 67 | 68 | env: Environment 69 | Environment object with region and account details 70 | """ 71 | 72 | super().__init__(scope, construct_id, **kwargs) 73 | account_id, region = account_props['account_id'], account_props['region'] 74 | 75 | # ---------------- Lambda Layer ------------------------ 76 | p_aws_sdk_pandas_layer_arn = AWSSDKPANDAS_LAYER_ARNS[region] 77 | p_aws_sdk_pandas_layer = lambda_.LayerVersion.from_layer_version_arn( 78 | scope=self, 79 | id='p_aws_sdk_pandas_layer', 80 | layer_version_arn=p_aws_sdk_pandas_layer_arn 81 | ) 82 | 83 | p_pyodbc_layer = lambda_.LayerVersion( 84 | scope=self, 85 | id='p_pyodbc_layer', 86 | layer_version_name='dz_conn_p_pyodbc_layer', 87 | code=lambda_.AssetCode('libs/python38/pyodbc-layer.zip'), 88 | compatible_runtimes=[lambda_.Runtime.PYTHON_3_8] 89 | ) 90 | 91 | p_oracledb_layer = lambda_.LayerVersion( 92 | scope=self, 93 | id='p_oracledb_layer', 94 | layer_version_name='dz_conn_p_oracledb_layer', 95 | code=lambda_.AssetCode('libs/python38/oracledb-layer.zip'), 96 | compatible_runtimes=[lambda_.Runtime.PYTHON_3_8] 97 | ) 98 | 99 | # ----------------------- Lake Formation --------------------------- 100 | p_lf_tag_key = 'dz_conn_p_access' 101 | p_lf_tag_value = 'True' 102 | 103 | p_lf_tag = lakeformation.CfnTag( 104 | scope= self, 105 | id= 'p_lf_tag', 106 | tag_key= p_lf_tag_key, 107 | tag_values= [p_lf_tag_value] 108 | ) 109 | 110 | p_lf_tag_principals = producer_props['p_lakeformation_tag_principals'] 111 | 112 | a_common_glue_role_name = GLOBAL_VARIABLES['account']['a_common_glue_role_name'] 113 | 114 | p_lf_tag_principals = producer_props['p_lakeformation_tag_principals'] 115 | p_lf_tag_principals[f'role_{a_common_glue_role_name}'] = { 116 | 'type_name': f'role/{a_common_glue_role_name}', 117 | 'lf_tag_permission': False, 118 | 'lf_tag_objects_permission': True 119 | } 120 | 121 | a_common_lambda_role_name = GLOBAL_VARIABLES['account']['a_common_lambda_role_name'] 122 | 123 | p_lf_tag_principals[f'role_{a_common_lambda_role_name}'] = { 124 | 'type_name': f'role/{a_common_lambda_role_name}', 125 | 'lf_tag_permission': True, 126 | 'lf_tag_objects_permission': True 127 | } 128 | 129 | p_lf_principal_id_prefix = f'arn:aws:iam::{account_id}' 130 | p_lf_principal_tag_permissions = [] 131 | p_lf_principal_tag_db_permissions = [] 132 | p_lf_principal_tag_table_permissions = [] 133 | 134 | p_lf_common_lambda_admin_settings = lakeformation.CfnDataLakeSettings( 135 | scope= self, 136 | id= "p_lf_common_lambda_admin_settings", 137 | admins=[ 138 | lakeformation.CfnDataLakeSettings.DataLakePrincipalProperty( 139 | data_lake_principal_identifier=f'{p_lf_principal_id_prefix}:role/{a_common_lambda_role_name}' 140 | ) 141 | ] 142 | ) 143 | 144 | for p_lf_tag_principal_id, p_lf_tag_principal in p_lf_tag_principals.items(): 145 | p_lf_principal_type_name = p_lf_tag_principal['type_name'] 146 | p_lf_tag_permission = p_lf_tag_principal['lf_tag_permission'] 147 | p_lf_tag_objects_permission = p_lf_tag_principal['lf_tag_objects_permission'] 148 | 149 | if p_lf_tag_permission: 150 | p_lf_principal_tag_permission = lakeformation.CfnPrincipalPermissions( 151 | scope= self, 152 | id= f'p_lf_principal_tag_permission_{p_lf_tag_principal_id}', 153 | principal= lakeformation.CfnPrincipalPermissions.DataLakePrincipalProperty( 154 | data_lake_principal_identifier= f'{p_lf_principal_id_prefix}:{p_lf_principal_type_name}' 155 | ), 156 | resource=lakeformation.CfnPrincipalPermissions.ResourceProperty( 157 | lf_tag=lakeformation.CfnPrincipalPermissions.LFTagKeyResourceProperty( 158 | catalog_id= account_id, 159 | tag_key= p_lf_tag_key, 160 | tag_values= [p_lf_tag_value] 161 | ) 162 | ), 163 | permissions= ["DESCRIBE", "ASSOCIATE"], 164 | permissions_with_grant_option= ["DESCRIBE", "ASSOCIATE"] 165 | ) 166 | 167 | p_lf_principal_tag_permission.node.add_dependency(p_lf_tag) 168 | p_lf_principal_tag_permissions.append(p_lf_principal_tag_permission) 169 | 170 | if p_lf_tag_objects_permission: 171 | p_lf_principal_tag_db_permission = lakeformation.CfnPrincipalPermissions( 172 | scope= self, 173 | id= f'p_lf_principal_tag_db_permission_{p_lf_tag_principal_id}', 174 | principal= lakeformation.CfnPrincipalPermissions.DataLakePrincipalProperty( 175 | data_lake_principal_identifier= f'{p_lf_principal_id_prefix}:{p_lf_principal_type_name}' 176 | ), 177 | resource=lakeformation.CfnPrincipalPermissions.ResourceProperty( 178 | lf_tag_policy=lakeformation.CfnPrincipalPermissions.LFTagPolicyResourceProperty( 179 | catalog_id= account_id, 180 | expression=[lakeformation.CfnPrincipalPermissions.LFTagProperty( 181 | tag_key= p_lf_tag_key, 182 | tag_values= [p_lf_tag_value] 183 | )], 184 | resource_type= 'DATABASE' 185 | ) 186 | ), 187 | permissions= ["ALL"], 188 | permissions_with_grant_option= ["ALL"] 189 | ) 190 | 191 | p_lf_principal_tag_db_permission.node.add_dependency(p_lf_tag) 192 | p_lf_principal_tag_db_permissions.append(p_lf_principal_tag_db_permission) 193 | 194 | p_lf_principal_tag_table_permission = lakeformation.CfnPrincipalPermissions( 195 | scope= self, 196 | id= f'lf-principal-tag-table-permission-{p_lf_tag_principal_id}', 197 | principal= lakeformation.CfnPrincipalPermissions.DataLakePrincipalProperty( 198 | data_lake_principal_identifier= f'{p_lf_principal_id_prefix}:{p_lf_principal_type_name}' 199 | ), 200 | resource=lakeformation.CfnPrincipalPermissions.ResourceProperty( 201 | lf_tag_policy=lakeformation.CfnPrincipalPermissions.LFTagPolicyResourceProperty( 202 | catalog_id= account_id, 203 | expression=[lakeformation.CfnPrincipalPermissions.LFTagProperty( 204 | tag_key= p_lf_tag_key, 205 | tag_values= [p_lf_tag_value] 206 | )], 207 | resource_type= 'TABLE' 208 | ) 209 | ), 210 | permissions= ["ALL"], 211 | permissions_with_grant_option= ["ALL"] 212 | ) 213 | 214 | p_lf_principal_tag_table_permission.node.add_dependency(p_lf_tag) 215 | p_lf_principal_tag_table_permissions.append(p_lf_principal_tag_table_permission) 216 | 217 | # ---------------- Lambda ------------------------ 218 | p_add_lf_tag_environment_dbs_lambda = lambda_.Function( 219 | scope= self, 220 | id= 'p_add_lf_tag_environment_dbs_lambda', 221 | function_name= GLOBAL_VARIABLES['producer']['p_add_lf_tag_environment_dbs_lambda_name'], 222 | runtime= lambda_.Runtime.PYTHON_3_11, 223 | code=lambda_.Code.from_asset(path.join('src/producer/code/lambda', "add_lf_tag_environment_dbs")), 224 | handler= "add_lf_tag_environment_dbs.handler", 225 | role= common_constructs['a_common_lambda_role'], 226 | environment= { 227 | 'P_LAKEFORMATION_TAG_KEY': p_lf_tag_key, 228 | 'P_LAKEFORMATION_TAG_VALUE': p_lf_tag_value 229 | } 230 | ) 231 | 232 | # -------------- Outputs -------------------- 233 | self.outputs = { 234 | 'p_aws_sdk_pandas_layer': p_aws_sdk_pandas_layer, 235 | 'p_pyodbc_layer': p_pyodbc_layer, 236 | 'p_oracledb_layer': p_oracledb_layer, 237 | 'p_add_lf_tag_environment_dbs_lambda': p_add_lf_tag_environment_dbs_lambda 238 | } 239 | 240 | 241 | 242 | 243 | -------------------------------------------------------------------------------- /src/producer/stacks/producer_workflows_stack.py: -------------------------------------------------------------------------------- 1 | from aws_cdk import ( 2 | Stack 3 | ) 4 | 5 | from constructs import Construct 6 | from src.producer.constructs.producer_subscription_grant_workflow import ProducerManageSubscriptionGrantWorkflowConstruct 7 | from src.producer.constructs.producer_subscription_revoke_workflow import ProducerManageSubscriptionRevokeWorkflowConstruct 8 | 9 | class ProducerWorkflowsStack(Stack): 10 | """ Class to represents the stack containing all producer workflows in account.""" 11 | 12 | def __init__(self, scope: Construct, construct_id: str, account_props: dict, workflows_props: list, common_constructs: dict, **kwargs) -> None: 13 | """ Class Constructor. Will deploy one of each producer workflow constructs based on properties specified as parameter. 14 | 15 | Parameters 16 | ---------- 17 | account_props : dict 18 | dict with common properties for account. 19 | For more details check config/account/a__config.py documentation and examples. 20 | 21 | workflows_props : dict 22 | dict with required properties for all workflows creation. 23 | For more details check config/account/a__config.py documentation and examples. 24 | 25 | common_constructs: dic 26 | dict with constructs common to the account. Created in and output of account common stack. 27 | 28 | env: Environment 29 | Environment object with region and account details 30 | """ 31 | 32 | super().__init__(scope, construct_id, **kwargs) 33 | 34 | env = kwargs.get('env') 35 | 36 | p_manage_subscription_grant_workflow_props = workflows_props['p_manage_subscription_grant'] 37 | 38 | ProducerManageSubscriptionGrantWorkflowConstruct( 39 | scope = self, 40 | construct_id = 'dz-conn-p-manage-subscription-grant-workflow-construct', 41 | account_props = account_props, 42 | workflow_props = p_manage_subscription_grant_workflow_props, 43 | common_constructs = common_constructs, 44 | env = env 45 | ) 46 | 47 | p_manage_subscription_revoke_workflow_props = workflows_props['p_manage_subscription_revoke'] 48 | 49 | ProducerManageSubscriptionRevokeWorkflowConstruct( 50 | scope = self, 51 | construct_id = 'dz-conn-p-manage-subscription-revoke-workflow-construct', 52 | account_props = account_props, 53 | workflow_props = p_manage_subscription_revoke_workflow_props, 54 | common_constructs = common_constructs, 55 | env = env 56 | ) 57 | --------------------------------------------------------------------------------