├── emr_eks_cdk ├── __init__.py ├── mwaa_plugins │ ├── requirements.txt │ └── emr_eks.py ├── studio_stack.py ├── mwaa_stack.py ├── studio_live_stack.py └── emr_eks_cdk_stack.py ├── requirements.txt ├── .gitignore ├── CODE_OF_CONDUCT.md ├── cdk.json ├── app.py ├── LICENSE ├── setup.py ├── CONTRIBUTING.md └── README.md /emr_eks_cdk/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | -------------------------------------------------------------------------------- /emr_eks_cdk/mwaa_plugins/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 >= 1.16.35 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | package-lock.json 3 | __pycache__ 4 | .pytest_cache 5 | .env 6 | .venv 7 | *.egg-info 8 | 9 | # CDK asset staging directory 10 | .cdk.staging 11 | cdk.out 12 | 13 | # VS Code 14 | emr-eks-cdk.code-workspace 15 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "python3 app.py", 3 | "context": { 4 | "@aws-cdk/core:enableStackNameDuplicates": "true", 5 | "aws-cdk:enableDiffNoFail": "true", 6 | "@aws-cdk/core:stackRelativeExports": "true", 7 | "@aws-cdk/aws-ecr-assets:dockerIgnoreSupport": true, 8 | "@aws-cdk/aws-secretsmanager:parseOwnedSecretName": true, 9 | "@aws-cdk/aws-kms:defaultKeyPolicies": true, 10 | "@aws-cdk/aws-s3:grantWriteWithoutAcl": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | #!/usr/bin/env python3 5 | 6 | from emr_eks_cdk.mwaa_stack import MwaaStack 7 | from aws_cdk import core 8 | from emr_eks_cdk.studio_stack import StudioStack 9 | from emr_eks_cdk.studio_live_stack import StudioLiveStack 10 | from emr_eks_cdk.emr_eks_cdk_stack import EmrEksCdkStack 11 | 12 | 13 | app = core.App() 14 | eks = EmrEksCdkStack(app, "emr-eks-cdk") 15 | private_subnets = eks.vpc.private_subnets 16 | private_subnet_ids = [n.subnet_id for n in private_subnets] 17 | 18 | MwaaStack(app, "mwaa-cdk", private_subnet_ids, eks.vpc) 19 | StudioStack(app, "studio-cdk", eks.job_role.role_arn, eks.emr_vc.attr_id) 20 | StudioLiveStack(app, "studio-live-cdk", eks.vpc) 21 | 22 | app.synth() 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | import setuptools 5 | 6 | 7 | with open("README.md") as fp: 8 | long_description = fp.read() 9 | 10 | 11 | setuptools.setup( 12 | name="emr_eks_cdk", 13 | version="0.0.1", 14 | 15 | description="An empty CDK Python app", 16 | long_description=long_description, 17 | long_description_content_type="text/markdown", 18 | 19 | author="author", 20 | 21 | package_dir={"": "emr_eks_cdk"}, 22 | packages=setuptools.find_packages(where="emr_eks_cdk"), 23 | 24 | install_requires=[ 25 | "aws-cdk.core==1.99.0", 26 | "aws-cdk.aws-mwaa==1.99.0", 27 | "aws-cdk.aws-emrcontainers==1.99.0", 28 | "aws-cdk.aws-eks==1.99.0", 29 | "aws-cdk.aws-ec2==1.99.0", 30 | "aws-cdk.aws-emr==1.99.0", 31 | "aws-cdk.aws_acmpca==1.99.0", 32 | "aws-cdk.aws-s3-deployment==1.99.0", 33 | "pyOpenSSL", 34 | "boto3", 35 | "awscli" 36 | ], 37 | 38 | python_requires=">=3.6", 39 | 40 | classifiers=[ 41 | "Development Status :: 4 - Beta", 42 | 43 | "Intended Audience :: Developers", 44 | 45 | "License :: OSI Approved :: Apache Software License", 46 | 47 | "Programming Language :: JavaScript", 48 | "Programming Language :: Python :: 3 :: Only", 49 | "Programming Language :: Python :: 3.6", 50 | "Programming Language :: Python :: 3.7", 51 | "Programming Language :: Python :: 3.8", 52 | 53 | "Topic :: Software Development :: Code Generators", 54 | "Topic :: Utilities", 55 | 56 | "Typing :: Typed", 57 | ], 58 | ) 59 | -------------------------------------------------------------------------------- /emr_eks_cdk/mwaa_plugins/emr_eks.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | from airflow import DAG 5 | 6 | from airflow.operators.bash_operator import BashOperator 7 | from airflow.operators.emr_containers_airflow_plugin import EmrContainersStartJobRun 8 | from airflow.sensors.emr_containers_airflow_plugin import EmrContainersJobRunSensor 9 | from airflow.models import Variable 10 | 11 | from airflow.utils.dates import days_ago 12 | from datetime import timedelta 13 | import os 14 | 15 | DAG_ID = os.path.basename(__file__).replace(".py", "") 16 | 17 | DEFAULT_ARGS = { 18 | 'owner': 'airflow', 19 | 'depends_on_past': False, 20 | 'email': ['you@amazon.com'], 21 | 'email_on_failure': False, 22 | 'email_on_retry': False, 23 | } 24 | 25 | JOB_DRIVER_ARG = { 26 | 'sparkSubmitJobDriver': {"entryPoint": "local:///usr/lib/spark/examples/src/main/python/pi.py","sparkSubmitParameters": "--conf spark.executors.instances=2 --conf spark.executors.memory=2G --conf spark.executor.cores=2 --conf spark.driver.cores=1"} 27 | } 28 | 29 | CONFIGURATION_OVERRIDES_ARG = { 30 | 'monitoringConfiguration': {"cloudWatchMonitoringConfiguration": {"logGroupName": "/emr-containers/jobs", "logStreamNamePrefix": "demo"}} 31 | } 32 | 33 | with DAG( 34 | dag_id=DAG_ID, 35 | default_args=DEFAULT_ARGS, 36 | dagrun_timeout=timedelta(hours=2), 37 | start_date=days_ago(1), 38 | schedule_interval='@once', 39 | tags=['emr_containers'], 40 | params={ 41 | "cluster_id": "", 42 | "role_arn": "" 43 | }, 44 | ) as dag: 45 | 46 | job_starter = EmrContainersStartJobRun( 47 | task_id='start_job', 48 | virtual_cluster_id=Variable.get("cluster_id"), 49 | execution_role_arn=Variable.get("role_arn"), 50 | #virtual_cluster_id="{{ dag_run.conf['cluster_id'] }}", 51 | #execution_role_arn="{{ dag_run.conf['role_arn'] }}", 52 | release_label='emr-6.2.0-latest', 53 | job_driver=JOB_DRIVER_ARG, 54 | configuration_overrides=CONFIGURATION_OVERRIDES_ARG, 55 | name='pi.py', 56 | client_token='dummy' 57 | ) 58 | 59 | job_checker = EmrContainersJobRunSensor( 60 | task_id='watch_job', 61 | virtual_cluster_id=Variable.get("cluster_id"), 62 | id="{{ task_instance.xcom_pull(task_ids='start_job', key='return_value') }}", 63 | aws_conn_id='aws_default' 64 | ) 65 | 66 | job_starter >> job_checker 67 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /emr_eks_cdk/studio_stack.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | from aws_cdk import aws_ec2 as ec2, aws_eks as eks, core, aws_emrcontainers as emrc, aws_iam as iam, aws_logs as logs, custom_resources as custom, aws_acmpca as acmpca 5 | from OpenSSL import crypto, SSL 6 | import random 7 | 8 | """ 9 | This stack deploys the following: 10 | - EMR Studio Prerequisites 11 | """ 12 | class StudioStack(core.Stack): 13 | 14 | 15 | def __init__(self, scope: core.Construct, construct_id: str, executionRoleArn: str, virtualClusterId: str, **kwargs) -> None: 16 | super().__init__(scope, construct_id, **kwargs) 17 | 18 | # policy to let Lambda invoke the api 19 | custom_policy_document = iam.PolicyDocument(statements=[ 20 | iam.PolicyStatement( 21 | effect=iam.Effect.ALLOW, 22 | actions=["ec2:CreateSecurityGroup", 23 | "ec2:RevokeSecurityGroupEgress", 24 | "ec2:CreateSecurityGroup", 25 | "ec2:DeleteSecurityGroup", 26 | "ec2:AuthorizeSecurityGroupEgress", 27 | "ec2:AuthorizeSecurityGroupIngress", 28 | "ec2:RevokeSecurityGroupIngress", 29 | "ec2:DeleteSecurityGroup" 30 | ], 31 | resources=["*"] 32 | ) 33 | ]) 34 | managed_policy = iam.ManagedPolicy(self, "EMR_on_EKS_security_group", 35 | document=custom_policy_document 36 | ) 37 | 38 | self.role = iam.Role( 39 | scope=self, 40 | id=f'{construct_id}-LambdaRole', 41 | assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'), 42 | managed_policies=[ 43 | iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole"), 44 | managed_policy 45 | ] 46 | ) 47 | 48 | # cert for endpoint 49 | crt, pkey = self.cert_gen(serialNumber=random.randint(1000,10000)) 50 | mycert = custom.AwsCustomResource(self, "CreateCert", 51 | on_update={ 52 | "service": "ACM", 53 | "action": "importCertificate", 54 | "parameters": { 55 | "Certificate": crt.decode("utf-8"), 56 | "PrivateKey": pkey.decode("utf-8") 57 | }, 58 | "physical_resource_id": custom.PhysicalResourceId.from_response("CertificateArn") 59 | }, 60 | policy=custom.AwsCustomResourcePolicy.from_sdk_calls(resources=custom.AwsCustomResourcePolicy.ANY_RESOURCE), 61 | role=self.role, 62 | function_name="CreateCertFn" 63 | ) 64 | 65 | # Set up managed endpoint for Studio 66 | endpoint = custom.AwsCustomResource(self, "CreateEndpoint", 67 | on_create={ 68 | "service": "EMRcontainers", 69 | "action": "createManagedEndpoint", 70 | "parameters": { 71 | "certificateArn": mycert.get_response_field("CertificateArn"), 72 | "executionRoleArn": executionRoleArn, 73 | "name": "emr-endpoint-eks-spark", 74 | "releaseLabel": "emr-6.2.0-latest", 75 | "type": "JUPYTER_ENTERPRISE_GATEWAY", 76 | "virtualClusterId": virtualClusterId, 77 | }, 78 | "physical_resource_id": custom.PhysicalResourceId.from_response("arn")}, 79 | policy=custom.AwsCustomResourcePolicy.from_sdk_calls(resources=custom.AwsCustomResourcePolicy.ANY_RESOURCE), 80 | role=self.role, 81 | function_name="CreateEpFn" 82 | ) 83 | endpoint.node.add_dependency(mycert) 84 | 85 | 86 | def cert_gen(self, 87 | emailAddress="emailAddress", 88 | commonName="emroneks.com", 89 | countryName="NT", 90 | localityName="localityName", 91 | stateOrProvinceName="stateOrProvinceName", 92 | organizationName="organizationName", 93 | organizationUnitName="organizationUnitName", 94 | serialNumber=1234, 95 | validityStartInSeconds=0, 96 | validityEndInSeconds=10*365*24*60*60, 97 | KEY_FILE = "private.key", 98 | CERT_FILE="selfsigned.crt"): 99 | #can look at generated file using openssl: 100 | #openssl x509 -inform pem -in selfsigned.crt -noout -text 101 | # create a key pair 102 | k = crypto.PKey() 103 | k.generate_key(crypto.TYPE_RSA, 2048) 104 | # create a self-signed cert 105 | cert = crypto.X509() 106 | cert.get_subject().C = countryName 107 | cert.get_subject().ST = stateOrProvinceName 108 | cert.get_subject().L = localityName 109 | cert.get_subject().O = organizationName 110 | cert.get_subject().OU = organizationUnitName 111 | cert.get_subject().CN = commonName 112 | cert.get_subject().emailAddress = emailAddress 113 | cert.set_serial_number(serialNumber) 114 | cert.gmtime_adj_notBefore(0) 115 | cert.gmtime_adj_notAfter(validityEndInSeconds) 116 | cert.set_issuer(cert.get_subject()) 117 | cert.set_pubkey(k) 118 | cert.sign(k, 'sha512') 119 | return (crypto.dump_certificate(crypto.FILETYPE_PEM, cert), crypto.dump_privatekey(crypto.FILETYPE_PEM, k)) 120 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # EMR on EKS via SDK 2 | 3 | This project helps you demonstrate EMR on EKS using the CDK for automation. 4 | 5 | It deploys the following: 6 | 7 | * An EKS cluster in a new VPC across 3 subnets 8 | * The Cluster has a default managed node group set to scale between 3 and 10 nodes. This node group can use regular instances or Graviton2. 9 | * It also has a Fargate profile set to use the `sparkfg` namespace 10 | * Two EMR virtual clusters on the EKS cluster 11 | * The first virtual cluster uses the `sparkns` namespace on a managed node group 12 | * The second virtual cluster uses the `sparkfg` namespace on a Fargate profile 13 | * All EMR on EKS configuration is done, including a cluster role bound to an IAM role 14 | * A default job execution role that has full CloudWatch and S3 access 15 | * A CloudWatch log group for use with Spark job runs 16 | * Optionally, sets up Apache Airflow and EMR Studio 17 | 18 | ## Deployment 19 | 20 | We tested this deployment using CDK version 1.99.0. 21 | 22 | After cloning the repo, download the Airflow plugin zip file: 23 | 24 | wget https://elasticmapreduce.s3.amazonaws.com/emr-containers/airflow/plugins/emr_containers_airflow_plugin.zip 25 | mv emr_containers_airflow_plugin.zip emr_eks_cdk/mwaa_plugins/ 26 | 27 | Now run: 28 | 29 | python3 -m venv .venv 30 | source .venv/bin/activate 31 | pip install -r requirements.txt 32 | cdk bootstrap aws:/// --context prefix= --context instance=m5.xlarge --context username= 33 | cdk synth --context prefix= --context instance=m5.xlarge --context username= 34 | cdk ls --context prefix= --context instance=m5.xlarge --context username= # list stacks 35 | cdk deploy --context prefix= --context instance=m5.xlarge --context username= 36 | 37 | If deployment fails due to an error creating the EKS cluster, just redeploy. This is a [known issue](https://github.com/aws/aws-cdk/issues/9027) in the CDK. 38 | 39 | Available stacks include: 40 | 41 | * emr-eks-cdk: The base stack 42 | * mwaa-cdk: Adds Airflow 43 | * studio-cdk: Adds EMR Studio prerequisites (requires SSO enabled in your account) 44 | * studio-cdk-live: Adds EMR Studio (requires SSO enabled in your account) 45 | 46 | Note that EMR Studio doesn't yet support Graviton2 nodes. Do not choose a Graviton2 instance type if you want to use Studio. 47 | 48 | ## Register EKS in kube config 49 | 50 | The CDK output should include the command you use to update your kube config to run commands on the cluster. It'll look like this: 51 | 52 | emr-eks-cdk.EksForSparkConfigCommandB4B8E93B = aws eks update-kubeconfig --name EksForSparkCF45D836-7d1075f7618943f2b56095bcbdd13709 --region us-west-2 --role-arn arn:aws:iam:::role/emr-eks-cdk-EksForSparkMastersRole19842074-1QPZ722TZ38S 53 | 54 | If you run multiple k8s clusters, you can list and switch cluster contexts like this: 55 | 56 | kubectl config get-contexts 57 | kubectl config use-context "arn:aws:eks:us-west-2::cluster/EksForSparkCF45D836-7d1075f7618943f2b56095bcbdd13709" 58 | 59 | The argument to `use-context` is the context name as reported by `get-contexts`. 60 | 61 | ## Test a job run 62 | 63 | First, identify your virtual cluster ID: 64 | 65 | aws emr-containers list-virtual-clusters 66 | 67 | The virtual cluster IDs are also in the CDK stack output. 68 | 69 | You can run a test application with this command: 70 | 71 | This command uses `local://` scheme to refer to a `pi.py` file as the `entryPoint` for the job. This file is already pre-built into the docker image and can be referenced using `local://` scheme. Note that the `local://` scheme can only be used to reference files that are pre-built into the docker image. It can not be used to refer to files in your local filesystem. 72 | 73 | aws emr-containers start-job-run \ 74 | --virtual-cluster-id \ 75 | --name sample-job-name \ 76 | --execution-role-arn \ 77 | --release-label emr-6.2.0-latest \ 78 | --job-driver '{"sparkSubmitJobDriver": {"entryPoint": "local:///usr/lib/spark/examples/src/main/python/pi.py","sparkSubmitParameters": "--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.executor.cores=2 --conf spark.driver.cores=1"}}' \ 79 | --configuration-overrides '{"monitoringConfiguration": {"cloudWatchMonitoringConfiguration": {"logGroupName": "", "logStreamNamePrefix": "SparkEMREKS"}}}' 80 | 81 | You can track job completion in the EMR console. 82 | 83 | ## Testing with Airflow 84 | 85 | Go to the MWAA console and open the Airflow UI. Activate the DAG by moving the slider to `On`. Add two Airflow variables: 86 | 87 | * cluster_id = your virtual cluster ID 88 | * role_arn = your job role ARN 89 | 90 | Then trigger the DAG. 91 | 92 | ## Running on Graviton2 node group 93 | 94 | Deploy the stack with `m6g.xlarge` as the instance type rather than `m5.xlarge`. Then include a node selector when you start the job. 95 | 96 | This command uses `local://` scheme to refer to a `pi.py` file as the `entryPoint` for the job. This file is already pre-built into the docker image and can be referenced using `local://` scheme. Note that the `local://` scheme can only be used to reference files that are pre-built into the docker image. It can not be used to refer to files in your local filesystem. 97 | 98 | aws emr-containers start-job-run \ 99 | --virtual-cluster-id \ 100 | --name sample-job-name \ 101 | --execution-role-arn \ 102 | --release-label emr-6.2.0-latest \ 103 | --job-driver '{"sparkSubmitJobDriver": {"entryPoint": "local:///usr/lib/spark/examples/src/main/python/pi.py","sparkSubmitParameters": "--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.executor.cores=2 --conf spark.driver.cores=1 --conf spark.kubernetes.node.selector.kubernetes.io/arch=arm64"}}' \ 104 | --configuration-overrides '{"monitoringConfiguration": {"cloudWatchMonitoringConfiguration": {"logGroupName": "", "logStreamNamePrefix": "SparkEMREKS"}}}' 105 | 106 | ## EMR Studio 107 | 108 | Deploy the `studio-cdk` script. Wait for it to deploy and check to make sure that the endpoint is active: 109 | 110 | aws emr-containers list-managed-endpoints --virtual-cluster-id | jq '.endpoints[].state' 111 | 112 | If the endpoint fails to create, try creating it again manually: 113 | 114 | aws emr-containers create-managed-endpoint \ 115 | --type JUPYTER_ENTERPRISE_GATEWAY \ 116 | --virtual-cluster-id \ 117 | --name \ 118 | --execution-role-arn 119 | --release-label 120 | --certificate-arn 121 | 122 | Now deploy the `studio-live-cdk` script. The script will output the URL for your Studio environment. 123 | 124 | ## Security 125 | 126 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 127 | 128 | ## License 129 | 130 | This library is licensed under the MIT-0 License. See the LICENSE file. 131 | -------------------------------------------------------------------------------- /emr_eks_cdk/mwaa_stack.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | from aws_cdk import core, aws_iam as iam, aws_s3 as s3, aws_s3_deployment as s3deploy, aws_mwaa as mwaa, custom_resources as custom, aws_ec2 as ec2 5 | from typing import List 6 | 7 | """ 8 | This stack deploys the following: 9 | - S3 bucket with dependencies uploaded 10 | - MWAA role 11 | - MWAA environment 12 | """ 13 | class MwaaStack(core.Stack): 14 | 15 | def __init__(self, scope: core.Construct, construct_id: str, subnets: List[str], vpc: ec2.IVpc, **kwargs) -> None: 16 | super().__init__(scope, construct_id, **kwargs) 17 | self.env_name = "MwaaForEmrOnEks" 18 | self.prefix_list_id = self.node.try_get_context("prefix") 19 | 20 | # Create S3 bucket for MWAA 21 | bucket = s3.Bucket(self, "MwaaBucket", 22 | encryption=s3.BucketEncryption.S3_MANAGED, 23 | block_public_access=s3.BlockPublicAccess.BLOCK_ALL, 24 | versioned = True 25 | ) 26 | core.CfnOutput( 27 | self, "BucketName", 28 | value=bucket.bucket_name 29 | ) 30 | 31 | # Create MWAA role 32 | role = iam.Role(self, "MwaaRole", 33 | assumed_by=iam.ServicePrincipal("airflow-env.amazonaws.com") 34 | ) 35 | role.add_to_policy(iam.PolicyStatement( 36 | resources=[f"arn:aws:airflow:{self.region}:{self.account}:environment/{self.env_name}"], 37 | actions=["airflow:PublishMetrics"], 38 | effect=iam.Effect.ALLOW 39 | )) 40 | role.add_to_policy(iam.PolicyStatement( 41 | resources=[f"arn:aws:s3:::{bucket.bucket_name}",f"arn:aws:s3:::{bucket.bucket_name}/*"], 42 | actions=["s3:ListAllMyBuckets"], 43 | effect=iam.Effect.DENY 44 | )) 45 | role.add_to_policy(iam.PolicyStatement( 46 | resources=[f"arn:aws:s3:::{bucket.bucket_name}",f"arn:aws:s3:::{bucket.bucket_name}/*"], 47 | actions=["s3:GetObject*","s3:GetBucket*","s3:List*"], 48 | effect=iam.Effect.ALLOW 49 | )) 50 | role.add_to_policy(iam.PolicyStatement( 51 | resources=[f"arn:aws:logs:{self.region}:{self.account}:log-group:airflow-{self.env_name}-*"], 52 | actions=["logs:CreateLogStream", 53 | "logs:CreateLogGroup", 54 | "logs:PutLogEvents", 55 | "logs:GetLogEvents", 56 | "logs:GetLogRecord", 57 | "logs:GetLogGroupFields", 58 | "logs:GetQueryResults", 59 | "logs:DescribeLogGroups"], 60 | effect=iam.Effect.ALLOW 61 | )) 62 | role.add_to_policy(iam.PolicyStatement( 63 | resources=["*"], 64 | actions=["cloudwatch:PutMetricData"], 65 | effect=iam.Effect.ALLOW 66 | )) 67 | role.add_to_policy(iam.PolicyStatement( 68 | resources=["*"], 69 | actions=[ 70 | "emr-containers:StartJobRun", 71 | "emr-containers:ListJobRuns", 72 | "emr-containers:DescribeJobRun", 73 | "emr-containers:CancelJobRun" 74 | ], 75 | effect=iam.Effect.ALLOW 76 | )) 77 | role.add_to_policy(iam.PolicyStatement( 78 | resources=[f"arn:aws:sqs:{self.region}:*:airflow-celery-*"], 79 | actions=["sqs:ChangeMessageVisibility", 80 | "sqs:DeleteMessage", 81 | "sqs:GetQueueAttributes", 82 | "sqs:GetQueueUrl", 83 | "sqs:ReceiveMessage", 84 | "sqs:SendMessage"], 85 | effect=iam.Effect.ALLOW 86 | )) 87 | string_like = core.CfnJson(self, "ConditionJson", 88 | value={ 89 | f"kms:ViaService": f"sqs.{self.region}.amazonaws.com" 90 | } 91 | ) 92 | role.add_to_policy(iam.PolicyStatement( 93 | not_resources=[f"arn:aws:kms:*:{self.account}:key/*"], 94 | actions=["kms:Decrypt", 95 | "kms:DescribeKey", 96 | "kms:GenerateDataKey*", 97 | "kms:Encrypt"], 98 | effect=iam.Effect.ALLOW, 99 | conditions={"StringLike": string_like} 100 | )) 101 | 102 | # Upload MWAA pre-reqs 103 | s3deploy.BucketDeployment(self, "DeployPlugin", 104 | sources=[s3deploy.Source.asset("./emr_eks_cdk/mwaa_plugins", exclude= ['**', '!emr_containers_airflow_plugin.zip'])], 105 | destination_bucket=bucket, 106 | destination_key_prefix="plug-ins" 107 | ) 108 | s3req = s3deploy.BucketDeployment(self, "DeployReq", 109 | sources=[s3deploy.Source.asset("./emr_eks_cdk/mwaa_plugins", exclude= ['**', '!requirements.txt'])], 110 | destination_bucket=bucket, 111 | destination_key_prefix="Requirements" 112 | ) 113 | s3deploy.BucketDeployment(self, "DeployDag", 114 | sources=[s3deploy.Source.asset("./emr_eks_cdk/mwaa_plugins", exclude= ['**', '!emr_eks.py'])], 115 | destination_bucket=bucket, 116 | destination_key_prefix="DAG" 117 | ) 118 | 119 | # Get object versions 120 | req_obj_version = custom.AwsCustomResource(self, "GetReqV", 121 | on_update={ 122 | "service": "S3", 123 | "action": "headObject", 124 | "parameters": { 125 | "Bucket": bucket.bucket_name, 126 | "Key": "Requirements/requirements.txt" 127 | }, 128 | "physical_resource_id": custom.PhysicalResourceId.from_response("VersionId")}, 129 | policy=custom.AwsCustomResourcePolicy.from_sdk_calls(resources=custom.AwsCustomResourcePolicy.ANY_RESOURCE), 130 | role = iam.Role( 131 | scope=self, 132 | id=f'{construct_id}-LambdaRole', 133 | assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'), 134 | managed_policies=[ 135 | iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole"), 136 | iam.ManagedPolicy.from_aws_managed_policy_name("AmazonS3FullAccess") 137 | ] 138 | ) 139 | ) 140 | core.CfnOutput( 141 | self, "ReqObjVersion", 142 | value=req_obj_version.get_response_field("VersionId") 143 | ) 144 | plugin_obj_version = custom.AwsCustomResource(self, "GetPluginV", 145 | on_update={ 146 | "service": "S3", 147 | "action": "headObject", 148 | "parameters": { 149 | "Bucket": bucket.bucket_name, 150 | "Key": "plug-ins/emr_containers_airflow_plugin.zip" 151 | }, 152 | "physical_resource_id": custom.PhysicalResourceId.from_response("VersionId")}, 153 | policy=custom.AwsCustomResourcePolicy.from_sdk_calls(resources=custom.AwsCustomResourcePolicy.ANY_RESOURCE), 154 | role = iam.Role( 155 | scope=self, 156 | id=f'{construct_id}-LambdaRole-2', 157 | assumed_by=iam.ServicePrincipal('lambda.amazonaws.com'), 158 | managed_policies=[ 159 | iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AWSLambdaBasicExecutionRole"), 160 | iam.ManagedPolicy.from_aws_managed_policy_name("AmazonS3FullAccess") 161 | ] 162 | ) 163 | ) 164 | core.CfnOutput( 165 | self, "PluginObjVersion", 166 | value=plugin_obj_version.get_response_field("VersionId") 167 | ) 168 | 169 | # Create security group 170 | mwaa_sg = ec2.SecurityGroup(self, "SecurityGroup", 171 | vpc=vpc, 172 | description="Allow inbound access to MWAA", 173 | allow_all_outbound=True 174 | ) 175 | mwaa_sg.add_ingress_rule(ec2.Peer.prefix_list(self.prefix_list_id), ec2.Port.all_tcp(), "allow inbound access from the prefix list") 176 | mwaa_sg.add_ingress_rule(mwaa_sg, ec2.Port.all_traffic(), "allow inbound access from the SG") 177 | 178 | mwaa_env = mwaa.CfnEnvironment(self, "MWAAEnv", 179 | name = self.env_name, 180 | dag_s3_path="DAG", 181 | environment_class="mw1.small", 182 | execution_role_arn=role.role_arn, 183 | logging_configuration = mwaa.CfnEnvironment.LoggingConfigurationProperty( 184 | dag_processing_logs=mwaa.CfnEnvironment.ModuleLoggingConfigurationProperty(enabled=True, log_level='INFO'), 185 | scheduler_logs=mwaa.CfnEnvironment.ModuleLoggingConfigurationProperty(enabled=True, log_level='INFO'), 186 | task_logs=mwaa.CfnEnvironment.ModuleLoggingConfigurationProperty(enabled=True, log_level='INFO'), 187 | webserver_logs=mwaa.CfnEnvironment.ModuleLoggingConfigurationProperty(enabled=True, log_level='INFO'), 188 | worker_logs=mwaa.CfnEnvironment.ModuleLoggingConfigurationProperty(enabled=True, log_level='INFO') 189 | ), 190 | network_configuration=mwaa.CfnEnvironment.NetworkConfigurationProperty( 191 | security_group_ids=[mwaa_sg.security_group_id], 192 | subnet_ids=subnets 193 | ), 194 | plugins_s3_path="plug-ins/emr_containers_airflow_plugin.zip", 195 | plugins_s3_object_version=plugin_obj_version.get_response_field("VersionId"), 196 | requirements_s3_path="Requirements/requirements.txt", 197 | requirements_s3_object_version=req_obj_version.get_response_field("VersionId"), 198 | source_bucket_arn=bucket.bucket_arn, 199 | webserver_access_mode='PUBLIC_ONLY' 200 | ) 201 | core.CfnOutput( 202 | self, "MWAA_NAME", value=self.env_name 203 | ) 204 | -------------------------------------------------------------------------------- /emr_eks_cdk/studio_live_stack.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | from aws_cdk import aws_ec2 as ec2, aws_eks as eks, core, aws_emrcontainers as emrc, aws_iam as iam, aws_s3 as s3, custom_resources as custom, aws_acmpca as acmpca, aws_emr as emr 5 | 6 | """ 7 | This stack deploys the following: 8 | - EMR Studio 9 | """ 10 | class StudioLiveStack(core.Stack): 11 | 12 | def __init__(self, scope: core.Construct, construct_id: str, vpc: ec2.IVpc, **kwargs) -> None: 13 | super().__init__(scope, construct_id, **kwargs) 14 | 15 | # Create S3 bucket for Studio 16 | bucket = s3.Bucket(self, "StudioBucket", 17 | encryption=s3.BucketEncryption.S3_MANAGED, 18 | block_public_access=s3.BlockPublicAccess.BLOCK_ALL, 19 | versioned = True 20 | ) 21 | 22 | # Create security groups 23 | eng_sg = ec2.SecurityGroup(self, "EngineSecurityGroup", 24 | vpc=vpc, 25 | description="EMR Studio Engine", 26 | allow_all_outbound=True 27 | ) 28 | core.Tags.of(eng_sg).add("for-use-with-amazon-emr-managed-policies", "true") 29 | ws_sg = ec2.SecurityGroup(self, "WorkspaceSecurityGroup", 30 | vpc=vpc, 31 | description="EMR Studio Workspace", 32 | allow_all_outbound=False 33 | ) 34 | core.Tags.of(ws_sg).add("for-use-with-amazon-emr-managed-policies", "true") 35 | ws_sg.add_egress_rule(ec2.Peer.any_ipv4(), ec2.Port.tcp(443), "allow egress on port 443") 36 | ws_sg.add_egress_rule(eng_sg, ec2.Port.tcp(18888), "allow egress on port 18888 to eng") 37 | eng_sg.add_ingress_rule(ws_sg, ec2.Port.tcp(18888), "allow ingress on port 18888 from ws") 38 | 39 | # Create Studio roles 40 | role = iam.Role(self, "StudioRole", 41 | assumed_by=iam.ServicePrincipal("elasticmapreduce.amazonaws.com"), 42 | managed_policies=[ 43 | iam.ManagedPolicy.from_aws_managed_policy_name("AmazonS3FullAccess") 44 | ] 45 | ) 46 | role.add_to_policy(iam.PolicyStatement( 47 | resources=["*"], 48 | actions=["ec2:AuthorizeSecurityGroupEgress", 49 | "ec2:AuthorizeSecurityGroupIngress", 50 | "ec2:CreateSecurityGroup", 51 | "ec2:CreateTags", 52 | "ec2:DescribeSecurityGroups", 53 | "ec2:RevokeSecurityGroupEgress", 54 | "ec2:RevokeSecurityGroupIngress", 55 | "ec2:CreateNetworkInterface", 56 | "ec2:CreateNetworkInterfacePermission", 57 | "ec2:DeleteNetworkInterface", 58 | "ec2:DeleteNetworkInterfacePermission", 59 | "ec2:DescribeNetworkInterfaces", 60 | "ec2:ModifyNetworkInterfaceAttribute", 61 | "ec2:DescribeTags", 62 | "ec2:DescribeInstances", 63 | "ec2:DescribeSubnets", 64 | "ec2:DescribeVpcs", 65 | "elasticmapreduce:ListInstances", 66 | "elasticmapreduce:DescribeCluster", 67 | "elasticmapreduce:ListSteps"], 68 | effect=iam.Effect.ALLOW 69 | )) 70 | core.Tags.of(role).add("for-use-with-amazon-emr-managed-policies", "true") 71 | 72 | user_role = iam.Role(self, "StudioUserRole", 73 | assumed_by=iam.ServicePrincipal("elasticmapreduce.amazonaws.com") 74 | ) 75 | core.Tags.of(role).add("for-use-with-amazon-emr-managed-policies", "true") 76 | user_role.add_to_policy(iam.PolicyStatement( 77 | actions=["elasticmapreduce:CreateEditor", 78 | "elasticmapreduce:DescribeEditor", 79 | "elasticmapreduce:ListEditors", 80 | "elasticmapreduce:StartEditor", 81 | "elasticmapreduce:StopEditor", 82 | "elasticmapreduce:DeleteEditor", 83 | "elasticmapreduce:OpenEditorInConsole", 84 | "elasticmapreduce:AttachEditor", 85 | "elasticmapreduce:DetachEditor", 86 | "elasticmapreduce:CreateRepository", 87 | "elasticmapreduce:DescribeRepository", 88 | "elasticmapreduce:DeleteRepository", 89 | "elasticmapreduce:ListRepositories", 90 | "elasticmapreduce:LinkRepository", 91 | "elasticmapreduce:UnlinkRepository", 92 | "elasticmapreduce:DescribeCluster", 93 | "elasticmapreduce:ListInstanceGroups", 94 | "elasticmapreduce:ListBootstrapActions", 95 | "elasticmapreduce:ListClusters", 96 | "elasticmapreduce:ListSteps", 97 | "elasticmapreduce:CreatePersistentAppUI", 98 | "elasticmapreduce:DescribePersistentAppUI", 99 | "elasticmapreduce:GetPersistentAppUIPresignedURL", 100 | "secretsmanager:CreateSecret", 101 | "secretsmanager:ListSecrets", 102 | "secretsmanager:TagResource", 103 | "emr-containers:DescribeVirtualCluster", 104 | "emr-containers:ListVirtualClusters", 105 | "emr-containers:DescribeManagedEndpoint", 106 | "emr-containers:ListManagedEndpoints", 107 | "emr-containers:CreateAccessTokenForManagedEndpoint", 108 | "emr-containers:DescribeJobRun", 109 | "emr-containers:ListJobRuns"], 110 | resources=["*"], 111 | effect=iam.Effect.ALLOW 112 | )) 113 | user_role.add_to_policy(iam.PolicyStatement( 114 | resources=["*"], 115 | actions=["servicecatalog:DescribeProduct", 116 | "servicecatalog:DescribeProductView", 117 | "servicecatalog:DescribeProvisioningParameters", 118 | "servicecatalog:ProvisionProduct", 119 | "servicecatalog:SearchProducts", 120 | "servicecatalog:UpdateProvisionedProduct", 121 | "servicecatalog:ListProvisioningArtifacts", 122 | "servicecatalog:DescribeRecord", 123 | "cloudformation:DescribeStackResources"], 124 | effect=iam.Effect.ALLOW 125 | )) 126 | user_role.add_to_policy(iam.PolicyStatement( 127 | resources=["*"], 128 | actions=["elasticmapreduce:RunJobFlow"], 129 | effect=iam.Effect.ALLOW 130 | )) 131 | user_role.add_to_policy(iam.PolicyStatement( 132 | resources=[role.role_arn, 133 | f"arn:aws:iam::{self.account}:role/EMR_DefaultRole", 134 | f"arn:aws:iam::{self.account}:role/EMR_EC2_DefaultRole"], 135 | actions=["iam:PassRole"], 136 | effect=iam.Effect.ALLOW 137 | )) 138 | user_role.add_to_policy(iam.PolicyStatement( 139 | resources=["arn:aws:s3:::*"], 140 | actions=["s3:ListAllMyBuckets", 141 | "s3:ListBucket", 142 | "s3:GetBucketLocation"], 143 | effect=iam.Effect.ALLOW 144 | )) 145 | user_role.add_to_policy(iam.PolicyStatement( 146 | resources=[f"arn:aws:s3:::{bucket.bucket_name}/*", 147 | f"arn:aws:s3:::aws-logs-{self.account}-{self.region}/elasticmapreduce/*"], 148 | actions=["s3:GetObject"], 149 | effect=iam.Effect.ALLOW 150 | )) 151 | 152 | policy_document = { 153 | "Version": "2012-10-17T00:00:00.000Z", 154 | "Statement": [ 155 | { 156 | "Action": [ 157 | "elasticmapreduce:CreateEditor", 158 | "elasticmapreduce:DescribeEditor", 159 | "elasticmapreduce:ListEditors", 160 | "elasticmapreduce:StartEditor", 161 | "elasticmapreduce:StopEditor", 162 | "elasticmapreduce:DeleteEditor", 163 | "elasticmapreduce:OpenEditorInConsole", 164 | "elasticmapreduce:AttachEditor", 165 | "elasticmapreduce:DetachEditor", 166 | "elasticmapreduce:CreateRepository", 167 | "elasticmapreduce:DescribeRepository", 168 | "elasticmapreduce:DeleteRepository", 169 | "elasticmapreduce:ListRepositories", 170 | "elasticmapreduce:LinkRepository", 171 | "elasticmapreduce:UnlinkRepository", 172 | "elasticmapreduce:DescribeCluster", 173 | "elasticmapreduce:ListInstanceGroups", 174 | "elasticmapreduce:ListBootstrapActions", 175 | "elasticmapreduce:ListClusters", 176 | "elasticmapreduce:ListSteps", 177 | "elasticmapreduce:CreatePersistentAppUI", 178 | "elasticmapreduce:DescribePersistentAppUI", 179 | "elasticmapreduce:GetPersistentAppUIPresignedURL", 180 | "secretsmanager:CreateSecret", 181 | "secretsmanager:ListSecrets", 182 | "emr-containers:DescribeVirtualCluster", 183 | "emr-containers:ListVirtualClusters", 184 | "emr-containers:DescribeManagedEndpoint", 185 | "emr-containers:ListManagedEndpoints", 186 | "emr-containers:CreateAccessTokenForManagedEndpoint", 187 | "emr-containers:DescribeJobRun", 188 | "emr-containers:ListJobRuns" 189 | ], 190 | "Resource": "*", 191 | "Effect": "Allow", 192 | "Sid": "AllowBasicActions" 193 | }, 194 | { 195 | "Action": [ 196 | "servicecatalog:DescribeProduct", 197 | "servicecatalog:DescribeProductView", 198 | "servicecatalog:DescribeProvisioningParameters", 199 | "servicecatalog:ProvisionProduct", 200 | "servicecatalog:SearchProducts", 201 | "servicecatalog:UpdateProvisionedProduct", 202 | "servicecatalog:ListProvisioningArtifacts", 203 | "servicecatalog:DescribeRecord", 204 | "cloudformation:DescribeStackResources" 205 | ], 206 | "Resource": "*", 207 | "Effect": "Allow", 208 | "Sid": "AllowIntermediateActions" 209 | }, 210 | { 211 | "Action": [ 212 | "elasticmapreduce:RunJobFlow" 213 | ], 214 | "Resource": "*", 215 | "Effect": "Allow", 216 | "Sid": "AllowAdvancedActions" 217 | }, 218 | { 219 | "Action": "iam:PassRole", 220 | "Resource": [ 221 | role.role_arn, 222 | f"arn:aws:iam::{self.account}:role/EMR_DefaultRole", 223 | f"arn:aws:iam::{self.account}:role/EMR_EC2_DefaultRole" 224 | ], 225 | "Effect": "Allow", 226 | "Sid": "PassRolePermission" 227 | }, 228 | { 229 | "Action": [ 230 | "s3:ListAllMyBuckets", 231 | "s3:ListBucket", 232 | "s3:GetBucketLocation" 233 | ], 234 | "Resource": "arn:aws:s3:::*", 235 | "Effect": "Allow", 236 | "Sid": "S3ListPermission" 237 | }, 238 | { 239 | "Action": [ 240 | "s3:GetObject" 241 | ], 242 | "Resource": [ 243 | f"arn:aws:s3:::{bucket.bucket_name}/*", 244 | f"arn:aws:s3:::aws-logs-{self.account}-{self.region}/elasticmapreduce/*" 245 | ], 246 | "Effect": "Allow", 247 | "Sid": "S3GetObjectPermission" 248 | } 249 | ] 250 | } 251 | custom_policy_document = iam.PolicyDocument.from_json(policy_document) 252 | new_managed_policy = iam.ManagedPolicy(self, "LBControlPolicy", 253 | document=custom_policy_document 254 | ) 255 | 256 | # Set up Studio 257 | studio = emr.CfnStudio(self, "MyEmrStudio", 258 | auth_mode = "SSO", default_s3_location = f"s3://{bucket.bucket_name}/studio/", 259 | engine_security_group_id = eng_sg.security_group_id, 260 | name = "MyEmrEksStudio", 261 | service_role = role.role_arn, 262 | subnet_ids = [n.subnet_id for n in vpc.private_subnets], 263 | user_role = user_role.role_arn, 264 | vpc_id = vpc.vpc_id, 265 | workspace_security_group_id = ws_sg.security_group_id, 266 | description=None, 267 | tags=None) 268 | core.CfnOutput( 269 | self, "StudioUrl", 270 | value=studio.attr_url 271 | ) 272 | 273 | # Create session mapping 274 | studiosm = emr.CfnStudioSessionMapping(self, "MyStudioSM", 275 | identity_name = self.node.try_get_context("username"), 276 | identity_type = "USER", 277 | session_policy_arn = new_managed_policy.managed_policy_arn, 278 | studio_id = studio.attr_studio_id) -------------------------------------------------------------------------------- /emr_eks_cdk/emr_eks_cdk_stack.py: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | from aws_cdk import aws_ec2 as ec2, aws_eks as eks, core, aws_emrcontainers as emrc, aws_iam as iam, aws_logs as logs, custom_resources as custom 5 | 6 | """ 7 | This stack deploys the following: 8 | - VPC 9 | - EKS cluster 10 | - EKS configuration to support EMR on EKS 11 | - EMR on EKS virtual cluster 12 | - CloudWatch log group 13 | """ 14 | class EmrEksCdkStack(core.Stack): 15 | 16 | 17 | def __init__(self, scope: core.Construct, construct_id: str, **kwargs) -> None: 18 | super().__init__(scope, construct_id, **kwargs) 19 | self.emr_namespace = "sparkns" 20 | self.emr_namespace_fg = "sparkfg" 21 | self.emrsvcrolearn = f"arn:aws:iam::{self.account}:role/AWSServiceRoleForAmazonEMRContainers" 22 | self.instance_type = self.node.try_get_context("instance") 23 | 24 | # VPC 25 | self.vpc = ec2.Vpc( 26 | self, 27 | "EMR-EKS-VPC", 28 | max_azs=3 29 | ) 30 | core.Tags.of(self.vpc).add("for-use-with-amazon-emr-managed-policies", "true") 31 | 32 | # EKS cluster 33 | self.cluster = eks.Cluster(self, "EksForSpark", 34 | version=eks.KubernetesVersion.V1_18, 35 | default_capacity=0, 36 | endpoint_access=eks.EndpointAccess.PUBLIC_AND_PRIVATE, 37 | vpc=self.vpc, 38 | vpc_subnets=[ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE)] 39 | ) 40 | 41 | 42 | # Default node group 43 | self.ng = self.cluster.add_nodegroup_capacity("base-node-group", 44 | instance_types=[ec2.InstanceType(self.instance_type)], 45 | min_size=3, 46 | max_size=10, 47 | disk_size=50 48 | ) 49 | 50 | # Create namespaces for EMR to use 51 | namespace = self.cluster.add_manifest(self.emr_namespace, { 52 | "apiVersion":"v1", 53 | "kind":"Namespace", 54 | "metadata":{"name": self.emr_namespace}, 55 | }) 56 | namespace_fg = self.cluster.add_manifest(self.emr_namespace_fg, { 57 | "apiVersion":"v1", 58 | "kind":"Namespace", 59 | "metadata":{"name": self.emr_namespace_fg}, 60 | }) 61 | 62 | # Fargate profile 63 | fgprofile = eks.FargateProfile(self, "SparkFargateProfile", cluster=self.cluster, 64 | selectors=[{"namespace": self.emr_namespace_fg}] 65 | ) 66 | 67 | # Create k8s cluster role for EMR 68 | emrrole = self.cluster.add_manifest("emrrole", { 69 | "apiVersion":"rbac.authorization.k8s.io/v1", 70 | "kind":"Role", 71 | "metadata":{"name": "emr-containers", "namespace": self.emr_namespace}, 72 | "rules": [ 73 | {"apiGroups": [""], "resources":["namespaces"],"verbs":["get"]}, 74 | {"apiGroups": [""], "resources":["serviceaccounts", "services", "configmaps", "events", "pods", "pods/log"],"verbs":["get", "list", "watch", "describe", "create", "edit", "delete", "deletecollection", "annotate", "patch", "label"]}, 75 | {"apiGroups": [""], "resources":["secrets"],"verbs":["create", "patch", "delete", "watch"]}, 76 | {"apiGroups": ["apps"], "resources":["statefulsets", "deployments"],"verbs":["get", "list", "watch", "describe", "create", "edit", "delete", "annotate", "patch", "label"]}, 77 | {"apiGroups": ["batch"], "resources":["jobs"],"verbs":["get", "list", "watch", "describe", "create", "edit", "delete", "annotate", "patch", "label"]}, 78 | {"apiGroups": ["extensions"], "resources":["ingresses"],"verbs":["get", "list", "watch", "describe", "create", "edit", "delete", "annotate", "patch", "label"]}, 79 | {"apiGroups": ["rbac.authorization.k8s.io"], "resources":["roles", "rolebindings"],"verbs":["get", "list", "watch", "describe", "create", "edit", "delete", "deletecollection", "annotate", "patch", "label"]} 80 | ] 81 | }) 82 | emrrole.node.add_dependency(namespace) 83 | emrrole_fg = self.cluster.add_manifest("emrrole_fg", { 84 | "apiVersion":"rbac.authorization.k8s.io/v1", 85 | "kind":"Role", 86 | "metadata":{"name": "emr-containers", "namespace": self.emr_namespace_fg}, 87 | "rules": [ 88 | {"apiGroups": [""], "resources":["namespaces"],"verbs":["get"]}, 89 | {"apiGroups": [""], "resources":["serviceaccounts", "services", "configmaps", "events", "pods", "pods/log"],"verbs":["get", "list", "watch", "describe", "create", "edit", "delete", "deletecollection", "annotate", "patch", "label"]}, 90 | {"apiGroups": [""], "resources":["secrets"],"verbs":["create", "patch", "delete", "watch"]}, 91 | {"apiGroups": ["apps"], "resources":["statefulsets", "deployments"],"verbs":["get", "list", "watch", "describe", "create", "edit", "delete", "annotate", "patch", "label"]}, 92 | {"apiGroups": ["batch"], "resources":["jobs"],"verbs":["get", "list", "watch", "describe", "create", "edit", "delete", "annotate", "patch", "label"]}, 93 | {"apiGroups": ["extensions"], "resources":["ingresses"],"verbs":["get", "list", "watch", "describe", "create", "edit", "delete", "annotate", "patch", "label"]}, 94 | {"apiGroups": ["rbac.authorization.k8s.io"], "resources":["roles", "rolebindings"],"verbs":["get", "list", "watch", "describe", "create", "edit", "delete", "deletecollection", "annotate", "patch", "label"]} 95 | ] 96 | }) 97 | emrrole_fg.node.add_dependency(namespace_fg) 98 | 99 | # Bind cluster role to user 100 | emrrolebind = self.cluster.add_manifest("emrrolebind", { 101 | "apiVersion":"rbac.authorization.k8s.io/v1", 102 | "kind":"RoleBinding", 103 | "metadata":{"name": "emr-containers", "namespace": self.emr_namespace}, 104 | "subjects":[{"kind": "User","name":"emr-containers","apiGroup": "rbac.authorization.k8s.io"}], 105 | "roleRef":{"kind":"Role","name":"emr-containers","apiGroup": "rbac.authorization.k8s.io"} 106 | }) 107 | emrrolebind.node.add_dependency(emrrole) 108 | emrrolebind_fg = self.cluster.add_manifest("emrrolebind_fg", { 109 | "apiVersion":"rbac.authorization.k8s.io/v1", 110 | "kind":"RoleBinding", 111 | "metadata":{"name": "emr-containers", "namespace": self.emr_namespace_fg}, 112 | "subjects":[{"kind": "User","name":"emr-containers","apiGroup": "rbac.authorization.k8s.io"}], 113 | "roleRef":{"kind":"Role","name":"emr-containers","apiGroup": "rbac.authorization.k8s.io"} 114 | }) 115 | emrrolebind_fg.node.add_dependency(emrrole_fg) 116 | 117 | # Map user to IAM role 118 | emrsvcrole = iam.Role.from_role_arn(self, "EmrSvcRole", self.emrsvcrolearn, mutable=False) 119 | self.cluster.aws_auth.add_role_mapping(emrsvcrole, groups=[], username="emr-containers") 120 | 121 | # Job execution role 122 | self.job_role = iam.Role(self, "EMR_EKS_Job_Role", assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), 123 | managed_policies=[ 124 | iam.ManagedPolicy.from_aws_managed_policy_name("AmazonS3FullAccess"), 125 | iam.ManagedPolicy.from_aws_managed_policy_name("AmazonEC2FullAccess"), 126 | iam.ManagedPolicy.from_aws_managed_policy_name("AWSGlueConsoleFullAccess"), 127 | iam.ManagedPolicy.from_aws_managed_policy_name("CloudWatchFullAccess")]) 128 | core.CfnOutput( 129 | self, "JobRoleArn", 130 | value=self.job_role.role_arn 131 | ) 132 | 133 | # Modify trust policy 134 | string_like = core.CfnJson(self, "ConditionJson", 135 | value={ 136 | f"{self.cluster.cluster_open_id_connect_issuer}:sub": f"system:serviceaccount:emr:emr-containers-sa-*-*-{self.account}-*" 137 | } 138 | ) 139 | self.job_role.assume_role_policy.add_statements( 140 | iam.PolicyStatement( 141 | effect=iam.Effect.ALLOW, 142 | actions=["sts:AssumeRoleWithWebIdentity"], 143 | principals=[iam.OpenIdConnectPrincipal(self.cluster.open_id_connect_provider, conditions={"StringLike": string_like})] 144 | ) 145 | ) 146 | string_aud = core.CfnJson(self, "ConditionJsonAud", 147 | value={ 148 | f"{self.cluster.cluster_open_id_connect_issuer}:aud": "sts.amazon.com" 149 | } 150 | ) 151 | self.job_role.assume_role_policy.add_statements( 152 | iam.PolicyStatement( 153 | effect=iam.Effect.ALLOW, 154 | actions=["sts:AssumeRoleWithWebIdentity"], 155 | principals=[iam.OpenIdConnectPrincipal(self.cluster.open_id_connect_provider, conditions={"StringEquals": string_aud})] 156 | ) 157 | ) 158 | 159 | # EMR virtual cluster 160 | self.emr_vc = emrc.CfnVirtualCluster(scope=self, 161 | id="EMRCluster", 162 | container_provider=emrc.CfnVirtualCluster.ContainerProviderProperty(id=self.cluster.cluster_name, 163 | info=emrc.CfnVirtualCluster.ContainerInfoProperty(eks_info=emrc.CfnVirtualCluster.EksInfoProperty(namespace=self.emr_namespace)), 164 | type="EKS" 165 | ), 166 | name="EMRCluster" 167 | ) 168 | self.emr_vc.node.add_dependency(namespace) 169 | self.emr_vc.node.add_dependency(emrrolebind) 170 | emr_vc_fg = emrc.CfnVirtualCluster(scope=self, 171 | id="EMRClusterFG", 172 | container_provider=emrc.CfnVirtualCluster.ContainerProviderProperty(id=self.cluster.cluster_name, 173 | info=emrc.CfnVirtualCluster.ContainerInfoProperty(eks_info=emrc.CfnVirtualCluster.EksInfoProperty(namespace=self.emr_namespace_fg)), 174 | type="EKS" 175 | ), 176 | name="EMRClusterFG" 177 | ) 178 | emr_vc_fg.node.add_dependency(namespace_fg) 179 | emr_vc_fg.node.add_dependency(emrrolebind_fg) 180 | core.CfnOutput( 181 | self, "VirtualClusterId", 182 | value=self.emr_vc.attr_id 183 | ) 184 | core.CfnOutput( 185 | self, "FgVirtualClusterId", 186 | value=emr_vc_fg.attr_id 187 | ) 188 | 189 | # Create log group 190 | log_group = logs.LogGroup(self, "LogGroup") 191 | core.CfnOutput( 192 | self, "LogGroupName", 193 | value=log_group.log_group_name 194 | ) 195 | 196 | # LB Controller role 197 | self.policy_document = { 198 | "Version": "2012-10-17", 199 | "Statement": [ 200 | { 201 | "Effect": "Allow", 202 | "Action": [ 203 | "iam:CreateServiceLinkedRole", 204 | "ec2:DescribeAccountAttributes", 205 | "ec2:DescribeAddresses", 206 | "ec2:DescribeAvailabilityZones", 207 | "ec2:DescribeInternetGateways", 208 | "ec2:DescribeVpcs", 209 | "ec2:DescribeSubnets", 210 | "ec2:DescribeSecurityGroups", 211 | "ec2:DescribeInstances", 212 | "ec2:DescribeNetworkInterfaces", 213 | "ec2:DescribeTags", 214 | "ec2:GetCoipPoolUsage", 215 | "ec2:DescribeCoipPools", 216 | "elasticloadbalancing:DescribeLoadBalancers", 217 | "elasticloadbalancing:DescribeLoadBalancerAttributes", 218 | "elasticloadbalancing:DescribeListeners", 219 | "elasticloadbalancing:DescribeListenerCertificates", 220 | "elasticloadbalancing:DescribeSSLPolicies", 221 | "elasticloadbalancing:DescribeRules", 222 | "elasticloadbalancing:DescribeTargetGroups", 223 | "elasticloadbalancing:DescribeTargetGroupAttributes", 224 | "elasticloadbalancing:DescribeTargetHealth", 225 | "elasticloadbalancing:DescribeTags" 226 | ], 227 | "Resource": "*" 228 | }, 229 | { 230 | "Effect": "Allow", 231 | "Action": [ 232 | "cognito-idp:DescribeUserPoolClient", 233 | "acm:ListCertificates", 234 | "acm:DescribeCertificate", 235 | "iam:ListServerCertificates", 236 | "iam:GetServerCertificate", 237 | "waf-regional:GetWebACL", 238 | "waf-regional:GetWebACLForResource", 239 | "waf-regional:AssociateWebACL", 240 | "waf-regional:DisassociateWebACL", 241 | "wafv2:GetWebACL", 242 | "wafv2:GetWebACLForResource", 243 | "wafv2:AssociateWebACL", 244 | "wafv2:DisassociateWebACL", 245 | "shield:GetSubscriptionState", 246 | "shield:DescribeProtection", 247 | "shield:CreateProtection", 248 | "shield:DeleteProtection" 249 | ], 250 | "Resource": "*" 251 | }, 252 | { 253 | "Effect": "Allow", 254 | "Action": [ 255 | "ec2:AuthorizeSecurityGroupIngress", 256 | "ec2:RevokeSecurityGroupIngress" 257 | ], 258 | "Resource": "*" 259 | }, 260 | { 261 | "Effect": "Allow", 262 | "Action": [ 263 | "ec2:CreateSecurityGroup" 264 | ], 265 | "Resource": "*" 266 | }, 267 | { 268 | "Effect": "Allow", 269 | "Action": [ 270 | "ec2:CreateTags" 271 | ], 272 | "Resource": "arn:aws:ec2:*:*:security-group/*", 273 | "Condition": { 274 | "StringEquals": { 275 | "ec2:CreateAction": "CreateSecurityGroup" 276 | }, 277 | "Null": { 278 | "aws:RequestTag/elbv2.k8s.aws/cluster": False 279 | } 280 | } 281 | }, 282 | { 283 | "Effect": "Allow", 284 | "Action": [ 285 | "ec2:CreateTags", 286 | "ec2:DeleteTags" 287 | ], 288 | "Resource": "arn:aws:ec2:*:*:security-group/*", 289 | "Condition": { 290 | "Null": { 291 | "aws:RequestTag/elbv2.k8s.aws/cluster": True, 292 | "aws:ResourceTag/elbv2.k8s.aws/cluster": False 293 | } 294 | } 295 | }, 296 | { 297 | "Effect": "Allow", 298 | "Action": [ 299 | "ec2:AuthorizeSecurityGroupIngress", 300 | "ec2:RevokeSecurityGroupIngress", 301 | "ec2:DeleteSecurityGroup" 302 | ], 303 | "Resource": "*", 304 | "Condition": { 305 | "Null": { 306 | "aws:ResourceTag/elbv2.k8s.aws/cluster": False 307 | } 308 | } 309 | }, 310 | { 311 | "Effect": "Allow", 312 | "Action": [ 313 | "elasticloadbalancing:CreateLoadBalancer", 314 | "elasticloadbalancing:CreateTargetGroup" 315 | ], 316 | "Resource": "*", 317 | "Condition": { 318 | "Null": { 319 | "aws:RequestTag/elbv2.k8s.aws/cluster": False 320 | } 321 | } 322 | }, 323 | { 324 | "Effect": "Allow", 325 | "Action": [ 326 | "elasticloadbalancing:CreateListener", 327 | "elasticloadbalancing:DeleteListener", 328 | "elasticloadbalancing:CreateRule", 329 | "elasticloadbalancing:DeleteRule" 330 | ], 331 | "Resource": "*" 332 | }, 333 | { 334 | "Effect": "Allow", 335 | "Action": [ 336 | "elasticloadbalancing:AddTags", 337 | "elasticloadbalancing:RemoveTags" 338 | ], 339 | "Resource": [ 340 | "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", 341 | "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", 342 | "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" 343 | ], 344 | "Condition": { 345 | "Null": { 346 | "aws:RequestTag/elbv2.k8s.aws/cluster": True, 347 | "aws:ResourceTag/elbv2.k8s.aws/cluster": False 348 | } 349 | } 350 | }, 351 | { 352 | "Effect": "Allow", 353 | "Action": [ 354 | "elasticloadbalancing:AddTags", 355 | "elasticloadbalancing:RemoveTags" 356 | ], 357 | "Resource": [ 358 | "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*", 359 | "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*", 360 | "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*", 361 | "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*" 362 | ] 363 | }, 364 | { 365 | "Effect": "Allow", 366 | "Action": [ 367 | "elasticloadbalancing:ModifyLoadBalancerAttributes", 368 | "elasticloadbalancing:SetIpAddressType", 369 | "elasticloadbalancing:SetSecurityGroups", 370 | "elasticloadbalancing:SetSubnets", 371 | "elasticloadbalancing:DeleteLoadBalancer", 372 | "elasticloadbalancing:ModifyTargetGroup", 373 | "elasticloadbalancing:ModifyTargetGroupAttributes", 374 | "elasticloadbalancing:DeleteTargetGroup" 375 | ], 376 | "Resource": "*", 377 | "Condition": { 378 | "Null": { 379 | "aws:ResourceTag/elbv2.k8s.aws/cluster": False 380 | } 381 | } 382 | }, 383 | { 384 | "Effect": "Allow", 385 | "Action": [ 386 | "elasticloadbalancing:RegisterTargets", 387 | "elasticloadbalancing:DeregisterTargets" 388 | ], 389 | "Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" 390 | }, 391 | { 392 | "Effect": "Allow", 393 | "Action": [ 394 | "elasticloadbalancing:SetWebAcl", 395 | "elasticloadbalancing:ModifyListener", 396 | "elasticloadbalancing:AddListenerCertificates", 397 | "elasticloadbalancing:RemoveListenerCertificates", 398 | "elasticloadbalancing:ModifyRule" 399 | ], 400 | "Resource": "*" 401 | } 402 | ] 403 | } 404 | self.custom_policy_document = iam.PolicyDocument.from_json(self.policy_document) 405 | self.new_managed_policy = iam.ManagedPolicy(self, "LBControlPolicy", 406 | document=self.custom_policy_document 407 | ) 408 | self.lb_role = iam.Role(self, "LBControllerRole", assumed_by=iam.ServicePrincipal("ec2.amazonaws.com"), 409 | managed_policies=[ 410 | self.new_managed_policy 411 | ]) 412 | string_eq = core.CfnJson(self, "ConditionJsonEq", 413 | value={ 414 | f"{self.cluster.cluster_open_id_connect_issuer}:sub": f"system:serviceaccount:kube-system:aws-load-balancer-controller" 415 | } 416 | ) 417 | self.lb_role.assume_role_policy.add_statements( 418 | iam.PolicyStatement( 419 | effect=iam.Effect.ALLOW, 420 | actions=["sts:AssumeRoleWithWebIdentity"], 421 | principals=[iam.OpenIdConnectPrincipal(self.cluster.open_id_connect_provider, conditions={"StringEquals": string_eq})] 422 | ) 423 | ) 424 | 425 | # Service Account 426 | self.lb_svc_acct = self.cluster.add_manifest("lb_svc_acct", { 427 | "apiVersion":"v1", 428 | "kind":"ServiceAccount", 429 | "metadata":{ 430 | "labels": { 431 | "app.kubernetes.io/component": "controller", 432 | "app.kubernetes.io/name": "aws-load-balancer-controller" 433 | }, 434 | "name": "aws-load-balancer-controller", 435 | "namespace": "kube-system", 436 | "annotations": {"eks.amazonaws.com/role-arn": self.lb_role.role_arn} 437 | }, 438 | }) 439 | 440 | # Helm chart 441 | self.cluster.add_helm_chart("lbcontroller", 442 | chart="aws-load-balancer-controller", 443 | repository="https://aws.github.io/eks-charts", 444 | release="aws-load-balancer-controller", 445 | namespace="kube-system", 446 | values= { 447 | "clusterName": self.cluster.cluster_name, 448 | "region": self.region, 449 | "vpcId": self.vpc.vpc_id, 450 | "serviceAccount": { 451 | "create": False, 452 | "name": "aws-load-balancer-controller" 453 | } 454 | }, 455 | wait=True 456 | ) 457 | --------------------------------------------------------------------------------