├── .gitignore ├── Best_Practices.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── airflow ├── Dockerfile ├── README.md ├── dags │ ├── example_emr_serverless.py │ └── example_end_to_end.py ├── emr_serverless │ ├── hooks │ │ └── emr.py │ ├── operators │ │ └── emr.py │ └── sensors │ │ └── emr.py ├── requirements-dev.txt ├── setup.py └── tests │ ├── test_dag_integrity.py │ └── test_simple_submit.py ├── cdk ├── README.md ├── emr-serverless-with-mwaa │ ├── .gitignore │ ├── README.md │ ├── app.py │ ├── assets │ │ └── airflow │ │ │ ├── dags │ │ │ ├── example_emr_serverless.py │ │ │ └── example_end_to_end.py │ │ │ └── requirements.txt │ ├── cdk.json │ ├── requirements-dev.txt │ ├── requirements.txt │ ├── source.bat │ ├── stacks │ │ ├── common.py │ │ ├── emr_serverless.py │ │ ├── emr_studio.py │ │ ├── mwaa.py │ │ └── vpc.py │ └── tests │ │ ├── __init__.py │ │ └── unit │ │ ├── __init__.py │ │ └── test_emr_serverless_with_mwaa_stack.py └── emr-serverless-with-sfn │ ├── .gitignore │ ├── README.md │ ├── app.py │ ├── assets │ └── jobs │ │ ├── pyspark-reader-example.py │ │ └── pyspark-writer-example.py │ ├── cdk.json │ ├── requirements-dev.txt │ ├── requirements.txt │ ├── source.bat │ ├── stacks │ ├── __init__.py │ ├── emr_serverless.py │ ├── emr_serverless_sm.py │ ├── emr_studio.py │ ├── sfn.py │ └── vpc.py │ └── tests │ ├── __init__.py │ └── unit │ ├── __init__.py │ └── test_emr_serverless_with_sfn_stack.py ├── cloudformation ├── README.md ├── emr-serverless-cloudwatch-dashboard │ ├── README.md │ ├── emr_serverless_cloudwatch_dashboard.yaml │ └── images │ │ ├── application_metrics.png │ │ ├── demo │ │ ├── initial_state.png │ │ ├── pi-default.png │ │ ├── pi-drivers-chart.png │ │ ├── pi-drivers.png │ │ └── pi-more-executors.png │ │ ├── driver_metrics.png │ │ ├── executor_metrics.png │ │ ├── job-metrics-breakdown.png │ │ ├── job-runs-overview.png │ │ ├── job_worker_metrics_Driver.jpg │ │ ├── job_worker_metrics_Executor.jpeg │ │ ├── pre_initialized_capacity_metrics.png │ │ └── snapshot_metrics.png ├── emr_serverless_full_deployment.yaml ├── emr_serverless_spark_app.yaml ├── kds_emr_serverless.yaml ├── lake-formation-and-apache-iceberg │ ├── README.md │ ├── athena_create.png │ ├── lake_formation_iceberg.yaml │ └── show_table.py └── mwaa_emr_serverless.yaml ├── examples ├── hive │ ├── README.md │ ├── create_table.sql │ └── extreme_weather.sql ├── java-api │ ├── README.md │ └── myapp │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── myapp │ │ │ ├── App.java │ │ │ └── EMRServerlessService.java │ │ └── test │ │ └── java │ │ └── com │ │ └── example │ │ └── myapp │ │ └── AppTest.java ├── java │ ├── README.md │ └── hello-world │ │ ├── .gitignore │ │ ├── pom.xml │ │ └── src │ │ └── main │ │ └── java │ │ └── HelloWorld.java ├── pyspark │ ├── README.md │ ├── custom-images │ │ ├── Dockerfile │ │ ├── README.md │ │ └── noaa_slugplot.py │ ├── custom_python_version │ │ ├── Dockerfile │ │ ├── README.md │ │ └── python_3.10.py │ ├── delta-lake │ │ ├── Dockerfile │ │ ├── README.md │ │ ├── main.py │ │ ├── poetry.lock │ │ └── pyproject.toml │ ├── dependencies │ │ ├── Dockerfile │ │ ├── Dockerfile.al2023 │ │ ├── Dockerfile.jars │ │ ├── README.md │ │ ├── ge_profile.py │ │ ├── pg_query.py │ │ └── pom.xml │ ├── extreme_weather.py │ ├── genomic │ │ ├── .gitignore │ │ ├── Dockerfile │ │ ├── Dockerfile.jars │ │ ├── README.md │ │ └── glow_demo.py │ └── streaming │ │ ├── Kinesis_data_stream │ │ ├── README.md │ │ └── transactions.py │ │ ├── README.md │ │ └── images │ │ ├── README.md │ │ ├── architecture-image.png │ │ ├── emrs-properties-image.png │ │ ├── kdg-image.png │ │ ├── results-image.png │ │ └── viewlogs-image.png └── python-api │ ├── .gitignore │ ├── README.md │ └── emr_serverless.py └── utilities ├── local-dev ├── Dockerfile ├── README.md └── jupyter-server.png ├── spark-ui ├── Dockerfile └── README.md └── tez-ui ├── Dockerfile ├── README.md ├── entrypoint.sh ├── event-log-sync.sh ├── hadoop-layout.sh ├── pom.xml ├── tez-ui.patch └── yarn-site.xml /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/.gitignore -------------------------------------------------------------------------------- /Best_Practices.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/Best_Practices.md -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/README.md -------------------------------------------------------------------------------- /airflow/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/airflow/Dockerfile -------------------------------------------------------------------------------- /airflow/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/airflow/README.md -------------------------------------------------------------------------------- /airflow/dags/example_emr_serverless.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/airflow/dags/example_emr_serverless.py -------------------------------------------------------------------------------- /airflow/dags/example_end_to_end.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/airflow/dags/example_end_to_end.py -------------------------------------------------------------------------------- /airflow/emr_serverless/hooks/emr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/airflow/emr_serverless/hooks/emr.py -------------------------------------------------------------------------------- /airflow/emr_serverless/operators/emr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/airflow/emr_serverless/operators/emr.py -------------------------------------------------------------------------------- /airflow/emr_serverless/sensors/emr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/airflow/emr_serverless/sensors/emr.py -------------------------------------------------------------------------------- /airflow/requirements-dev.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/airflow/requirements-dev.txt -------------------------------------------------------------------------------- /airflow/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/airflow/setup.py -------------------------------------------------------------------------------- /airflow/tests/test_dag_integrity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/airflow/tests/test_dag_integrity.py -------------------------------------------------------------------------------- /airflow/tests/test_simple_submit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/airflow/tests/test_simple_submit.py -------------------------------------------------------------------------------- /cdk/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/README.md -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/.gitignore -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/README.md -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/app.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/assets/airflow/dags/example_emr_serverless.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/assets/airflow/dags/example_emr_serverless.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/assets/airflow/dags/example_end_to_end.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/assets/airflow/dags/example_end_to_end.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/assets/airflow/requirements.txt: -------------------------------------------------------------------------------- 1 | apache-airflow-providers-amazon==7.0.0 2 | boto3>=1.23.9 -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/cdk.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/cdk.json -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest==6.2.5 2 | -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/requirements.txt -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/source.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/source.bat -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/stacks/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/stacks/common.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/stacks/emr_serverless.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/stacks/emr_serverless.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/stacks/emr_studio.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/stacks/emr_studio.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/stacks/mwaa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/stacks/mwaa.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/stacks/vpc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/stacks/vpc.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cdk/emr-serverless-with-mwaa/tests/unit/test_emr_serverless_with_mwaa_stack.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-mwaa/tests/unit/test_emr_serverless_with_mwaa_stack.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/.gitignore -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/README.md -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/app.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/assets/jobs/pyspark-reader-example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/assets/jobs/pyspark-reader-example.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/assets/jobs/pyspark-writer-example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/assets/jobs/pyspark-writer-example.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/cdk.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/cdk.json -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest==6.2.5 2 | pyspark==3.4.1 3 | -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/requirements.txt -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/source.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/source.bat -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/stacks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/stacks/emr_serverless.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/stacks/emr_serverless.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/stacks/emr_serverless_sm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/stacks/emr_serverless_sm.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/stacks/emr_studio.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/stacks/emr_studio.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/stacks/sfn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/stacks/sfn.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/stacks/vpc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/stacks/vpc.py -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cdk/emr-serverless-with-sfn/tests/unit/test_emr_serverless_with_sfn_stack.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cdk/emr-serverless-with-sfn/tests/unit/test_emr_serverless_with_sfn_stack.py -------------------------------------------------------------------------------- /cloudformation/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/README.md -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/README.md -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/emr_serverless_cloudwatch_dashboard.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/emr_serverless_cloudwatch_dashboard.yaml -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/application_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/application_metrics.png -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/demo/initial_state.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/demo/initial_state.png -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/demo/pi-default.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/demo/pi-default.png -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/demo/pi-drivers-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/demo/pi-drivers-chart.png -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/demo/pi-drivers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/demo/pi-drivers.png -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/demo/pi-more-executors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/demo/pi-more-executors.png -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/driver_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/driver_metrics.png -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/executor_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/executor_metrics.png -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/job-metrics-breakdown.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/job-metrics-breakdown.png -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/job-runs-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/job-runs-overview.png -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/job_worker_metrics_Driver.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/job_worker_metrics_Driver.jpg -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/job_worker_metrics_Executor.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/job_worker_metrics_Executor.jpeg -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/pre_initialized_capacity_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/pre_initialized_capacity_metrics.png -------------------------------------------------------------------------------- /cloudformation/emr-serverless-cloudwatch-dashboard/images/snapshot_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr-serverless-cloudwatch-dashboard/images/snapshot_metrics.png -------------------------------------------------------------------------------- /cloudformation/emr_serverless_full_deployment.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr_serverless_full_deployment.yaml -------------------------------------------------------------------------------- /cloudformation/emr_serverless_spark_app.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/emr_serverless_spark_app.yaml -------------------------------------------------------------------------------- /cloudformation/kds_emr_serverless.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/kds_emr_serverless.yaml -------------------------------------------------------------------------------- /cloudformation/lake-formation-and-apache-iceberg/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/lake-formation-and-apache-iceberg/README.md -------------------------------------------------------------------------------- /cloudformation/lake-formation-and-apache-iceberg/athena_create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/lake-formation-and-apache-iceberg/athena_create.png -------------------------------------------------------------------------------- /cloudformation/lake-formation-and-apache-iceberg/lake_formation_iceberg.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/lake-formation-and-apache-iceberg/lake_formation_iceberg.yaml -------------------------------------------------------------------------------- /cloudformation/lake-formation-and-apache-iceberg/show_table.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/lake-formation-and-apache-iceberg/show_table.py -------------------------------------------------------------------------------- /cloudformation/mwaa_emr_serverless.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/cloudformation/mwaa_emr_serverless.yaml -------------------------------------------------------------------------------- /examples/hive/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/hive/README.md -------------------------------------------------------------------------------- /examples/hive/create_table.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/hive/create_table.sql -------------------------------------------------------------------------------- /examples/hive/extreme_weather.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/hive/extreme_weather.sql -------------------------------------------------------------------------------- /examples/java-api/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/java-api/README.md -------------------------------------------------------------------------------- /examples/java-api/myapp/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/java-api/myapp/.gitignore -------------------------------------------------------------------------------- /examples/java-api/myapp/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/java-api/myapp/pom.xml -------------------------------------------------------------------------------- /examples/java-api/myapp/src/main/java/com/example/myapp/App.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/java-api/myapp/src/main/java/com/example/myapp/App.java -------------------------------------------------------------------------------- /examples/java-api/myapp/src/main/java/com/example/myapp/EMRServerlessService.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/java-api/myapp/src/main/java/com/example/myapp/EMRServerlessService.java -------------------------------------------------------------------------------- /examples/java-api/myapp/src/test/java/com/example/myapp/AppTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/java-api/myapp/src/test/java/com/example/myapp/AppTest.java -------------------------------------------------------------------------------- /examples/java/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/java/README.md -------------------------------------------------------------------------------- /examples/java/hello-world/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | -------------------------------------------------------------------------------- /examples/java/hello-world/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/java/hello-world/pom.xml -------------------------------------------------------------------------------- /examples/java/hello-world/src/main/java/HelloWorld.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/java/hello-world/src/main/java/HelloWorld.java -------------------------------------------------------------------------------- /examples/pyspark/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/README.md -------------------------------------------------------------------------------- /examples/pyspark/custom-images/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/custom-images/Dockerfile -------------------------------------------------------------------------------- /examples/pyspark/custom-images/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/custom-images/README.md -------------------------------------------------------------------------------- /examples/pyspark/custom-images/noaa_slugplot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/custom-images/noaa_slugplot.py -------------------------------------------------------------------------------- /examples/pyspark/custom_python_version/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/custom_python_version/Dockerfile -------------------------------------------------------------------------------- /examples/pyspark/custom_python_version/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/custom_python_version/README.md -------------------------------------------------------------------------------- /examples/pyspark/custom_python_version/python_3.10.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/custom_python_version/python_3.10.py -------------------------------------------------------------------------------- /examples/pyspark/delta-lake/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/delta-lake/Dockerfile -------------------------------------------------------------------------------- /examples/pyspark/delta-lake/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/delta-lake/README.md -------------------------------------------------------------------------------- /examples/pyspark/delta-lake/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/delta-lake/main.py -------------------------------------------------------------------------------- /examples/pyspark/delta-lake/poetry.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/delta-lake/poetry.lock -------------------------------------------------------------------------------- /examples/pyspark/delta-lake/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/delta-lake/pyproject.toml -------------------------------------------------------------------------------- /examples/pyspark/dependencies/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/dependencies/Dockerfile -------------------------------------------------------------------------------- /examples/pyspark/dependencies/Dockerfile.al2023: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/dependencies/Dockerfile.al2023 -------------------------------------------------------------------------------- /examples/pyspark/dependencies/Dockerfile.jars: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/dependencies/Dockerfile.jars -------------------------------------------------------------------------------- /examples/pyspark/dependencies/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/dependencies/README.md -------------------------------------------------------------------------------- /examples/pyspark/dependencies/ge_profile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/dependencies/ge_profile.py -------------------------------------------------------------------------------- /examples/pyspark/dependencies/pg_query.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/dependencies/pg_query.py -------------------------------------------------------------------------------- /examples/pyspark/dependencies/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/dependencies/pom.xml -------------------------------------------------------------------------------- /examples/pyspark/extreme_weather.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/extreme_weather.py -------------------------------------------------------------------------------- /examples/pyspark/genomic/.gitignore: -------------------------------------------------------------------------------- 1 | dependencies/ 2 | jars/ 3 | -------------------------------------------------------------------------------- /examples/pyspark/genomic/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/genomic/Dockerfile -------------------------------------------------------------------------------- /examples/pyspark/genomic/Dockerfile.jars: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/genomic/Dockerfile.jars -------------------------------------------------------------------------------- /examples/pyspark/genomic/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/genomic/README.md -------------------------------------------------------------------------------- /examples/pyspark/genomic/glow_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/genomic/glow_demo.py -------------------------------------------------------------------------------- /examples/pyspark/streaming/Kinesis_data_stream/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/streaming/Kinesis_data_stream/README.md -------------------------------------------------------------------------------- /examples/pyspark/streaming/Kinesis_data_stream/transactions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/streaming/Kinesis_data_stream/transactions.py -------------------------------------------------------------------------------- /examples/pyspark/streaming/README.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /examples/pyspark/streaming/images/README.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /examples/pyspark/streaming/images/architecture-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/streaming/images/architecture-image.png -------------------------------------------------------------------------------- /examples/pyspark/streaming/images/emrs-properties-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/streaming/images/emrs-properties-image.png -------------------------------------------------------------------------------- /examples/pyspark/streaming/images/kdg-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/streaming/images/kdg-image.png -------------------------------------------------------------------------------- /examples/pyspark/streaming/images/results-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/streaming/images/results-image.png -------------------------------------------------------------------------------- /examples/pyspark/streaming/images/viewlogs-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/pyspark/streaming/images/viewlogs-image.png -------------------------------------------------------------------------------- /examples/python-api/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/python-api/.gitignore -------------------------------------------------------------------------------- /examples/python-api/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/python-api/README.md -------------------------------------------------------------------------------- /examples/python-api/emr_serverless.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/examples/python-api/emr_serverless.py -------------------------------------------------------------------------------- /utilities/local-dev/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/local-dev/Dockerfile -------------------------------------------------------------------------------- /utilities/local-dev/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/local-dev/README.md -------------------------------------------------------------------------------- /utilities/local-dev/jupyter-server.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/local-dev/jupyter-server.png -------------------------------------------------------------------------------- /utilities/spark-ui/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/spark-ui/Dockerfile -------------------------------------------------------------------------------- /utilities/spark-ui/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/spark-ui/README.md -------------------------------------------------------------------------------- /utilities/tez-ui/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/tez-ui/Dockerfile -------------------------------------------------------------------------------- /utilities/tez-ui/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/tez-ui/README.md -------------------------------------------------------------------------------- /utilities/tez-ui/entrypoint.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/tez-ui/entrypoint.sh -------------------------------------------------------------------------------- /utilities/tez-ui/event-log-sync.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/tez-ui/event-log-sync.sh -------------------------------------------------------------------------------- /utilities/tez-ui/hadoop-layout.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/tez-ui/hadoop-layout.sh -------------------------------------------------------------------------------- /utilities/tez-ui/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/tez-ui/pom.xml -------------------------------------------------------------------------------- /utilities/tez-ui/tez-ui.patch: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/tez-ui/tez-ui.patch -------------------------------------------------------------------------------- /utilities/tez-ui/yarn-site.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/emr-serverless-samples/HEAD/utilities/tez-ui/yarn-site.xml --------------------------------------------------------------------------------