├── .gitignore ├── .idea ├── .gitignore ├── aws-airflow-demo.iml ├── aws.xml ├── codeStyles │ └── codeStyleConfig.xml ├── inspectionProfiles │ ├── Project_Default.xml │ └── profiles_settings.xml ├── misc.xml ├── modules.xml └── vcs.xml ├── LICENSE ├── README.md ├── airflow_variables ├── admin_variables.json └── dag_run_config.json ├── dags ├── airflow_v1 │ ├── bakery_sales.py │ ├── clean_mwaa_db.py │ ├── get_airflow_cfg.py │ ├── get_env_vars.py │ ├── get_py_pkgs.py │ ├── inject_bash_command.py │ ├── multiple_steps.py │ ├── spark_pi_example.py │ └── throw_exception.py └── airflow_v2 │ ├── emr_demo__bakery_sales_v2.py │ ├── emr_demo__multiple_steps_v2.py │ ├── emr_demo__spark_pi_example_v2.py │ ├── lineage_emission.py │ ├── utilities_demo__get_airflow_cfg_v2.py │ ├── utilities_demo__get_env_vars_v2.py │ ├── utilities_demo__get_py_pkgs_v2.py │ ├── utilities_demo__inject_bash_command_v2.py │ └── utilities_demo__throw_exception_v2.py ├── diagram ├── Airflow_UI.png └── Architecture.png ├── emr_steps └── emr_steps.json ├── helper_scripts ├── clone_repo_upload_to_s3.py ├── dags_to_s3_from_github.py └── trigger_dag.py ├── iam_policy └── airflow_emr_policy.json ├── job_flow_overrides └── job_flow_overrides.json ├── requirements.txt ├── requirements_mwaa.txt └── unused ├── community_dags ├── zdebug_airflow_dag.py └── zdebug_dag.py └── sns_example_v2.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/.gitignore -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/.idea/.gitignore -------------------------------------------------------------------------------- /.idea/aws-airflow-demo.iml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/.idea/aws-airflow-demo.iml -------------------------------------------------------------------------------- /.idea/aws.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/.idea/aws.xml -------------------------------------------------------------------------------- /.idea/codeStyles/codeStyleConfig.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/.idea/codeStyles/codeStyleConfig.xml -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/.idea/inspectionProfiles/Project_Default.xml -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/.idea/inspectionProfiles/profiles_settings.xml -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/.idea/misc.xml -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/.idea/modules.xml -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/.idea/vcs.xml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/README.md -------------------------------------------------------------------------------- /airflow_variables/admin_variables.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/airflow_variables/admin_variables.json -------------------------------------------------------------------------------- /airflow_variables/dag_run_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/airflow_variables/dag_run_config.json -------------------------------------------------------------------------------- /dags/airflow_v1/bakery_sales.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v1/bakery_sales.py -------------------------------------------------------------------------------- /dags/airflow_v1/clean_mwaa_db.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v1/clean_mwaa_db.py -------------------------------------------------------------------------------- /dags/airflow_v1/get_airflow_cfg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v1/get_airflow_cfg.py -------------------------------------------------------------------------------- /dags/airflow_v1/get_env_vars.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v1/get_env_vars.py -------------------------------------------------------------------------------- /dags/airflow_v1/get_py_pkgs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v1/get_py_pkgs.py -------------------------------------------------------------------------------- /dags/airflow_v1/inject_bash_command.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v1/inject_bash_command.py -------------------------------------------------------------------------------- /dags/airflow_v1/multiple_steps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v1/multiple_steps.py -------------------------------------------------------------------------------- /dags/airflow_v1/spark_pi_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v1/spark_pi_example.py -------------------------------------------------------------------------------- /dags/airflow_v1/throw_exception.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v1/throw_exception.py -------------------------------------------------------------------------------- /dags/airflow_v2/emr_demo__bakery_sales_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v2/emr_demo__bakery_sales_v2.py -------------------------------------------------------------------------------- /dags/airflow_v2/emr_demo__multiple_steps_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v2/emr_demo__multiple_steps_v2.py -------------------------------------------------------------------------------- /dags/airflow_v2/emr_demo__spark_pi_example_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v2/emr_demo__spark_pi_example_v2.py -------------------------------------------------------------------------------- /dags/airflow_v2/lineage_emission.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v2/lineage_emission.py -------------------------------------------------------------------------------- /dags/airflow_v2/utilities_demo__get_airflow_cfg_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v2/utilities_demo__get_airflow_cfg_v2.py -------------------------------------------------------------------------------- /dags/airflow_v2/utilities_demo__get_env_vars_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v2/utilities_demo__get_env_vars_v2.py -------------------------------------------------------------------------------- /dags/airflow_v2/utilities_demo__get_py_pkgs_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v2/utilities_demo__get_py_pkgs_v2.py -------------------------------------------------------------------------------- /dags/airflow_v2/utilities_demo__inject_bash_command_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v2/utilities_demo__inject_bash_command_v2.py -------------------------------------------------------------------------------- /dags/airflow_v2/utilities_demo__throw_exception_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/dags/airflow_v2/utilities_demo__throw_exception_v2.py -------------------------------------------------------------------------------- /diagram/Airflow_UI.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/diagram/Airflow_UI.png -------------------------------------------------------------------------------- /diagram/Architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/diagram/Architecture.png -------------------------------------------------------------------------------- /emr_steps/emr_steps.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/emr_steps/emr_steps.json -------------------------------------------------------------------------------- /helper_scripts/clone_repo_upload_to_s3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/helper_scripts/clone_repo_upload_to_s3.py -------------------------------------------------------------------------------- /helper_scripts/dags_to_s3_from_github.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/helper_scripts/dags_to_s3_from_github.py -------------------------------------------------------------------------------- /helper_scripts/trigger_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/helper_scripts/trigger_dag.py -------------------------------------------------------------------------------- /iam_policy/airflow_emr_policy.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/iam_policy/airflow_emr_policy.json -------------------------------------------------------------------------------- /job_flow_overrides/job_flow_overrides.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/job_flow_overrides/job_flow_overrides.json -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/requirements.txt -------------------------------------------------------------------------------- /requirements_mwaa.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/requirements_mwaa.txt -------------------------------------------------------------------------------- /unused/community_dags/zdebug_airflow_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/unused/community_dags/zdebug_airflow_dag.py -------------------------------------------------------------------------------- /unused/community_dags/zdebug_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/unused/community_dags/zdebug_dag.py -------------------------------------------------------------------------------- /unused/sns_example_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/garystafford/aws-airflow-demo/HEAD/unused/sns_example_v2.py --------------------------------------------------------------------------------