├── .github └── workflows │ ├── ci_cd.yml │ └── destroy-infra.yml ├── .gitignore ├── Infra ├── locals.tf ├── main.tf ├── modules │ ├── eventbridge │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── glue_catalog_database │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── glue_catalog_table │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── glue_classifier │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── glue_crawler │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── glue_iam │ │ ├── main.tf │ │ └── output.tf │ ├── glue_job │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── glue_trigger │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── lambda │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ ├── request_layer │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf │ └── s3 │ │ ├── main.tf │ │ ├── output.tf │ │ └── variables.tf └── providers.tf ├── Makefile ├── README.md ├── env └── base.env ├── etl ├── extract │ ├── System │ │ ├── LocalLocation.py │ │ ├── location.py │ │ └── workspace.py │ ├── __init__.py │ ├── extract_data.py │ └── system_config.yml ├── glue_etl_job │ ├── __init__.py │ └── transform_data.py └── load │ ├── __init__.py │ └── load_data.py ├── images └── architecture.png ├── requirements.txt └── terraform.tfstate /.github/workflows/ci_cd.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/.github/workflows/ci_cd.yml -------------------------------------------------------------------------------- /.github/workflows/destroy-infra.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/.github/workflows/destroy-infra.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/.gitignore -------------------------------------------------------------------------------- /Infra/locals.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/locals.tf -------------------------------------------------------------------------------- /Infra/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/main.tf -------------------------------------------------------------------------------- /Infra/modules/eventbridge/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/eventbridge/main.tf -------------------------------------------------------------------------------- /Infra/modules/eventbridge/output.tf: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Infra/modules/eventbridge/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/eventbridge/variables.tf -------------------------------------------------------------------------------- /Infra/modules/glue_catalog_database/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_catalog_database/main.tf -------------------------------------------------------------------------------- /Infra/modules/glue_catalog_database/output.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_catalog_database/output.tf -------------------------------------------------------------------------------- /Infra/modules/glue_catalog_database/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_catalog_database/variables.tf -------------------------------------------------------------------------------- /Infra/modules/glue_catalog_table/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_catalog_table/main.tf -------------------------------------------------------------------------------- /Infra/modules/glue_catalog_table/output.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_catalog_table/output.tf -------------------------------------------------------------------------------- /Infra/modules/glue_catalog_table/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_catalog_table/variables.tf -------------------------------------------------------------------------------- /Infra/modules/glue_classifier/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_classifier/main.tf -------------------------------------------------------------------------------- /Infra/modules/glue_classifier/output.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_classifier/output.tf -------------------------------------------------------------------------------- /Infra/modules/glue_classifier/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_classifier/variables.tf -------------------------------------------------------------------------------- /Infra/modules/glue_crawler/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_crawler/main.tf -------------------------------------------------------------------------------- /Infra/modules/glue_crawler/output.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_crawler/output.tf -------------------------------------------------------------------------------- /Infra/modules/glue_crawler/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_crawler/variables.tf -------------------------------------------------------------------------------- /Infra/modules/glue_iam/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_iam/main.tf -------------------------------------------------------------------------------- /Infra/modules/glue_iam/output.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_iam/output.tf -------------------------------------------------------------------------------- /Infra/modules/glue_job/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_job/main.tf -------------------------------------------------------------------------------- /Infra/modules/glue_job/output.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_job/output.tf -------------------------------------------------------------------------------- /Infra/modules/glue_job/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_job/variables.tf -------------------------------------------------------------------------------- /Infra/modules/glue_trigger/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_trigger/main.tf -------------------------------------------------------------------------------- /Infra/modules/glue_trigger/output.tf: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Infra/modules/glue_trigger/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/glue_trigger/variables.tf -------------------------------------------------------------------------------- /Infra/modules/lambda/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/lambda/main.tf -------------------------------------------------------------------------------- /Infra/modules/lambda/output.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/lambda/output.tf -------------------------------------------------------------------------------- /Infra/modules/lambda/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/lambda/variables.tf -------------------------------------------------------------------------------- /Infra/modules/request_layer/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/request_layer/main.tf -------------------------------------------------------------------------------- /Infra/modules/request_layer/output.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/request_layer/output.tf -------------------------------------------------------------------------------- /Infra/modules/request_layer/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/request_layer/variables.tf -------------------------------------------------------------------------------- /Infra/modules/s3/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/s3/main.tf -------------------------------------------------------------------------------- /Infra/modules/s3/output.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/s3/output.tf -------------------------------------------------------------------------------- /Infra/modules/s3/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Infra/modules/s3/variables.tf -------------------------------------------------------------------------------- /Infra/providers.tf: -------------------------------------------------------------------------------- 1 | 2 | terraform { 3 | backend "s3" {} 4 | } -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/README.md -------------------------------------------------------------------------------- /env/base.env: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/env/base.env -------------------------------------------------------------------------------- /etl/extract/System/LocalLocation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/etl/extract/System/LocalLocation.py -------------------------------------------------------------------------------- /etl/extract/System/location.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/etl/extract/System/location.py -------------------------------------------------------------------------------- /etl/extract/System/workspace.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/etl/extract/System/workspace.py -------------------------------------------------------------------------------- /etl/extract/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /etl/extract/extract_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/etl/extract/extract_data.py -------------------------------------------------------------------------------- /etl/extract/system_config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/etl/extract/system_config.yml -------------------------------------------------------------------------------- /etl/glue_etl_job/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /etl/glue_etl_job/transform_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/etl/glue_etl_job/transform_data.py -------------------------------------------------------------------------------- /etl/load/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /etl/load/load_data.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/images/architecture.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | PyYAML 3 | -------------------------------------------------------------------------------- /terraform.tfstate: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/g-lorena/aws_etl_pipeline/HEAD/terraform.tfstate --------------------------------------------------------------------------------