├── .github
└── workflows
│ ├── ci_cd.yml
│ └── destroy-infra.yml
├── .gitignore
├── Infra
├── locals.tf
├── main.tf
├── modules
│ ├── eventbridge
│ │ ├── main.tf
│ │ ├── output.tf
│ │ └── variables.tf
│ ├── glue_catalog_database
│ │ ├── main.tf
│ │ ├── output.tf
│ │ └── variables.tf
│ ├── glue_catalog_table
│ │ ├── main.tf
│ │ ├── output.tf
│ │ └── variables.tf
│ ├── glue_classifier
│ │ ├── main.tf
│ │ ├── output.tf
│ │ └── variables.tf
│ ├── glue_crawler
│ │ ├── main.tf
│ │ ├── output.tf
│ │ └── variables.tf
│ ├── glue_iam
│ │ ├── main.tf
│ │ └── output.tf
│ ├── glue_job
│ │ ├── main.tf
│ │ ├── output.tf
│ │ └── variables.tf
│ ├── glue_trigger
│ │ ├── main.tf
│ │ ├── output.tf
│ │ └── variables.tf
│ ├── lambda
│ │ ├── main.tf
│ │ ├── output.tf
│ │ └── variables.tf
│ ├── request_layer
│ │ ├── main.tf
│ │ ├── output.tf
│ │ └── variables.tf
│ └── s3
│ │ ├── main.tf
│ │ ├── output.tf
│ │ └── variables.tf
└── providers.tf
├── Makefile
├── README.md
├── env
└── base.env
├── etl
├── extract
│ ├── System
│ │ ├── LocalLocation.py
│ │ ├── location.py
│ │ └── workspace.py
│ ├── __init__.py
│ ├── extract_data.py
│ └── system_config.yml
├── glue_etl_job
│ ├── __init__.py
│ └── transform_data.py
└── load
│ ├── __init__.py
│ └── load_data.py
├── images
└── architecture.png
├── requirements.txt
└── terraform.tfstate
/.github/workflows/ci_cd.yml:
--------------------------------------------------------------------------------
1 | name: "Terraform action"
2 | on:
3 | push:
4 | branches:
5 | - main
6 | pull_request:
7 | permissions:
8 | id-token: write # This is required for aws oidc connection
9 | contents: read # This is required for actions/checkout
10 | pull-requests: write # This is required for gh bot to comment PR
11 | env:
12 | TF_LOG: INFO
13 | AWS_REGION: ${{ secrets.AWS_REGION }}
14 | jobs:
15 | deploy:
16 | runs-on: ubuntu-latest
17 | defaults:
18 | run:
19 | shell: bash
20 | working-directory: .
21 | steps:
22 | - name: Git checkout
23 | uses: actions/checkout@v3
24 |
25 | - name: Configure AWS credentials from AWS account
26 | uses: aws-actions/configure-aws-credentials@v1
27 | with:
28 | role-to-assume: ${{ secrets.AWS_ROLE }}
29 | aws-region: ${{ secrets.AWS_REGION }}
30 | role-session-name: GitHub-OIDC-TERRAFORM
31 |
32 | - name: Setup Terraform
33 | uses: hashicorp/setup-terraform@v2
34 | with:
35 | terraform_version: 1.7.5
36 |
37 | - name: Terraform fmt
38 | id: fmt
39 | run: terraform fmt -check
40 | continue-on-error: true
41 |
42 | - name: Terraform Init
43 | id: init
44 | env:
45 | AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
46 | AWS_BUCKET_KEY_NAME: ${{ secrets.AWS_BUCKET_KEY_NAME }}
47 | run: make terraform-init
48 |
49 | - name: Terraform Validate
50 | id: validate
51 | run: make terraform-validate
52 |
53 | - name: Terraform Plan
54 | id: plan
55 | run: make terraform-plan
56 | if: github.event_name == 'pull_request'
57 | continue-on-error: true
58 |
59 | - uses: actions/github-script@v6
60 | if: github.event_name == 'pull_request'
61 | env:
62 | PLAN: "terraform\n${{ steps.plan.outputs.stdout }}"
63 | with:
64 | github-token: ${{ secrets.GITHUB_TOKEN }}
65 | script: |
66 | const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\`
67 | #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\`
68 | #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\`
69 | Validation Output
70 |
71 | \`\`\`\n
72 | ${{ steps.validate.outputs.stdout }}
73 | \`\`\`
74 |
75 | Show Plan
80 |
81 | \`\`\`\n
82 | ${process.env.PLAN}
83 | \`\`\`
84 |
85 |