├── .gitignore ├── README.md ├── dbt ├── .gitignore ├── README.md ├── dbt_project.yml ├── macros │ ├── get_comunidad.sql │ └── get_velmedia_description.sql ├── models │ ├── core │ │ ├── dim_monthly_climatic_data.sql │ │ ├── dim_station_count_by_region.sql │ │ ├── fact_weather.sql │ │ └── schema.yml │ └── staging │ │ ├── schema.yml │ │ ├── stg_climatic_values.sql │ │ └── stg_weather_stations.sql ├── package-lock.yml └── packages.yml ├── dlt ├── README.md ├── extract_climatic_values.py ├── extract_weather_stations.py ├── load_data_to_bq.py ├── requirements.txt └── schemas │ ├── export │ ├── climatic_values_to_gcs.schema.yaml │ └── weather_stations_to_gcs.schema.yaml │ └── import │ ├── climatic_values_to_gcs.schema.yaml │ └── weather_stations_to_gcs.schema.yaml ├── images ├── aemetelt.gif ├── bq-datasets-filled.PNG ├── bucket-creation.PNG ├── clustered.PNG ├── datasets-creation.PNG ├── dbt-connection.PNG ├── dbt-sa.PNG ├── dbt_build.PNG ├── dbt_model.PNG ├── gcs-parquet.PNG ├── kestra-flows.PNG ├── kestra-kv-store.PNG ├── kestra-master-input.PNG ├── kestra-master-ok.PNG ├── kestra-topology.PNG ├── looker.PNG ├── partitionedandclustered.PNG ├── terraform-service-account-permissions.PNG └── terraform-service-account.PNG ├── kestra ├── README.md ├── docker-compose.yml └── flows │ ├── extract_weather_data.yaml │ ├── load_gcp_kv.yaml │ ├── load_weather_data.yaml │ ├── master_flow.yaml │ └── transform_dbt.yaml └── terraform ├── README.md ├── main.tf └── variables.tf /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/README.md -------------------------------------------------------------------------------- /dbt/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/.gitignore -------------------------------------------------------------------------------- /dbt/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/README.md -------------------------------------------------------------------------------- /dbt/dbt_project.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/dbt_project.yml -------------------------------------------------------------------------------- /dbt/macros/get_comunidad.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/macros/get_comunidad.sql -------------------------------------------------------------------------------- /dbt/macros/get_velmedia_description.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/macros/get_velmedia_description.sql -------------------------------------------------------------------------------- /dbt/models/core/dim_monthly_climatic_data.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/models/core/dim_monthly_climatic_data.sql -------------------------------------------------------------------------------- /dbt/models/core/dim_station_count_by_region.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/models/core/dim_station_count_by_region.sql -------------------------------------------------------------------------------- /dbt/models/core/fact_weather.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/models/core/fact_weather.sql -------------------------------------------------------------------------------- /dbt/models/core/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/models/core/schema.yml -------------------------------------------------------------------------------- /dbt/models/staging/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/models/staging/schema.yml -------------------------------------------------------------------------------- /dbt/models/staging/stg_climatic_values.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/models/staging/stg_climatic_values.sql -------------------------------------------------------------------------------- /dbt/models/staging/stg_weather_stations.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/models/staging/stg_weather_stations.sql -------------------------------------------------------------------------------- /dbt/package-lock.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/package-lock.yml -------------------------------------------------------------------------------- /dbt/packages.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dbt/packages.yml -------------------------------------------------------------------------------- /dlt/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dlt/README.md -------------------------------------------------------------------------------- /dlt/extract_climatic_values.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dlt/extract_climatic_values.py -------------------------------------------------------------------------------- /dlt/extract_weather_stations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dlt/extract_weather_stations.py -------------------------------------------------------------------------------- /dlt/load_data_to_bq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dlt/load_data_to_bq.py -------------------------------------------------------------------------------- /dlt/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dlt/requirements.txt -------------------------------------------------------------------------------- /dlt/schemas/export/climatic_values_to_gcs.schema.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dlt/schemas/export/climatic_values_to_gcs.schema.yaml -------------------------------------------------------------------------------- /dlt/schemas/export/weather_stations_to_gcs.schema.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dlt/schemas/export/weather_stations_to_gcs.schema.yaml -------------------------------------------------------------------------------- /dlt/schemas/import/climatic_values_to_gcs.schema.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dlt/schemas/import/climatic_values_to_gcs.schema.yaml -------------------------------------------------------------------------------- /dlt/schemas/import/weather_stations_to_gcs.schema.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/dlt/schemas/import/weather_stations_to_gcs.schema.yaml -------------------------------------------------------------------------------- /images/aemetelt.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/aemetelt.gif -------------------------------------------------------------------------------- /images/bq-datasets-filled.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/bq-datasets-filled.PNG -------------------------------------------------------------------------------- /images/bucket-creation.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/bucket-creation.PNG -------------------------------------------------------------------------------- /images/clustered.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/clustered.PNG -------------------------------------------------------------------------------- /images/datasets-creation.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/datasets-creation.PNG -------------------------------------------------------------------------------- /images/dbt-connection.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/dbt-connection.PNG -------------------------------------------------------------------------------- /images/dbt-sa.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/dbt-sa.PNG -------------------------------------------------------------------------------- /images/dbt_build.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/dbt_build.PNG -------------------------------------------------------------------------------- /images/dbt_model.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/dbt_model.PNG -------------------------------------------------------------------------------- /images/gcs-parquet.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/gcs-parquet.PNG -------------------------------------------------------------------------------- /images/kestra-flows.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/kestra-flows.PNG -------------------------------------------------------------------------------- /images/kestra-kv-store.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/kestra-kv-store.PNG -------------------------------------------------------------------------------- /images/kestra-master-input.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/kestra-master-input.PNG -------------------------------------------------------------------------------- /images/kestra-master-ok.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/kestra-master-ok.PNG -------------------------------------------------------------------------------- /images/kestra-topology.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/kestra-topology.PNG -------------------------------------------------------------------------------- /images/looker.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/looker.PNG -------------------------------------------------------------------------------- /images/partitionedandclustered.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/partitionedandclustered.PNG -------------------------------------------------------------------------------- /images/terraform-service-account-permissions.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/terraform-service-account-permissions.PNG -------------------------------------------------------------------------------- /images/terraform-service-account.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/images/terraform-service-account.PNG -------------------------------------------------------------------------------- /kestra/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/kestra/README.md -------------------------------------------------------------------------------- /kestra/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/kestra/docker-compose.yml -------------------------------------------------------------------------------- /kestra/flows/extract_weather_data.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/kestra/flows/extract_weather_data.yaml -------------------------------------------------------------------------------- /kestra/flows/load_gcp_kv.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/kestra/flows/load_gcp_kv.yaml -------------------------------------------------------------------------------- /kestra/flows/load_weather_data.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/kestra/flows/load_weather_data.yaml -------------------------------------------------------------------------------- /kestra/flows/master_flow.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/kestra/flows/master_flow.yaml -------------------------------------------------------------------------------- /kestra/flows/transform_dbt.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/kestra/flows/transform_dbt.yaml -------------------------------------------------------------------------------- /terraform/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/terraform/README.md -------------------------------------------------------------------------------- /terraform/main.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/terraform/main.tf -------------------------------------------------------------------------------- /terraform/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsrl/aemet-elt/HEAD/terraform/variables.tf --------------------------------------------------------------------------------