├── .editorconfig ├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ └── scraper-test.yaml ├── .gitignore ├── Dockerfile ├── Makefile ├── Pipfile ├── Pipfile.lock ├── README.md ├── airflow ├── dags │ └── main_dag.py └── tasks │ ├── __init__.py │ ├── configuration.env.example │ ├── dbt_transform │ ├── dbt_project.yml │ ├── generate_dbt_profile.py │ └── models │ │ ├── avg_audio_signature_headphone.sql │ │ ├── avg_audio_signature_iem.sql │ │ ├── avg_driver_rating_headphone.sql │ │ ├── highest_rank_company_headphone.sql │ │ ├── highest_rank_company_iem.sql │ │ ├── highest_value_company_headphone.sql │ │ ├── highest_value_company_iem.sql │ │ ├── schema.yml │ │ └── staging │ │ ├── schema.yml │ │ ├── stg_companynames.sql │ │ └── stg_maprankvalues.sql │ ├── rds_load │ ├── queries │ │ ├── create_tables.sql │ │ ├── load_main_tables.sql │ │ └── load_temp_tables.sql │ └── upload_to_rds.py │ ├── redshift_load │ ├── queries │ │ ├── create_tables.sql │ │ ├── load_main_tables.sql │ │ └── load_temp_tables.sql │ └── upload_to_redshift.py │ ├── scraper_extract │ ├── __init__.py │ ├── models.py │ ├── scraper.py │ └── test_scraper.py │ ├── upload_to_s3.py │ ├── utilities.py │ └── validate_sanitize_bronze.py ├── docker-compose.yaml ├── generate_config.sh ├── images ├── architecture.jpeg └── metabase_dashboard.jpeg ├── pyproject.toml └── terraform ├── budget.tf ├── output.tf ├── provider.tf ├── rds.tf ├── redshift.tf ├── s3.tf └── variable.tf /.editorconfig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/.editorconfig -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.sql linguist-detectable=true 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/.github/dependabot.yml -------------------------------------------------------------------------------- /.github/workflows/scraper-test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/.github/workflows/scraper-test.yaml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/.gitignore -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/Dockerfile -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/Makefile -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/Pipfile -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/Pipfile.lock -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/README.md -------------------------------------------------------------------------------- /airflow/dags/main_dag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/dags/main_dag.py -------------------------------------------------------------------------------- /airflow/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /airflow/tasks/configuration.env.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/configuration.env.example -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/dbt_project.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/dbt_project.yml -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/generate_dbt_profile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/generate_dbt_profile.py -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/models/avg_audio_signature_headphone.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/models/avg_audio_signature_headphone.sql -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/models/avg_audio_signature_iem.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/models/avg_audio_signature_iem.sql -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/models/avg_driver_rating_headphone.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/models/avg_driver_rating_headphone.sql -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/models/highest_rank_company_headphone.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/models/highest_rank_company_headphone.sql -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/models/highest_rank_company_iem.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/models/highest_rank_company_iem.sql -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/models/highest_value_company_headphone.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/models/highest_value_company_headphone.sql -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/models/highest_value_company_iem.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/models/highest_value_company_iem.sql -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/models/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/models/schema.yml -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/models/staging/schema.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/models/staging/schema.yml -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/models/staging/stg_companynames.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/models/staging/stg_companynames.sql -------------------------------------------------------------------------------- /airflow/tasks/dbt_transform/models/staging/stg_maprankvalues.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/dbt_transform/models/staging/stg_maprankvalues.sql -------------------------------------------------------------------------------- /airflow/tasks/rds_load/queries/create_tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/rds_load/queries/create_tables.sql -------------------------------------------------------------------------------- /airflow/tasks/rds_load/queries/load_main_tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/rds_load/queries/load_main_tables.sql -------------------------------------------------------------------------------- /airflow/tasks/rds_load/queries/load_temp_tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/rds_load/queries/load_temp_tables.sql -------------------------------------------------------------------------------- /airflow/tasks/rds_load/upload_to_rds.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/rds_load/upload_to_rds.py -------------------------------------------------------------------------------- /airflow/tasks/redshift_load/queries/create_tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/redshift_load/queries/create_tables.sql -------------------------------------------------------------------------------- /airflow/tasks/redshift_load/queries/load_main_tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/redshift_load/queries/load_main_tables.sql -------------------------------------------------------------------------------- /airflow/tasks/redshift_load/queries/load_temp_tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/redshift_load/queries/load_temp_tables.sql -------------------------------------------------------------------------------- /airflow/tasks/redshift_load/upload_to_redshift.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/redshift_load/upload_to_redshift.py -------------------------------------------------------------------------------- /airflow/tasks/scraper_extract/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /airflow/tasks/scraper_extract/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/scraper_extract/models.py -------------------------------------------------------------------------------- /airflow/tasks/scraper_extract/scraper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/scraper_extract/scraper.py -------------------------------------------------------------------------------- /airflow/tasks/scraper_extract/test_scraper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/scraper_extract/test_scraper.py -------------------------------------------------------------------------------- /airflow/tasks/upload_to_s3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/upload_to_s3.py -------------------------------------------------------------------------------- /airflow/tasks/utilities.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/utilities.py -------------------------------------------------------------------------------- /airflow/tasks/validate_sanitize_bronze.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/airflow/tasks/validate_sanitize_bronze.py -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/docker-compose.yaml -------------------------------------------------------------------------------- /generate_config.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/generate_config.sh -------------------------------------------------------------------------------- /images/architecture.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/images/architecture.jpeg -------------------------------------------------------------------------------- /images/metabase_dashboard.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/images/metabase_dashboard.jpeg -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 100 3 | -------------------------------------------------------------------------------- /terraform/budget.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/terraform/budget.tf -------------------------------------------------------------------------------- /terraform/output.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/terraform/output.tf -------------------------------------------------------------------------------- /terraform/provider.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/terraform/provider.tf -------------------------------------------------------------------------------- /terraform/rds.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/terraform/rds.tf -------------------------------------------------------------------------------- /terraform/redshift.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/terraform/redshift.tf -------------------------------------------------------------------------------- /terraform/s3.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/terraform/s3.tf -------------------------------------------------------------------------------- /terraform/variable.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ris-tlp/audiophile-e2e-pipeline/HEAD/terraform/variable.tf --------------------------------------------------------------------------------