├── 1. Authentication ├── 1.access_adls_using_access_keys.py ├── 2.access_adls_using_sas_token.py ├── 3.access_adls_using_service_principal.py ├── 4.access_adls_using_cluster_scoped_credentials.py ├── 5.explore_dbutils_secrets_utility.py ├── 6.explore_dbfs_root.py ├── 7.mount_adls_using_service_principal.py └── 8.mount_adls_containers_for_project.py ├── 2. Includes_configs_and_commonfunctions ├── common_functions.py └── configuration.py ├── 3. Data Ingestion ├── 0.ingest_all_files.py ├── 1.ingest_circuits_file.py ├── 2.ingest_races_file.py ├── 3.ingest_constructors_file.py ├── 4.ingest_drivers_file.py ├── 5.ingest_results_file.py ├── 6.ingest_pit_stops_file.py ├── 7.ingest_lap_times_file.py ├── 8.ingest_qualifying_file.py └── 9.create_processed_database.sql ├── 4. raw └── 1.create_raw_tables.sql ├── 5. Data Transformation ├── 0.create_presentation_database.sql ├── 1.race_results.py ├── 2.driver_standings.py ├── 3.constructor_standings.py ├── 4.calculated_race_results.py └── calculated_race_results_sql.sql ├── 6. Data Analysis and Visualization ├── 1.find_dominant_drivers.sql ├── 2.find_dominant_teams.sql ├── 3.viz_dominant_drivers.sql └── 4.viz_dominant_teams.sql ├── 7. Power Bi Reports ├── 412.png ├── 413.png ├── 414.png ├── 415.png ├── 416.png ├── 423.png ├── 424.png ├── 425.png └── 426.png ├── Incremental_load_data ├── 2021-03-21 │ ├── .DS_Store │ ├── circuits.csv │ ├── constructors.json │ ├── drivers.json │ ├── lap_times │ │ ├── .DS_Store │ │ ├── lap_times_split_1.csv │ │ ├── lap_times_split_2.csv │ │ ├── lap_times_split_3.csv │ │ ├── lap_times_split_4.csv │ │ └── lap_times_split_5.csv │ ├── pit_stops.json │ ├── qualifying │ │ ├── .DS_Store │ │ ├── qualifying_split_1.json │ │ └── qualifying_split_2.json │ ├── races.csv │ └── results.json ├── 2021-03-28 │ ├── circuits.csv │ ├── constructors.json │ ├── drivers.json │ ├── lap_times │ │ └── lap_times_split_1.csv │ ├── pit_stops.json │ ├── qualifying │ │ └── qualifying_split_1.json │ ├── races.csv │ └── results.json └── 2021-04-18 │ ├── circuits.csv │ ├── constructors.json │ ├── drivers.json │ ├── lap_times │ └── lap_times_split_1.csv │ ├── pit_stops.json │ ├── qualifying │ └── qualifying_split_1.json │ ├── races.csv │ └── results.json ├── LICENSE ├── README.md ├── Screenshots ├── Step1 - Creating Resources.pdf ├── Step10 - Data Analysis(Delta Format).pdf ├── Step11 - Data Ingestion and Transformation using ADF.pdf ├── Step12 - Creating Power Bi Reports.pdf ├── Step2 - Setting Authentication.pdf ├── Step3 - Data Ingestion.pdf ├── Step4 - Data Transformation.pdf ├── Step5 - Creating External and Managed Tables.pdf ├── Step6 - Data Analysis.pdf ├── Step7 - Ingestion using Incremental Load.pdf ├── Step8 - Transformation using Incremental Load.pdf └── Step9 - Data Ingestion(Delta Format).pdf ├── demo ├── 1.filter_demo.py ├── 10.delta_lake_demo.py ├── 2.join_demo.py ├── 3.aggregation_demo.py ├── 4.sql_temp_view_demo.py ├── 5.sql_temp_view_demo.py ├── 6.sql_objects_demo.sql ├── 7.sql_basics_demo.sql ├── 8.sql_functions_demo.sql └── 9.sql_joins_demo.sql └── utils ├── 1.prepare_for_incremental_load.sql └── read.md /1. Authentication/1.access_adls_using_access_keys.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/1. Authentication/1.access_adls_using_access_keys.py -------------------------------------------------------------------------------- /1. Authentication/2.access_adls_using_sas_token.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/1. Authentication/2.access_adls_using_sas_token.py -------------------------------------------------------------------------------- /1. Authentication/3.access_adls_using_service_principal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/1. Authentication/3.access_adls_using_service_principal.py -------------------------------------------------------------------------------- /1. Authentication/4.access_adls_using_cluster_scoped_credentials.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/1. Authentication/4.access_adls_using_cluster_scoped_credentials.py -------------------------------------------------------------------------------- /1. Authentication/5.explore_dbutils_secrets_utility.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/1. Authentication/5.explore_dbutils_secrets_utility.py -------------------------------------------------------------------------------- /1. Authentication/6.explore_dbfs_root.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/1. Authentication/6.explore_dbfs_root.py -------------------------------------------------------------------------------- /1. Authentication/7.mount_adls_using_service_principal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/1. Authentication/7.mount_adls_using_service_principal.py -------------------------------------------------------------------------------- /1. Authentication/8.mount_adls_containers_for_project.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/1. Authentication/8.mount_adls_containers_for_project.py -------------------------------------------------------------------------------- /2. Includes_configs_and_commonfunctions/common_functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/2. Includes_configs_and_commonfunctions/common_functions.py -------------------------------------------------------------------------------- /2. Includes_configs_and_commonfunctions/configuration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/2. Includes_configs_and_commonfunctions/configuration.py -------------------------------------------------------------------------------- /3. Data Ingestion/0.ingest_all_files.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/3. Data Ingestion/0.ingest_all_files.py -------------------------------------------------------------------------------- /3. Data Ingestion/1.ingest_circuits_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/3. Data Ingestion/1.ingest_circuits_file.py -------------------------------------------------------------------------------- /3. Data Ingestion/2.ingest_races_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/3. Data Ingestion/2.ingest_races_file.py -------------------------------------------------------------------------------- /3. Data Ingestion/3.ingest_constructors_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/3. Data Ingestion/3.ingest_constructors_file.py -------------------------------------------------------------------------------- /3. Data Ingestion/4.ingest_drivers_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/3. Data Ingestion/4.ingest_drivers_file.py -------------------------------------------------------------------------------- /3. Data Ingestion/5.ingest_results_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/3. Data Ingestion/5.ingest_results_file.py -------------------------------------------------------------------------------- /3. Data Ingestion/6.ingest_pit_stops_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/3. Data Ingestion/6.ingest_pit_stops_file.py -------------------------------------------------------------------------------- /3. Data Ingestion/7.ingest_lap_times_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/3. Data Ingestion/7.ingest_lap_times_file.py -------------------------------------------------------------------------------- /3. Data Ingestion/8.ingest_qualifying_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/3. Data Ingestion/8.ingest_qualifying_file.py -------------------------------------------------------------------------------- /3. Data Ingestion/9.create_processed_database.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/3. Data Ingestion/9.create_processed_database.sql -------------------------------------------------------------------------------- /4. raw/1.create_raw_tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/4. raw/1.create_raw_tables.sql -------------------------------------------------------------------------------- /5. Data Transformation/0.create_presentation_database.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/5. Data Transformation/0.create_presentation_database.sql -------------------------------------------------------------------------------- /5. Data Transformation/1.race_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/5. Data Transformation/1.race_results.py -------------------------------------------------------------------------------- /5. Data Transformation/2.driver_standings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/5. Data Transformation/2.driver_standings.py -------------------------------------------------------------------------------- /5. Data Transformation/3.constructor_standings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/5. Data Transformation/3.constructor_standings.py -------------------------------------------------------------------------------- /5. Data Transformation/4.calculated_race_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/5. Data Transformation/4.calculated_race_results.py -------------------------------------------------------------------------------- /5. Data Transformation/calculated_race_results_sql.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/5. Data Transformation/calculated_race_results_sql.sql -------------------------------------------------------------------------------- /6. Data Analysis and Visualization/1.find_dominant_drivers.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/6. Data Analysis and Visualization/1.find_dominant_drivers.sql -------------------------------------------------------------------------------- /6. Data Analysis and Visualization/2.find_dominant_teams.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/6. Data Analysis and Visualization/2.find_dominant_teams.sql -------------------------------------------------------------------------------- /6. Data Analysis and Visualization/3.viz_dominant_drivers.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/6. Data Analysis and Visualization/3.viz_dominant_drivers.sql -------------------------------------------------------------------------------- /6. Data Analysis and Visualization/4.viz_dominant_teams.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/6. Data Analysis and Visualization/4.viz_dominant_teams.sql -------------------------------------------------------------------------------- /7. Power Bi Reports/412.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/7. Power Bi Reports/412.png -------------------------------------------------------------------------------- /7. Power Bi Reports/413.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/7. Power Bi Reports/413.png -------------------------------------------------------------------------------- /7. Power Bi Reports/414.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/7. Power Bi Reports/414.png -------------------------------------------------------------------------------- /7. Power Bi Reports/415.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/7. Power Bi Reports/415.png -------------------------------------------------------------------------------- /7. Power Bi Reports/416.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/7. Power Bi Reports/416.png -------------------------------------------------------------------------------- /7. Power Bi Reports/423.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/7. Power Bi Reports/423.png -------------------------------------------------------------------------------- /7. Power Bi Reports/424.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/7. Power Bi Reports/424.png -------------------------------------------------------------------------------- /7. Power Bi Reports/425.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/7. Power Bi Reports/425.png -------------------------------------------------------------------------------- /7. Power Bi Reports/426.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/7. Power Bi Reports/426.png -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/.DS_Store -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/circuits.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/circuits.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/constructors.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/constructors.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/drivers.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/drivers.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/lap_times/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/lap_times/.DS_Store -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/lap_times/lap_times_split_1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/lap_times/lap_times_split_1.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/lap_times/lap_times_split_2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/lap_times/lap_times_split_2.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/lap_times/lap_times_split_3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/lap_times/lap_times_split_3.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/lap_times/lap_times_split_4.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/lap_times/lap_times_split_4.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/lap_times/lap_times_split_5.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/lap_times/lap_times_split_5.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/pit_stops.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/pit_stops.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/qualifying/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/qualifying/.DS_Store -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/qualifying/qualifying_split_1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/qualifying/qualifying_split_1.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/qualifying/qualifying_split_2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/qualifying/qualifying_split_2.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/races.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/races.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-21/results.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-21/results.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-28/circuits.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-28/circuits.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-28/constructors.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-28/constructors.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-28/drivers.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-28/drivers.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-28/lap_times/lap_times_split_1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-28/lap_times/lap_times_split_1.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-28/pit_stops.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-28/pit_stops.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-28/qualifying/qualifying_split_1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-28/qualifying/qualifying_split_1.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-28/races.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-28/races.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-03-28/results.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-03-28/results.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-04-18/circuits.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-04-18/circuits.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-04-18/constructors.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-04-18/constructors.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-04-18/drivers.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-04-18/drivers.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-04-18/lap_times/lap_times_split_1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-04-18/lap_times/lap_times_split_1.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-04-18/pit_stops.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-04-18/pit_stops.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-04-18/qualifying/qualifying_split_1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-04-18/qualifying/qualifying_split_1.json -------------------------------------------------------------------------------- /Incremental_load_data/2021-04-18/races.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-04-18/races.csv -------------------------------------------------------------------------------- /Incremental_load_data/2021-04-18/results.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Incremental_load_data/2021-04-18/results.json -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/README.md -------------------------------------------------------------------------------- /Screenshots/Step1 - Creating Resources.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Screenshots/Step1 - Creating Resources.pdf -------------------------------------------------------------------------------- /Screenshots/Step10 - Data Analysis(Delta Format).pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Screenshots/Step10 - Data Analysis(Delta Format).pdf -------------------------------------------------------------------------------- /Screenshots/Step11 - Data Ingestion and Transformation using ADF.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Screenshots/Step11 - Data Ingestion and Transformation using ADF.pdf -------------------------------------------------------------------------------- /Screenshots/Step12 - Creating Power Bi Reports.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Screenshots/Step12 - Creating Power Bi Reports.pdf -------------------------------------------------------------------------------- /Screenshots/Step2 - Setting Authentication.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Screenshots/Step2 - Setting Authentication.pdf -------------------------------------------------------------------------------- /Screenshots/Step3 - Data Ingestion.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Screenshots/Step3 - Data Ingestion.pdf -------------------------------------------------------------------------------- /Screenshots/Step4 - Data Transformation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Screenshots/Step4 - Data Transformation.pdf -------------------------------------------------------------------------------- /Screenshots/Step5 - Creating External and Managed Tables.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Screenshots/Step5 - Creating External and Managed Tables.pdf -------------------------------------------------------------------------------- /Screenshots/Step6 - Data Analysis.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Screenshots/Step6 - Data Analysis.pdf -------------------------------------------------------------------------------- /Screenshots/Step7 - Ingestion using Incremental Load.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Screenshots/Step7 - Ingestion using Incremental Load.pdf -------------------------------------------------------------------------------- /Screenshots/Step8 - Transformation using Incremental Load.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Screenshots/Step8 - Transformation using Incremental Load.pdf -------------------------------------------------------------------------------- /Screenshots/Step9 - Data Ingestion(Delta Format).pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/Screenshots/Step9 - Data Ingestion(Delta Format).pdf -------------------------------------------------------------------------------- /demo/1.filter_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/demo/1.filter_demo.py -------------------------------------------------------------------------------- /demo/10.delta_lake_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/demo/10.delta_lake_demo.py -------------------------------------------------------------------------------- /demo/2.join_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/demo/2.join_demo.py -------------------------------------------------------------------------------- /demo/3.aggregation_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/demo/3.aggregation_demo.py -------------------------------------------------------------------------------- /demo/4.sql_temp_view_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/demo/4.sql_temp_view_demo.py -------------------------------------------------------------------------------- /demo/5.sql_temp_view_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/demo/5.sql_temp_view_demo.py -------------------------------------------------------------------------------- /demo/6.sql_objects_demo.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/demo/6.sql_objects_demo.sql -------------------------------------------------------------------------------- /demo/7.sql_basics_demo.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/demo/7.sql_basics_demo.sql -------------------------------------------------------------------------------- /demo/8.sql_functions_demo.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/demo/8.sql_functions_demo.sql -------------------------------------------------------------------------------- /demo/9.sql_joins_demo.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/demo/9.sql_joins_demo.sql -------------------------------------------------------------------------------- /utils/1.prepare_for_incremental_load.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Eshwarreddyt/Data-Engineering-project-on-Formula1-Racing-using-Azure-Databricks/HEAD/utils/1.prepare_for_incremental_load.sql -------------------------------------------------------------------------------- /utils/read.md: -------------------------------------------------------------------------------- 1 | 2 | --------------------------------------------------------------------------------