├── 1- Databricks Lakehouse Platform ├── 1.0 - Creating Clusters.py ├── 1.1 - Notebook Basics.py ├── 1.2 - Understanding Delta Tables.sql ├── 1.3 - Advanced Delta Lake Features.sql ├── 1.4 - Databases and Tables on Databricks.sql ├── 1.5A - Views.sql └── 1.5B - Views (Session 2).sql ├── 2- ELT with Spark SQL and Python ├── 2.1 - Querying Files.sql ├── 2.2 - Writing to Tables.sql ├── 2.3 - Advanced Transformations.sql └── 2.4 - Higher Order Functions and SQL UDFs.sql ├── 3- Incremental Data Processing ├── 3.1 - Structured Streaming.py ├── 3.2 - Auto Loader.py └── 3.3 - Multi-Hop Architecture.py ├── 4- Production Pipelines ├── 4.1 - Delta Live Tables.sql ├── 4.1.2 - Books Pipeline.sql ├── 4.2 - Pipeline Results.py └── 4.3 - Land New Data Task.py ├── 5- Data Governance └── 5.1 - Managing Permissions.sql ├── Includes ├── Copy-Datasets.py ├── Setup.py └── images │ └── bookstore_schema.png ├── Labs ├── 1- Databricks Lakehouse Platform │ ├── 1.0L - Creating Clusters.py │ ├── 1.1L - Notebook Basics.py │ ├── 1.2L - Delta Lake.sql │ └── 1.3L - Databases and Tables on Databricks.sql ├── 2- ELT with Spark SQL and Python │ ├── 2.1L - Querying Files.sql │ └── 2.2L - Advanced ETL.sql ├── 3- Incremental Data Processing │ ├── 3.1L - Spark Structured Streaming.py │ └── 3.2L - Multi-Hop Architecture.py ├── 4- Production Pipelines │ ├── 4.1L - Delta Live Tables.sql │ ├── 4.2L - Jobs - Land New Data.py │ └── 4.3L - Databricks SQL.sql ├── Includes │ ├── Setup-Lab.py │ └── images │ │ ├── bar_graph.png │ │ ├── cluster_par1.png │ │ ├── cluster_par2.png │ │ ├── cluster_par3.png │ │ ├── dashboard.png │ │ ├── line_graph.png │ │ ├── markdown.png │ │ ├── school_job.png │ │ └── school_schema.png └── Solutions │ ├── 1- Databricks Lakehouse Platform │ ├── 1.0L Solution - Creating Clusters.py │ ├── 1.1L Solution - Notebook Basics.py │ ├── 1.2L Solution - Delta Lake.sql │ ├── 1.3L Solution - Databases and Tables on Databricks.sql │ └── helper.py │ ├── 2- ELT with Spark SQL and Python │ ├── 2.1L Solution - Querying Files.sql │ └── 2.2L Solution - Advanced ETL.sql │ ├── 3- Incremental Data Processing │ ├── 3.1L Solution - Spark Structured Streaming.py │ └── 3.2L Solution - Multi-Hop Architecture.py │ ├── 4- Production Pipelines │ ├── 4.1L Solution - Delta Live Tables.sql │ ├── 4.2L Solution - Jobs - Land New Data.py │ └── 4.3L Solution - Databricks SQL.sql │ └── Includes │ └── Setup-Lab.py └── README.md /1- Databricks Lakehouse Platform/1.0 - Creating Clusters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/1- Databricks Lakehouse Platform/1.0 - Creating Clusters.py -------------------------------------------------------------------------------- /1- Databricks Lakehouse Platform/1.1 - Notebook Basics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/1- Databricks Lakehouse Platform/1.1 - Notebook Basics.py -------------------------------------------------------------------------------- /1- Databricks Lakehouse Platform/1.2 - Understanding Delta Tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/1- Databricks Lakehouse Platform/1.2 - Understanding Delta Tables.sql -------------------------------------------------------------------------------- /1- Databricks Lakehouse Platform/1.3 - Advanced Delta Lake Features.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/1- Databricks Lakehouse Platform/1.3 - Advanced Delta Lake Features.sql -------------------------------------------------------------------------------- /1- Databricks Lakehouse Platform/1.4 - Databases and Tables on Databricks.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/1- Databricks Lakehouse Platform/1.4 - Databases and Tables on Databricks.sql -------------------------------------------------------------------------------- /1- Databricks Lakehouse Platform/1.5A - Views.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/1- Databricks Lakehouse Platform/1.5A - Views.sql -------------------------------------------------------------------------------- /1- Databricks Lakehouse Platform/1.5B - Views (Session 2).sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/1- Databricks Lakehouse Platform/1.5B - Views (Session 2).sql -------------------------------------------------------------------------------- /2- ELT with Spark SQL and Python/2.1 - Querying Files.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/2- ELT with Spark SQL and Python/2.1 - Querying Files.sql -------------------------------------------------------------------------------- /2- ELT with Spark SQL and Python/2.2 - Writing to Tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/2- ELT with Spark SQL and Python/2.2 - Writing to Tables.sql -------------------------------------------------------------------------------- /2- ELT with Spark SQL and Python/2.3 - Advanced Transformations.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/2- ELT with Spark SQL and Python/2.3 - Advanced Transformations.sql -------------------------------------------------------------------------------- /2- ELT with Spark SQL and Python/2.4 - Higher Order Functions and SQL UDFs.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/2- ELT with Spark SQL and Python/2.4 - Higher Order Functions and SQL UDFs.sql -------------------------------------------------------------------------------- /3- Incremental Data Processing/3.1 - Structured Streaming.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/3- Incremental Data Processing/3.1 - Structured Streaming.py -------------------------------------------------------------------------------- /3- Incremental Data Processing/3.2 - Auto Loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/3- Incremental Data Processing/3.2 - Auto Loader.py -------------------------------------------------------------------------------- /3- Incremental Data Processing/3.3 - Multi-Hop Architecture.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/3- Incremental Data Processing/3.3 - Multi-Hop Architecture.py -------------------------------------------------------------------------------- /4- Production Pipelines/4.1 - Delta Live Tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/4- Production Pipelines/4.1 - Delta Live Tables.sql -------------------------------------------------------------------------------- /4- Production Pipelines/4.1.2 - Books Pipeline.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/4- Production Pipelines/4.1.2 - Books Pipeline.sql -------------------------------------------------------------------------------- /4- Production Pipelines/4.2 - Pipeline Results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/4- Production Pipelines/4.2 - Pipeline Results.py -------------------------------------------------------------------------------- /4- Production Pipelines/4.3 - Land New Data Task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/4- Production Pipelines/4.3 - Land New Data Task.py -------------------------------------------------------------------------------- /5- Data Governance/5.1 - Managing Permissions.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/5- Data Governance/5.1 - Managing Permissions.sql -------------------------------------------------------------------------------- /Includes/Copy-Datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Includes/Copy-Datasets.py -------------------------------------------------------------------------------- /Includes/Setup.py: -------------------------------------------------------------------------------- 1 | # Databricks notebook source 2 | full_name = "Derar Alhussein" 3 | -------------------------------------------------------------------------------- /Includes/images/bookstore_schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Includes/images/bookstore_schema.png -------------------------------------------------------------------------------- /Labs/1- Databricks Lakehouse Platform/1.0L - Creating Clusters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/1- Databricks Lakehouse Platform/1.0L - Creating Clusters.py -------------------------------------------------------------------------------- /Labs/1- Databricks Lakehouse Platform/1.1L - Notebook Basics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/1- Databricks Lakehouse Platform/1.1L - Notebook Basics.py -------------------------------------------------------------------------------- /Labs/1- Databricks Lakehouse Platform/1.2L - Delta Lake.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/1- Databricks Lakehouse Platform/1.2L - Delta Lake.sql -------------------------------------------------------------------------------- /Labs/1- Databricks Lakehouse Platform/1.3L - Databases and Tables on Databricks.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/1- Databricks Lakehouse Platform/1.3L - Databases and Tables on Databricks.sql -------------------------------------------------------------------------------- /Labs/2- ELT with Spark SQL and Python/2.1L - Querying Files.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/2- ELT with Spark SQL and Python/2.1L - Querying Files.sql -------------------------------------------------------------------------------- /Labs/2- ELT with Spark SQL and Python/2.2L - Advanced ETL.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/2- ELT with Spark SQL and Python/2.2L - Advanced ETL.sql -------------------------------------------------------------------------------- /Labs/3- Incremental Data Processing/3.1L - Spark Structured Streaming.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/3- Incremental Data Processing/3.1L - Spark Structured Streaming.py -------------------------------------------------------------------------------- /Labs/3- Incremental Data Processing/3.2L - Multi-Hop Architecture.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/3- Incremental Data Processing/3.2L - Multi-Hop Architecture.py -------------------------------------------------------------------------------- /Labs/4- Production Pipelines/4.1L - Delta Live Tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/4- Production Pipelines/4.1L - Delta Live Tables.sql -------------------------------------------------------------------------------- /Labs/4- Production Pipelines/4.2L - Jobs - Land New Data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/4- Production Pipelines/4.2L - Jobs - Land New Data.py -------------------------------------------------------------------------------- /Labs/4- Production Pipelines/4.3L - Databricks SQL.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/4- Production Pipelines/4.3L - Databricks SQL.sql -------------------------------------------------------------------------------- /Labs/Includes/Setup-Lab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Includes/Setup-Lab.py -------------------------------------------------------------------------------- /Labs/Includes/images/bar_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Includes/images/bar_graph.png -------------------------------------------------------------------------------- /Labs/Includes/images/cluster_par1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Includes/images/cluster_par1.png -------------------------------------------------------------------------------- /Labs/Includes/images/cluster_par2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Includes/images/cluster_par2.png -------------------------------------------------------------------------------- /Labs/Includes/images/cluster_par3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Includes/images/cluster_par3.png -------------------------------------------------------------------------------- /Labs/Includes/images/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Includes/images/dashboard.png -------------------------------------------------------------------------------- /Labs/Includes/images/line_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Includes/images/line_graph.png -------------------------------------------------------------------------------- /Labs/Includes/images/markdown.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Includes/images/markdown.png -------------------------------------------------------------------------------- /Labs/Includes/images/school_job.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Includes/images/school_job.png -------------------------------------------------------------------------------- /Labs/Includes/images/school_schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Includes/images/school_schema.png -------------------------------------------------------------------------------- /Labs/Solutions/1- Databricks Lakehouse Platform/1.0L Solution - Creating Clusters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/1- Databricks Lakehouse Platform/1.0L Solution - Creating Clusters.py -------------------------------------------------------------------------------- /Labs/Solutions/1- Databricks Lakehouse Platform/1.1L Solution - Notebook Basics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/1- Databricks Lakehouse Platform/1.1L Solution - Notebook Basics.py -------------------------------------------------------------------------------- /Labs/Solutions/1- Databricks Lakehouse Platform/1.2L Solution - Delta Lake.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/1- Databricks Lakehouse Platform/1.2L Solution - Delta Lake.sql -------------------------------------------------------------------------------- /Labs/Solutions/1- Databricks Lakehouse Platform/1.3L Solution - Databases and Tables on Databricks.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/1- Databricks Lakehouse Platform/1.3L Solution - Databases and Tables on Databricks.sql -------------------------------------------------------------------------------- /Labs/Solutions/1- Databricks Lakehouse Platform/helper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/1- Databricks Lakehouse Platform/helper.py -------------------------------------------------------------------------------- /Labs/Solutions/2- ELT with Spark SQL and Python/2.1L Solution - Querying Files.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/2- ELT with Spark SQL and Python/2.1L Solution - Querying Files.sql -------------------------------------------------------------------------------- /Labs/Solutions/2- ELT with Spark SQL and Python/2.2L Solution - Advanced ETL.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/2- ELT with Spark SQL and Python/2.2L Solution - Advanced ETL.sql -------------------------------------------------------------------------------- /Labs/Solutions/3- Incremental Data Processing/3.1L Solution - Spark Structured Streaming.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/3- Incremental Data Processing/3.1L Solution - Spark Structured Streaming.py -------------------------------------------------------------------------------- /Labs/Solutions/3- Incremental Data Processing/3.2L Solution - Multi-Hop Architecture.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/3- Incremental Data Processing/3.2L Solution - Multi-Hop Architecture.py -------------------------------------------------------------------------------- /Labs/Solutions/4- Production Pipelines/4.1L Solution - Delta Live Tables.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/4- Production Pipelines/4.1L Solution - Delta Live Tables.sql -------------------------------------------------------------------------------- /Labs/Solutions/4- Production Pipelines/4.2L Solution - Jobs - Land New Data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/4- Production Pipelines/4.2L Solution - Jobs - Land New Data.py -------------------------------------------------------------------------------- /Labs/Solutions/4- Production Pipelines/4.3L Solution - Databricks SQL.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/4- Production Pipelines/4.3L Solution - Databricks SQL.sql -------------------------------------------------------------------------------- /Labs/Solutions/Includes/Setup-Lab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/Labs/Solutions/Includes/Setup-Lab.py -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/derar-alhussein/Databricks-Certified-Data-Engineer-Associate/HEAD/README.md --------------------------------------------------------------------------------