├── .idea ├── .gitignore ├── inspectionProfiles │ └── profiles_settings.xml ├── misc.xml ├── modules.xml ├── vcs.xml └── youtube_project.iml ├── docs ├── architecture ├── architecture.png ├── database_schema.drawio.png └── readme.md ├── resources ├── __init__.py ├── dev │ ├── config.py │ └── requirements.txt ├── prod │ ├── config.py │ └── requirements.txt ├── qa │ ├── config.py │ └── requirements.txt └── sql_scripts │ └── table_scripts.sql └── src ├── __init__.py ├── main ├── __init__.py ├── delete │ ├── aws_delete.py │ ├── database_delete.py │ └── local_file_delete.py ├── download │ └── aws_file_download.py ├── move │ └── move_files.py ├── read │ ├── aws_read.py │ └── database_read.py ├── transformations │ └── jobs │ │ ├── customer_mart_sql_tranform_write.py │ │ └── dimension_tables_join.py ├── upload │ └── upload_to_s3.py ├── utility │ ├── encrypt_decrypt.py │ ├── logging_config.py │ ├── my_sql_session.py │ ├── s3_client_object.py │ └── spark_session.py └── write │ ├── database_write.py │ └── parquet_writer.py └── test ├── __init__.py ├── extra_column_csv_generated_data.py ├── generate_csv_data.py ├── generate_customer_table_data.py ├── generate_datewise_sales_data.py ├── less_column_csv_generated_data.py ├── sales_data_upload_s3.py └── scratch_pad.py /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/.idea/inspectionProfiles/profiles_settings.xml -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/.idea/misc.xml -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/.idea/modules.xml -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/.idea/vcs.xml -------------------------------------------------------------------------------- /.idea/youtube_project.iml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/.idea/youtube_project.iml -------------------------------------------------------------------------------- /docs/architecture: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/docs/architecture.png -------------------------------------------------------------------------------- /docs/database_schema.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/docs/database_schema.drawio.png -------------------------------------------------------------------------------- /docs/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/docs/readme.md -------------------------------------------------------------------------------- /resources/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/dev/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/resources/dev/config.py -------------------------------------------------------------------------------- /resources/dev/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/resources/dev/requirements.txt -------------------------------------------------------------------------------- /resources/prod/config.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/prod/requirements.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/qa/config.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/qa/requirements.txt: -------------------------------------------------------------------------------- 1 | key=5 -------------------------------------------------------------------------------- /resources/sql_scripts/table_scripts.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/resources/sql_scripts/table_scripts.sql -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/main/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/main/delete/aws_delete.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/delete/aws_delete.py -------------------------------------------------------------------------------- /src/main/delete/database_delete.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/main/delete/local_file_delete.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/delete/local_file_delete.py -------------------------------------------------------------------------------- /src/main/download/aws_file_download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/download/aws_file_download.py -------------------------------------------------------------------------------- /src/main/move/move_files.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/move/move_files.py -------------------------------------------------------------------------------- /src/main/read/aws_read.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/read/aws_read.py -------------------------------------------------------------------------------- /src/main/read/database_read.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/read/database_read.py -------------------------------------------------------------------------------- /src/main/transformations/jobs/customer_mart_sql_tranform_write.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/transformations/jobs/customer_mart_sql_tranform_write.py -------------------------------------------------------------------------------- /src/main/transformations/jobs/dimension_tables_join.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/transformations/jobs/dimension_tables_join.py -------------------------------------------------------------------------------- /src/main/upload/upload_to_s3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/upload/upload_to_s3.py -------------------------------------------------------------------------------- /src/main/utility/encrypt_decrypt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/utility/encrypt_decrypt.py -------------------------------------------------------------------------------- /src/main/utility/logging_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/utility/logging_config.py -------------------------------------------------------------------------------- /src/main/utility/my_sql_session.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/utility/my_sql_session.py -------------------------------------------------------------------------------- /src/main/utility/s3_client_object.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/utility/s3_client_object.py -------------------------------------------------------------------------------- /src/main/utility/spark_session.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/utility/spark_session.py -------------------------------------------------------------------------------- /src/main/write/database_write.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/write/database_write.py -------------------------------------------------------------------------------- /src/main/write/parquet_writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/main/write/parquet_writer.py -------------------------------------------------------------------------------- /src/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/extra_column_csv_generated_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/test/extra_column_csv_generated_data.py -------------------------------------------------------------------------------- /src/test/generate_csv_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/test/generate_csv_data.py -------------------------------------------------------------------------------- /src/test/generate_customer_table_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/test/generate_customer_table_data.py -------------------------------------------------------------------------------- /src/test/generate_datewise_sales_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/test/generate_datewise_sales_data.py -------------------------------------------------------------------------------- /src/test/less_column_csv_generated_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/test/less_column_csv_generated_data.py -------------------------------------------------------------------------------- /src/test/sales_data_upload_s3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/test/sales_data_upload_s3.py -------------------------------------------------------------------------------- /src/test/scratch_pad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manisnitt/youtube_de_project1/HEAD/src/test/scratch_pad.py --------------------------------------------------------------------------------