├── README.md ├── azure-data-factory ├── README.md ├── dataset │ ├── SqlDBTables.json │ ├── SqlServerCopy.json │ ├── address.json │ ├── address_parquet.json │ └── parquetTables.json ├── factory │ └── adf-e2e-de.json ├── integrationRuntime │ └── SelfHostedIntegrationRuntime.json ├── linkedService │ ├── AzureDataLakeStorage1.json │ ├── AzureDatabricks1.json │ ├── keyvault_e2e_de.json │ └── onpremsqlserver.json ├── pipeline │ ├── copy_all_tables.json │ └── copy_pipeline.json ├── publish_config.json └── trigger │ └── scheduled_trigger.json ├── data-lake-storage-gen2 ├── README.md ├── bronze │ └── SalesLT │ │ ├── Address │ │ └── Address.parquet │ │ ├── Customer │ │ └── Customer.parquet │ │ ├── CustomerAddress │ │ └── CustomerAddress.parquet │ │ ├── Product │ │ └── Product.parquet │ │ ├── ProductCategory │ │ └── ProductCategory.parquet │ │ ├── ProductDescription │ │ └── ProductDescription.parquet │ │ ├── ProductModel │ │ └── ProductModel.parquet │ │ ├── ProductModelProductDescription │ │ └── ProductModelProductDescription.parquet │ │ ├── README.md │ │ ├── SalesOrderDetail │ │ └── SalesOrderDetail.parquet │ │ └── SalesOrderHeader │ │ └── SalesOrderHeader.parquet ├── gold │ └── SalesLT │ │ ├── Address │ │ ├── _delta_log │ │ │ ├── 00000000000000000000.crc │ │ │ ├── 00000000000000000000.json │ │ │ ├── 00000000000000000001.crc │ │ │ ├── 00000000000000000001.json │ │ │ ├── 00000000000000000002.crc │ │ │ ├── 00000000000000000002.json │ │ │ ├── 00000000000000000003.crc │ │ │ ├── 00000000000000000003.json │ │ │ ├── 00000000000000000004.crc │ │ │ └── 00000000000000000004.json │ │ ├── part-00000-3aee34cb-97fc-4d61-bdce-46918fa5a157.c000.snappy.parquet │ │ ├── part-00000-4d485ce5-a1b5-40ae-bcbe-d27ffb8e1260.c000.snappy.parquet │ │ ├── part-00000-5cefcac2-34ed-425f-b8b7-6efcddad9bea.c000.snappy.parquet │ │ ├── part-00000-7c26d82f-ebe1-4654-8bf8-37cf7ef205ba.c000.snappy.parquet │ │ └── part-00000-90b1b60b-fe36-443c-8f84-46191f7f6e13.c000.snappy.parquet │ │ ├── Customer │ │ ├── _delta_log │ │ │ ├── 00000000000000000000.crc │ │ │ ├── 00000000000000000000.json │ │ │ ├── 00000000000000000001.crc │ │ │ ├── 00000000000000000001.json │ │ │ ├── 00000000000000000002.crc │ │ │ ├── 00000000000000000002.json │ │ │ ├── 00000000000000000003.crc │ │ │ ├── 00000000000000000003.json │ │ │ ├── 00000000000000000004.crc │ │ │ └── 00000000000000000004.json │ │ ├── part-00000-03ef8173-49f3-444e-afd7-ec7f6bbbbe7f.c000.snappy.parquet │ │ ├── part-00000-1f61cd0f-fea3-4d22-9c2a-542a015df15c.c000.snappy.parquet │ │ ├── part-00000-6930ca08-5aca-4652-828c-891d05f81ec0.c000.snappy.parquet │ │ ├── part-00000-7adc454d-a91d-4ee7-b714-6ea2e091c15d.c000.snappy.parquet │ │ └── part-00000-e10500d5-3de1-4224-93a6-e783afbac4d7.c000.snappy.parquet │ │ ├── CustomerAddress │ │ ├── _delta_log │ │ │ ├── 00000000000000000000.crc │ │ │ ├── 00000000000000000000.json │ │ │ ├── 00000000000000000001.crc │ │ │ ├── 00000000000000000001.json │ │ │ ├── 00000000000000000002.crc │ │ │ ├── 00000000000000000002.json │ │ │ ├── 00000000000000000003.crc │ │ │ ├── 00000000000000000003.json │ │ │ ├── 00000000000000000004.crc │ │ │ └── 00000000000000000004.json │ │ ├── part-00000-03370ca9-9a27-4999-abca-9dd24e1c6cd6.c000.snappy.parquet │ │ ├── part-00000-3b750be7-380a-4729-b83d-8b822ae6d0fb.c000.snappy.parquet │ │ ├── part-00000-5abcc85d-9e37-4f48-872c-524b343a8e3b.c000.snappy.parquet │ │ ├── part-00000-6e3b853b-2217-46ae-967f-d3f83819de4e.c000.snappy.parquet │ │ └── part-00000-b45f9d2d-af60-4f0b-8be7-886037161274.c000.snappy.parquet │ │ ├── Product │ │ ├── _delta_log │ │ │ ├── 00000000000000000000.crc │ │ │ ├── 00000000000000000000.json │ │ │ ├── 00000000000000000001.crc │ │ │ ├── 00000000000000000001.json │ │ │ ├── 00000000000000000002.crc │ │ │ ├── 00000000000000000002.json │ │ │ ├── 00000000000000000003.crc │ │ │ ├── 00000000000000000003.json │ │ │ ├── 00000000000000000004.crc │ │ │ └── 00000000000000000004.json │ │ ├── part-00000-1eef65d4-6707-41e6-ae25-f738de1da9cf.c000.snappy.parquet │ │ ├── part-00000-2fe88964-31b6-4b59-bd8d-5b691b5a05b8.c000.snappy.parquet │ │ ├── part-00000-7c5c647a-3926-47f9-bcab-c4e6eb3d9a13.c000.snappy.parquet │ │ ├── part-00000-cfa1dbcf-7515-4b3e-beef-e904fd54c45f.c000.snappy.parquet │ │ └── part-00000-f96f920b-09b6-4f60-824c-a6b68042970b.c000.snappy.parquet │ │ ├── ProductCategory │ │ ├── _delta_log │ │ │ ├── 00000000000000000000.crc │ │ │ ├── 00000000000000000000.json │ │ │ ├── 00000000000000000001.crc │ │ │ ├── 00000000000000000001.json │ │ │ ├── 00000000000000000002.crc │ │ │ ├── 00000000000000000002.json │ │ │ ├── 00000000000000000003.crc │ │ │ ├── 00000000000000000003.json │ │ │ ├── 00000000000000000004.crc │ │ │ └── 00000000000000000004.json │ │ ├── part-00000-1f78343d-da82-4095-8bf2-6d7cba6cabe9.c000.snappy.parquet │ │ ├── part-00000-25274960-344c-48e7-8562-7e542f540cc0.c000.snappy.parquet │ │ ├── part-00000-606aaa81-2085-4b7e-b1d4-f3199ad62efe.c000.snappy.parquet │ │ ├── part-00000-6cb09020-98e0-4140-bbff-495204e1f9be.c000.snappy.parquet │ │ └── part-00000-7591c294-967e-459b-9cc7-7ce18cfced34.c000.snappy.parquet │ │ ├── ProductDescription │ │ ├── _delta_log │ │ │ ├── 00000000000000000000.crc │ │ │ ├── 00000000000000000000.json │ │ │ ├── 00000000000000000001.crc │ │ │ ├── 00000000000000000001.json │ │ │ ├── 00000000000000000002.crc │ │ │ ├── 00000000000000000002.json │ │ │ ├── 00000000000000000003.crc │ │ │ ├── 00000000000000000003.json │ │ │ ├── 00000000000000000004.crc │ │ │ └── 00000000000000000004.json │ │ ├── part-00000-0ed176d7-e043-4567-9fa3-7b4f43aa5073.c000.snappy.parquet │ │ ├── part-00000-44a4b557-d31b-4272-9fba-24a51a85641d.c000.snappy.parquet │ │ ├── part-00000-8ca9f841-3b00-4e90-8e15-c849db38b4ad.c000.snappy.parquet │ │ ├── part-00000-d35b56c6-d9c2-4fa1-8448-8905394eea6f.c000.snappy.parquet │ │ └── part-00000-fe3f0965-0bd4-432b-93ca-41e4aa6f9083.c000.snappy.parquet │ │ ├── ProductModel │ │ ├── _delta_log │ │ │ ├── 00000000000000000000.crc │ │ │ ├── 00000000000000000000.json │ │ │ ├── 00000000000000000001.crc │ │ │ ├── 00000000000000000001.json │ │ │ ├── 00000000000000000002.crc │ │ │ ├── 00000000000000000002.json │ │ │ ├── 00000000000000000003.crc │ │ │ ├── 00000000000000000003.json │ │ │ ├── 00000000000000000004.crc │ │ │ └── 00000000000000000004.json │ │ ├── part-00000-17e63a75-8057-42b1-9d08-3cab580f2926.c000.snappy.parquet │ │ ├── part-00000-51367f6b-b87c-430a-9749-87905fdf11b7.c000.snappy.parquet │ │ ├── part-00000-6390ead3-2d99-406f-b886-3ec6311252aa.c000.snappy.parquet │ │ ├── part-00000-966877de-a0f4-4d1a-b132-8705b2075fa5.c000.snappy.parquet │ │ └── part-00000-f6e07124-9e23-40c2-9346-c204b63a771c.c000.snappy.parquet │ │ ├── ProductModelProductDescription │ │ ├── _delta_log │ │ │ ├── 00000000000000000000.crc │ │ │ ├── 00000000000000000000.json │ │ │ ├── 00000000000000000001.crc │ │ │ ├── 00000000000000000001.json │ │ │ ├── 00000000000000000002.crc │ │ │ ├── 00000000000000000002.json │ │ │ ├── 00000000000000000003.crc │ │ │ ├── 00000000000000000003.json │ │ │ ├── 00000000000000000004.crc │ │ │ └── 00000000000000000004.json │ │ ├── part-00000-1dcc6e2e-134c-400e-900b-60f896071cf7.c000.snappy.parquet │ │ ├── part-00000-28da8942-1f80-443c-b8f2-607ccb62de09.c000.snappy.parquet │ │ ├── part-00000-46c25c88-801d-4e4c-a766-15d92b0ecab1.c000.snappy.parquet │ │ ├── part-00000-b4a3c6c1-9a5a-4c11-bb9d-5895d8468a72.c000.snappy.parquet │ │ └── part-00000-bca75324-03ae-41c8-9e7f-7d5a004b028b.c000.snappy.parquet │ │ ├── README.md │ │ ├── SalesOrderDetail │ │ ├── _delta_log │ │ │ ├── 00000000000000000000.crc │ │ │ ├── 00000000000000000000.json │ │ │ ├── 00000000000000000001.crc │ │ │ ├── 00000000000000000001.json │ │ │ ├── 00000000000000000002.crc │ │ │ ├── 00000000000000000002.json │ │ │ ├── 00000000000000000003.crc │ │ │ ├── 00000000000000000003.json │ │ │ ├── 00000000000000000004.crc │ │ │ └── 00000000000000000004.json │ │ ├── part-00000-59cdede4-1d43-415a-82a7-88aee027951d.c000.snappy.parquet │ │ ├── part-00000-669afd67-5f5f-4963-bbd7-a34be97320e9.c000.snappy.parquet │ │ ├── part-00000-7bab9271-7425-4463-880e-0d54da91a17f.c000.snappy.parquet │ │ ├── part-00000-8ed851db-99a5-4266-b90a-209c054c3556.c000.snappy.parquet │ │ └── part-00000-f96d4cd0-3c85-4fed-92b8-363bdd365dd9.c000.snappy.parquet │ │ └── SalesOrderHeader │ │ ├── _delta_log │ │ ├── 00000000000000000000.crc │ │ ├── 00000000000000000000.json │ │ ├── 00000000000000000001.crc │ │ ├── 00000000000000000001.json │ │ ├── 00000000000000000002.crc │ │ ├── 00000000000000000002.json │ │ ├── 00000000000000000003.crc │ │ ├── 00000000000000000003.json │ │ ├── 00000000000000000004.crc │ │ ├── 00000000000000000004.json │ │ ├── 00000000000000000005.crc │ │ └── 00000000000000000005.json │ │ ├── part-00000-68eb09ef-5b6e-4bfd-93ed-f4b371c770f7.c000.snappy.parquet │ │ ├── part-00000-97a3a6f7-6eb3-4507-a016-e11aca50b04b.c000.snappy.parquet │ │ ├── part-00000-abc99ad2-8957-40ce-b204-135858c1074c.c000.snappy.parquet │ │ ├── part-00000-d68e0209-6901-417a-a0f2-376bc6178d56.c000.snappy.parquet │ │ ├── part-00000-e0ffa8b3-72b4-45dc-baef-1b91784bdd69.c000.snappy.parquet │ │ └── part-00000-e37187e9-eac6-4980-be7b-3dd0ce355f13.c000.snappy.parquet └── silver │ └── SalesLT │ ├── Address │ ├── _delta_log │ │ ├── 00000000000000000000.crc │ │ ├── 00000000000000000000.json │ │ ├── 00000000000000000001.crc │ │ ├── 00000000000000000001.json │ │ ├── 00000000000000000002.crc │ │ ├── 00000000000000000002.json │ │ ├── 00000000000000000003.crc │ │ └── 00000000000000000003.json │ ├── part-00000-2af1da9e-9c9f-44b4-ab12-07a808431a51.c000.snappy.parquet │ ├── part-00000-935e5376-8e68-4187-9083-cd6096164583.c000.snappy.parquet │ ├── part-00000-9de57176-d172-44a6-ad0c-f672f88f07dd.c000.snappy.parquet │ └── part-00000-c76587e9-df3c-4923-81d8-0f8233e57cfa.c000.snappy.parquet │ ├── Customer │ ├── _delta_log │ │ ├── 00000000000000000000.crc │ │ ├── 00000000000000000000.json │ │ ├── 00000000000000000001.crc │ │ ├── 00000000000000000001.json │ │ ├── 00000000000000000002.crc │ │ ├── 00000000000000000002.json │ │ ├── 00000000000000000003.crc │ │ └── 00000000000000000003.json │ ├── part-00000-6fe882c9-802e-48dd-ab79-52f3696b1267.c000.snappy.parquet │ ├── part-00000-8438bb92-2365-4ba3-bd6b-9a5b0b00dfbe.c000.snappy.parquet │ ├── part-00000-ab2e71eb-9a57-44f1-86ca-40a2dd159199.c000.snappy.parquet │ └── part-00000-f176e8c3-30c9-4ac3-8ee7-ffd5d1da6309.c000.snappy.parquet │ ├── CustomerAddress │ ├── _delta_log │ │ ├── 00000000000000000000.crc │ │ ├── 00000000000000000000.json │ │ ├── 00000000000000000001.crc │ │ ├── 00000000000000000001.json │ │ ├── 00000000000000000002.crc │ │ ├── 00000000000000000002.json │ │ ├── 00000000000000000003.crc │ │ └── 00000000000000000003.json │ ├── part-00000-94054d72-4505-4626-a3fc-f0322b317f2f.c000.snappy.parquet │ ├── part-00000-a16cd430-64c7-4feb-a248-8c61b3ce0596.c000.snappy.parquet │ ├── part-00000-f180a8c0-37aa-4cd2-ad15-6bfd3720778c.c000.snappy.parquet │ └── part-00000-f356d19e-84a7-467d-a773-1c1580c87003.c000.snappy.parquet │ ├── Product │ ├── _delta_log │ │ ├── 00000000000000000000.crc │ │ ├── 00000000000000000000.json │ │ ├── 00000000000000000001.crc │ │ ├── 00000000000000000001.json │ │ ├── 00000000000000000002.crc │ │ ├── 00000000000000000002.json │ │ ├── 00000000000000000003.crc │ │ └── 00000000000000000003.json │ ├── part-00000-156b6231-6271-4a97-b452-256af25cfccc.c000.snappy.parquet │ ├── part-00000-1ca7db36-43ff-4ac4-8d08-814a662b2389.c000.snappy.parquet │ ├── part-00000-54066ff5-094a-42e8-977e-daaeeaabf33c.c000.snappy.parquet │ └── part-00000-96c7b771-3000-4a00-a2e0-7f9de722d125.c000.snappy.parquet │ ├── ProductCategory │ ├── _delta_log │ │ ├── 00000000000000000000.crc │ │ ├── 00000000000000000000.json │ │ ├── 00000000000000000001.crc │ │ ├── 00000000000000000001.json │ │ ├── 00000000000000000002.crc │ │ ├── 00000000000000000002.json │ │ ├── 00000000000000000003.crc │ │ └── 00000000000000000003.json │ ├── part-00000-065d8fa0-632e-46e9-a9b2-41f293e74443.c000.snappy.parquet │ ├── part-00000-3f9d00db-9eb4-4f32-9a43-7afa8f755e30.c000.snappy.parquet │ ├── part-00000-7233e55b-f933-4145-9230-ff4f3895412a.c000.snappy.parquet │ └── part-00000-d15ac2d3-df3d-4b79-8242-46745767cb61.c000.snappy.parquet │ ├── ProductDescription │ ├── _delta_log │ │ ├── 00000000000000000000.crc │ │ ├── 00000000000000000000.json │ │ ├── 00000000000000000001.crc │ │ ├── 00000000000000000001.json │ │ ├── 00000000000000000002.crc │ │ ├── 00000000000000000002.json │ │ ├── 00000000000000000003.crc │ │ └── 00000000000000000003.json │ ├── part-00000-54652dbc-5593-477d-a803-1c5794d26b50.c000.snappy.parquet │ ├── part-00000-9f7770fe-b8df-4254-81c4-5c756ccf240b.c000.snappy.parquet │ ├── part-00000-a421b1ed-f6bc-432b-b9c1-d695e84e688f.c000.snappy.parquet │ └── part-00000-c0fae63d-cb2a-40fb-937e-a9b86893ffb7.c000.snappy.parquet │ ├── ProductModel │ ├── _delta_log │ │ ├── 00000000000000000000.crc │ │ ├── 00000000000000000000.json │ │ ├── 00000000000000000001.crc │ │ ├── 00000000000000000001.json │ │ ├── 00000000000000000002.crc │ │ ├── 00000000000000000002.json │ │ ├── 00000000000000000003.crc │ │ └── 00000000000000000003.json │ ├── part-00000-3cb413bf-7a3a-402a-8894-5c654a71287b.c000.snappy.parquet │ ├── part-00000-44128a55-ea09-4b60-9dbb-b5655e46587d.c000.snappy.parquet │ ├── part-00000-c122b43a-9175-4a24-bc54-d3dd0df79af3.c000.snappy.parquet │ └── part-00000-c1bafcba-b75c-4d04-acb8-c3251ae5982b.c000.snappy.parquet │ ├── ProductModelProductDescription │ ├── _delta_log │ │ ├── 00000000000000000000.crc │ │ ├── 00000000000000000000.json │ │ ├── 00000000000000000001.crc │ │ ├── 00000000000000000001.json │ │ ├── 00000000000000000002.crc │ │ ├── 00000000000000000002.json │ │ ├── 00000000000000000003.crc │ │ └── 00000000000000000003.json │ ├── part-00000-22186c52-bf1d-4f4c-8778-4e9208e26afa.c000.snappy.parquet │ ├── part-00000-58ae7ee1-40e3-4bdf-b837-d4a27d4db73a.c000.snappy.parquet │ ├── part-00000-98a512fd-3130-43eb-ae2b-0cf8240295e7.c000.snappy.parquet │ └── part-00000-e1b6f5d6-2a8f-423f-a647-5d5f44cd690d.c000.snappy.parquet │ ├── README.md │ ├── SalesOrderDetail │ ├── _delta_log │ │ ├── 00000000000000000000.crc │ │ ├── 00000000000000000000.json │ │ ├── 00000000000000000001.crc │ │ ├── 00000000000000000001.json │ │ ├── 00000000000000000002.crc │ │ ├── 00000000000000000002.json │ │ ├── 00000000000000000003.crc │ │ └── 00000000000000000003.json │ ├── part-00000-4a573464-37b2-4be3-a44c-21cb911a64db.c000.snappy.parquet │ ├── part-00000-4e7d429d-7245-4c92-b2ef-5c2b2bf8fb40.c000.snappy.parquet │ ├── part-00000-622fe0a5-4672-454d-94e4-19a554881dd3.c000.snappy.parquet │ └── part-00000-950ad01b-18f3-4e2c-a828-b80f56c12b3f.c000.snappy.parquet │ └── SalesOrderHeader │ ├── _delta_log │ ├── 00000000000000000000.crc │ ├── 00000000000000000000.json │ ├── 00000000000000000001.crc │ ├── 00000000000000000001.json │ ├── 00000000000000000002.crc │ ├── 00000000000000000002.json │ ├── 00000000000000000003.crc │ └── 00000000000000000003.json │ ├── part-00000-27ca24a7-440f-4459-bcae-032171b6c591.c000.snappy.parquet │ ├── part-00000-2ad6b566-7166-4c24-a844-c52cc9a5f2d6.c000.snappy.parquet │ ├── part-00000-5257ddff-d3e8-4921-a995-0fd2eb150b8b.c000.snappy.parquet │ └── part-00000-ada3fe23-8a3a-48fc-be5f-0ef93b631b30.c000.snappy.parquet ├── databricks-notebooks ├── README.md ├── bronze to silver.py ├── silver to gold.py └── storagemount.py ├── key-vault └── README.md ├── power-bi-dashboard ├── README.md └── pbi-e2e-de-dashboard.pbix ├── resource-group ├── ExportedTemplate-rg-e2e-de │ ├── parameters.json │ └── template.json └── README.md └── synapse-analytics ├── README.md ├── credential └── WorkspaceSystemIdentity.json ├── dataset └── goldtables.json ├── integrationRuntime └── AutoResolveIntegrationRuntime.json ├── linkedService ├── serverlessSQLdb.json ├── synw-e2e-de-WorkspaceDefaultSqlServer.json └── synw-e2e-de-WorkspaceDefaultStorage.json ├── pipeline └── Create view.json ├── publish_config.json └── sqlscript └── sp_CreateSQLServerlessView_gold.json /azure-data-factory/README.md: -------------------------------------------------------------------------------- 1 | # Azure Data Factory - (adf-e2e-de) 2 | 3 | Azure Data Factory was the tool used to engineer most of the extract/transform/load (ETL) pipeline. 4 | 5 | **Extract:** 6 | - Connected to the local SQL server database using the Key Vault secrets assigned for the username and password. 7 | - Queried for all of the tables in the on-prem database. 8 | - Copied the tables into the Bronze container in the data lake to undergo transformations. 9 | 10 | **Transform:** 11 | - Queued the Databricks notebooks to run and process the Bronze layer data 12 | - Initially cleansed data from the Bronze layer and loaded to the Silver layer for further transformation. 13 | - Further refined data from the Silver layer and loaded to Gold layer to be used for reporting and analytics. 14 | 15 | **Load:** 16 | - The load piece was completed by Azure Synapse Analytics 17 | - The Gold layer data was loaded into a serverless SQL database. 18 | 19 | Additionally, a trigger was created to run this pipeline daily--completely automating the refinement of new row/column data. 20 | 21 | ![adf-pipeline](https://github.com/user-attachments/assets/124ab5d1-e781-4229-9f0b-3dce0bc3806f) 22 | -------------------------------------------------------------------------------- /azure-data-factory/dataset/SqlDBTables.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SqlDBTables", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "onpremsqlserver", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "SqlServerTable", 10 | "schema": [] 11 | }, 12 | "type": "Microsoft.DataFactory/factories/datasets" 13 | } -------------------------------------------------------------------------------- /azure-data-factory/dataset/SqlServerCopy.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SqlServerCopy", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "onpremsqlserver", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "SqlServerTable", 10 | "schema": [] 11 | }, 12 | "type": "Microsoft.DataFactory/factories/datasets" 13 | } -------------------------------------------------------------------------------- /azure-data-factory/dataset/address.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "address", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "onpremsqlserver", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "SqlServerTable", 10 | "schema": [ 11 | { 12 | "name": "AddressID", 13 | "type": "int", 14 | "precision": 10 15 | }, 16 | { 17 | "name": "AddressLine1", 18 | "type": "nvarchar" 19 | }, 20 | { 21 | "name": "AddressLine2", 22 | "type": "nvarchar" 23 | }, 24 | { 25 | "name": "City", 26 | "type": "nvarchar" 27 | }, 28 | { 29 | "name": "StateProvince", 30 | "type": "nvarchar" 31 | }, 32 | { 33 | "name": "CountryRegion", 34 | "type": "nvarchar" 35 | }, 36 | { 37 | "name": "PostalCode", 38 | "type": "nvarchar" 39 | }, 40 | { 41 | "name": "rowguid", 42 | "type": "uniqueidentifier" 43 | }, 44 | { 45 | "name": "ModifiedDate", 46 | "type": "datetime", 47 | "precision": 23, 48 | "scale": 3 49 | } 50 | ], 51 | "typeProperties": { 52 | "schema": "SalesLT", 53 | "table": "Address" 54 | } 55 | }, 56 | "type": "Microsoft.DataFactory/factories/datasets" 57 | } -------------------------------------------------------------------------------- /azure-data-factory/dataset/address_parquet.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "address_parquet", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileSystem": "bronze" 14 | }, 15 | "compressionCodec": "snappy" 16 | }, 17 | "schema": [] 18 | }, 19 | "type": "Microsoft.DataFactory/factories/datasets" 20 | } -------------------------------------------------------------------------------- /azure-data-factory/dataset/parquetTables.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "parquetTables", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "schemaname": { 10 | "type": "string" 11 | }, 12 | "tablename": { 13 | "type": "string" 14 | } 15 | }, 16 | "annotations": [], 17 | "type": "Parquet", 18 | "typeProperties": { 19 | "location": { 20 | "type": "AzureBlobFSLocation", 21 | "fileName": { 22 | "value": "@concat(dataset().tablename,'.parquet')", 23 | "type": "Expression" 24 | }, 25 | "folderPath": { 26 | "value": "@concat(dataset().schemaname, '/', dataset().tablename)", 27 | "type": "Expression" 28 | }, 29 | "fileSystem": "bronze" 30 | }, 31 | "compressionCodec": "snappy" 32 | }, 33 | "schema": [] 34 | }, 35 | "type": "Microsoft.DataFactory/factories/datasets" 36 | } -------------------------------------------------------------------------------- /azure-data-factory/factory/adf-e2e-de.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adf-e2e-de", 3 | "location": "eastus2", 4 | "identity": { 5 | "type": "SystemAssigned", 6 | "principalId": "a124033d-3983-4547-b8d5-6415353cb9ed", 7 | "tenantId": "d5afdef0-f7d5-487e-b3e7-0caf0a9fefe7" 8 | } 9 | } -------------------------------------------------------------------------------- /azure-data-factory/integrationRuntime/SelfHostedIntegrationRuntime.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SelfHostedIntegrationRuntime", 3 | "properties": { 4 | "type": "SelfHosted", 5 | "description": "Used to connect SQL Server" 6 | } 7 | } -------------------------------------------------------------------------------- /azure-data-factory/linkedService/AzureDataLakeStorage1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDataLakeStorage1", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureBlobFS", 7 | "typeProperties": { 8 | "url": "https://gen2e2ede.dfs.core.windows.net/", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQDdDRDQyQkNELTEyMTMtNDYzOS1BNzYwLTAwMTNDRUMyMjc5Q19iYzBmZTQ3Ni0yODY4LTQ5OGQtYjc0Ny1iNzNmMDhlMjM0YjIiDQp9" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /azure-data-factory/linkedService/AzureDatabricks1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDatabricks1", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureDatabricks", 7 | "typeProperties": { 8 | "domain": "https://adb-2690968036054434.14.azuredatabricks.net", 9 | "accessToken": { 10 | "type": "AzureKeyVaultSecret", 11 | "store": { 12 | "referenceName": "keyvault_e2e_de", 13 | "type": "LinkedServiceReference" 14 | }, 15 | "secretName": "dbwtoken" 16 | }, 17 | "existingClusterId": "0714-153644-9703069z" 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /azure-data-factory/linkedService/keyvault_e2e_de.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "keyvault_e2e_de", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureKeyVault", 7 | "typeProperties": { 8 | "baseUrl": "https://kv-e2e-de.vault.azure.net/" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /azure-data-factory/linkedService/onpremsqlserver.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "onpremsqlserver", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "SqlServer", 7 | "typeProperties": { 8 | "server": "DESKTOP-VSCM7DE\\SQLEXPRESS", 9 | "database": "AdventureWorksLT2022", 10 | "encrypt": "optional", 11 | "trustServerCertificate": false, 12 | "authenticationType": "SQL", 13 | "userName": "dbdr", 14 | "password": { 15 | "type": "AzureKeyVaultSecret", 16 | "store": { 17 | "referenceName": "keyvault_e2e_de", 18 | "type": "LinkedServiceReference" 19 | }, 20 | "secretName": "localdb-password" 21 | } 22 | }, 23 | "connectVia": { 24 | "referenceName": "SelfHostedIntegrationRuntime", 25 | "type": "IntegrationRuntimeReference" 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /azure-data-factory/pipeline/copy_pipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "copy_pipeline", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Copy address table", 7 | "type": "Copy", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "0.12:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "source": { 19 | "type": "SqlServerSource", 20 | "queryTimeout": "02:00:00", 21 | "partitionOption": "None" 22 | }, 23 | "sink": { 24 | "type": "ParquetSink", 25 | "storeSettings": { 26 | "type": "AzureBlobFSWriteSettings" 27 | }, 28 | "formatSettings": { 29 | "type": "ParquetWriteSettings" 30 | } 31 | }, 32 | "enableStaging": false, 33 | "translator": { 34 | "type": "TabularTranslator", 35 | "typeConversion": true, 36 | "typeConversionSettings": { 37 | "allowDataTruncation": true, 38 | "treatBooleanAsNumber": false 39 | } 40 | } 41 | }, 42 | "inputs": [ 43 | { 44 | "referenceName": "address", 45 | "type": "DatasetReference" 46 | } 47 | ], 48 | "outputs": [ 49 | { 50 | "referenceName": "address_parquet", 51 | "type": "DatasetReference" 52 | } 53 | ] 54 | } 55 | ], 56 | "annotations": [], 57 | "lastPublishTime": "2024-07-13T23:42:29Z" 58 | }, 59 | "type": "Microsoft.DataFactory/factories/pipelines" 60 | } -------------------------------------------------------------------------------- /azure-data-factory/publish_config.json: -------------------------------------------------------------------------------- 1 | {"publishBranch":"adf_publish","enableGitComment":true} -------------------------------------------------------------------------------- /azure-data-factory/trigger/scheduled_trigger.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "scheduled_trigger", 3 | "properties": { 4 | "description": "Pulls updated information from the local database through the pipeline. Ensures the PowerBI dashboard has the most current information.", 5 | "annotations": [], 6 | "runtimeState": "Started", 7 | "pipelines": [ 8 | { 9 | "pipelineReference": { 10 | "referenceName": "copy_all_tables", 11 | "type": "PipelineReference" 12 | } 13 | } 14 | ], 15 | "type": "ScheduleTrigger", 16 | "typeProperties": { 17 | "recurrence": { 18 | "frequency": "Day", 19 | "interval": 1, 20 | "startTime": "2024-07-15T12:39:00", 21 | "endTime": "2024-07-16T12:39:00", 22 | "timeZone": "Eastern Standard Time", 23 | "schedule": { 24 | "minutes": [ 25 | 0 26 | ], 27 | "hours": [ 28 | 13 29 | ] 30 | } 31 | } 32 | } 33 | } 34 | } -------------------------------------------------------------------------------- /data-lake-storage-gen2/README.md: -------------------------------------------------------------------------------- 1 | # Azure Data Lake Storage Gen2 - (gen2e2ede) 2 | 3 | This is the data lake that stored this project's data. The data was organized into three containers for the Bronze, Silver, and Gold layers. 4 | 5 | ![gen2-e2e-de-dash](https://github.com/user-attachments/assets/d04cde41-fa07-4bb7-8204-eb2c8def0136) 6 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/bronze/SalesLT/Address/Address.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/bronze/SalesLT/Address/Address.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/bronze/SalesLT/Customer/Customer.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/bronze/SalesLT/Customer/Customer.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/bronze/SalesLT/CustomerAddress/CustomerAddress.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/bronze/SalesLT/CustomerAddress/CustomerAddress.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/bronze/SalesLT/Product/Product.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/bronze/SalesLT/Product/Product.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/bronze/SalesLT/ProductCategory/ProductCategory.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/bronze/SalesLT/ProductCategory/ProductCategory.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/bronze/SalesLT/ProductDescription/ProductDescription.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/bronze/SalesLT/ProductDescription/ProductDescription.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/bronze/SalesLT/ProductModel/ProductModel.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/bronze/SalesLT/ProductModel/ProductModel.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/bronze/SalesLT/ProductModelProductDescription/ProductModelProductDescription.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/bronze/SalesLT/ProductModelProductDescription/ProductModelProductDescription.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/bronze/SalesLT/README.md: -------------------------------------------------------------------------------- 1 | # Bronze Layer 2 | 3 | This version of data is "raw" and yet to be transformed. It is exactly the same as the Microsoft dataset that is hosted on the on-prem SQL server database. 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/bronze/SalesLT/SalesOrderDetail/SalesOrderDetail.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/bronze/SalesLT/SalesOrderDetail/SalesOrderDetail.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/bronze/SalesLT/SalesOrderHeader/SalesOrderHeader.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/bronze/SalesLT/SalesOrderHeader/SalesOrderHeader.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Address/_delta_log/00000000000000000001.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1720993282362,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"824972020226640","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_f67c78c6-ca25-4cf6-91ed-a7486846071c","runId":"534322008629160","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"450","numOutputBytes":"35011"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"2c2c33a9-0ba4-43b9-ae72-f79328e8131a"}} 2 | {"add":{"path":"part-00000-5cefcac2-34ed-425f-b8b7-6efcddad9bea.c000.snappy.parquet","partitionValues":{},"size":35011,"modificationTime":1720993282000,"dataChange":true,"stats":"{\"numRecords\":450,\"minValues\":{\"Address_ID\":9,\"Address_Line1\":\"#500-75 O'Connor Street\",\"Address_Line2\":\"19th Floor\",\"City\":\"Abingdon\",\"State_Province\":\"Alberta\",\"Country_Region\":\"Canada\",\"Postal_Code\":\"48034\",\"rowguid\":\"001d1a67-585c-44b2-aac6-1199afd7\",\"Modified_Date\":\"2005-07-01\"},\"maxValues\":{\"Address_ID\":11382,\"Address_Line1\":\"Wymbush\",\"Address_Line2\":\"Suite 2501\",\"City\":\"Zeeland\",\"State_Province\":\"Wyoming\",\"Country_Region\":\"United States\",\"Postal_Code\":\"Y03 4TN\",\"rowguid\":\"ff2be5e9-7d22-4c1c-acf8-199148e9�\",\"Modified_Date\":\"2008-06-01\"},\"nullCount\":{\"Address_ID\":0,\"Address_Line1\":0,\"Address_Line2\":439,\"City\":0,\"State_Province\":0,\"Country_Region\":0,\"Postal_Code\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1720993282000000","MIN_INSERTION_TIME":"1720993282000000","MAX_INSERTION_TIME":"1720993282000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-7c26d82f-ebe1-4654-8bf8-37cf7ef205ba.c000.snappy.parquet","deletionTimestamp":1720993282361,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":35011,"tags":{"INSERTION_TIME":"1720992038000000","MIN_INSERTION_TIME":"1720992038000000","MAX_INSERTION_TIME":"1720992038000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Address/_delta_log/00000000000000000002.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1720993413945,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"874354820143471","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_bf79d0e2-e145-472c-bbe5-fc4d39ab2505","runId":"117196242773545","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":1,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"450","numOutputBytes":"35011"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"1ecffb42-64e5-4279-81eb-89d85d109951"}} 2 | {"add":{"path":"part-00000-3aee34cb-97fc-4d61-bdce-46918fa5a157.c000.snappy.parquet","partitionValues":{},"size":35011,"modificationTime":1720993413000,"dataChange":true,"stats":"{\"numRecords\":450,\"minValues\":{\"Address_ID\":9,\"Address_Line1\":\"#500-75 O'Connor Street\",\"Address_Line2\":\"19th Floor\",\"City\":\"Abingdon\",\"State_Province\":\"Alberta\",\"Country_Region\":\"Canada\",\"Postal_Code\":\"48034\",\"rowguid\":\"001d1a67-585c-44b2-aac6-1199afd7\",\"Modified_Date\":\"2005-07-01\"},\"maxValues\":{\"Address_ID\":11382,\"Address_Line1\":\"Wymbush\",\"Address_Line2\":\"Suite 2501\",\"City\":\"Zeeland\",\"State_Province\":\"Wyoming\",\"Country_Region\":\"United States\",\"Postal_Code\":\"Y03 4TN\",\"rowguid\":\"ff2be5e9-7d22-4c1c-acf8-199148e9�\",\"Modified_Date\":\"2008-06-01\"},\"nullCount\":{\"Address_ID\":0,\"Address_Line1\":0,\"Address_Line2\":439,\"City\":0,\"State_Province\":0,\"Country_Region\":0,\"Postal_Code\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1720993413000000","MIN_INSERTION_TIME":"1720993413000000","MAX_INSERTION_TIME":"1720993413000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-5cefcac2-34ed-425f-b8b7-6efcddad9bea.c000.snappy.parquet","deletionTimestamp":1720993413940,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":35011,"tags":{"INSERTION_TIME":"1720993282000000","MIN_INSERTION_TIME":"1720993282000000","MAX_INSERTION_TIME":"1720993282000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Address/_delta_log/00000000000000000003.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1721063317257,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"470636195993389","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_1b032853-22d4-4da9-a571-4799af77050e","runId":"397864626697191","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":2,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"450","numOutputBytes":"35011"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"1ebe72f3-9906-4c77-ba37-9beadff65e36"}} 2 | {"add":{"path":"part-00000-90b1b60b-fe36-443c-8f84-46191f7f6e13.c000.snappy.parquet","partitionValues":{},"size":35011,"modificationTime":1721063317000,"dataChange":true,"stats":"{\"numRecords\":450,\"minValues\":{\"Address_ID\":9,\"Address_Line1\":\"#500-75 O'Connor Street\",\"Address_Line2\":\"19th Floor\",\"City\":\"Abingdon\",\"State_Province\":\"Alberta\",\"Country_Region\":\"Canada\",\"Postal_Code\":\"48034\",\"rowguid\":\"001d1a67-585c-44b2-aac6-1199afd7\",\"Modified_Date\":\"2005-07-01\"},\"maxValues\":{\"Address_ID\":11382,\"Address_Line1\":\"Wymbush\",\"Address_Line2\":\"Suite 2501\",\"City\":\"Zeeland\",\"State_Province\":\"Wyoming\",\"Country_Region\":\"United States\",\"Postal_Code\":\"Y03 4TN\",\"rowguid\":\"ff2be5e9-7d22-4c1c-acf8-199148e9�\",\"Modified_Date\":\"2008-06-01\"},\"nullCount\":{\"Address_ID\":0,\"Address_Line1\":0,\"Address_Line2\":439,\"City\":0,\"State_Province\":0,\"Country_Region\":0,\"Postal_Code\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1721063317000000","MIN_INSERTION_TIME":"1721063317000000","MAX_INSERTION_TIME":"1721063317000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-3aee34cb-97fc-4d61-bdce-46918fa5a157.c000.snappy.parquet","deletionTimestamp":1721063317256,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":35011,"tags":{"INSERTION_TIME":"1720993413000000","MIN_INSERTION_TIME":"1720993413000000","MAX_INSERTION_TIME":"1720993413000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Address/_delta_log/00000000000000000004.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1721066684906,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"870726067990048","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_68595b3b-eabb-408f-a8f2-1440bd06f58f","runId":"428767336825282","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3283779565200853"},"clusterId":"0714-153644-9703069z","readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"450","numOutputBytes":"35011"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"0e9f65c8-6e8d-4a62-9ad3-85d01dd32cd9"}} 2 | {"add":{"path":"part-00000-4d485ce5-a1b5-40ae-bcbe-d27ffb8e1260.c000.snappy.parquet","partitionValues":{},"size":35011,"modificationTime":1721066684000,"dataChange":true,"stats":"{\"numRecords\":450,\"minValues\":{\"Address_ID\":9,\"Address_Line1\":\"#500-75 O'Connor Street\",\"Address_Line2\":\"19th Floor\",\"City\":\"Abingdon\",\"State_Province\":\"Alberta\",\"Country_Region\":\"Canada\",\"Postal_Code\":\"48034\",\"rowguid\":\"001d1a67-585c-44b2-aac6-1199afd7\",\"Modified_Date\":\"2005-07-01\"},\"maxValues\":{\"Address_ID\":11382,\"Address_Line1\":\"Wymbush\",\"Address_Line2\":\"Suite 2501\",\"City\":\"Zeeland\",\"State_Province\":\"Wyoming\",\"Country_Region\":\"United States\",\"Postal_Code\":\"Y03 4TN\",\"rowguid\":\"ff2be5e9-7d22-4c1c-acf8-199148e9�\",\"Modified_Date\":\"2008-06-01\"},\"nullCount\":{\"Address_ID\":0,\"Address_Line1\":0,\"Address_Line2\":439,\"City\":0,\"State_Province\":0,\"Country_Region\":0,\"Postal_Code\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1721066684000000","MIN_INSERTION_TIME":"1721066684000000","MAX_INSERTION_TIME":"1721066684000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-90b1b60b-fe36-443c-8f84-46191f7f6e13.c000.snappy.parquet","deletionTimestamp":1721066684905,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":35011,"tags":{"INSERTION_TIME":"1721063317000000","MIN_INSERTION_TIME":"1721063317000000","MAX_INSERTION_TIME":"1721063317000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Address/part-00000-3aee34cb-97fc-4d61-bdce-46918fa5a157.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Address/part-00000-3aee34cb-97fc-4d61-bdce-46918fa5a157.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Address/part-00000-4d485ce5-a1b5-40ae-bcbe-d27ffb8e1260.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Address/part-00000-4d485ce5-a1b5-40ae-bcbe-d27ffb8e1260.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Address/part-00000-5cefcac2-34ed-425f-b8b7-6efcddad9bea.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Address/part-00000-5cefcac2-34ed-425f-b8b7-6efcddad9bea.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Address/part-00000-7c26d82f-ebe1-4654-8bf8-37cf7ef205ba.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Address/part-00000-7c26d82f-ebe1-4654-8bf8-37cf7ef205ba.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Address/part-00000-90b1b60b-fe36-443c-8f84-46191f7f6e13.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Address/part-00000-90b1b60b-fe36-443c-8f84-46191f7f6e13.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Customer/_delta_log/00000000000000000003.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1721063319289,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"470636195993389","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_1b032853-22d4-4da9-a571-4799af77050e","runId":"397864626697191","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":2,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"849","numOutputBytes":"89857"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"3b9b7aa8-45c1-4655-94c1-00dbd912b224"}} 2 | {"add":{"path":"part-00000-7adc454d-a91d-4ee7-b714-6ea2e091c15d.c000.snappy.parquet","partitionValues":{},"size":89857,"modificationTime":1721063319000,"dataChange":true,"stats":"{\"numRecords\":849,\"minValues\":{\"Customer_ID\":1,\"Title\":\"Mr.\",\"First_Name\":\"A.\",\"Middle_Name\":\"A.\",\"Last_Name\":\"Abel\",\"Suffix\":\"II\",\"Company_Name\":\"A Bike Store\",\"Sales_Person\":\"adventure-works\\\\John\",\"Email_Address\":\"a0@adventure-works.com\",\"Phone\":\"1 (11) 500 555-0110\",\"Password_Hash\":\"+CFHIbYlE+nsmSbCpJOTLZXChbtYA9Jz\",\"Password_Salt\":\"+Yo0P+0=\",\"rowguid\":\"009f9cfc-23a3-4b66-92be-bacce909\",\"Modified_Date\":\"2005-07-01\"},\"maxValues\":{\"Customer_ID\":595959,\"Title\":\"Sra.\",\"First_Name\":\"Yvonne\",\"Middle_Name\":\"Z.\",\"Last_Name\":\"Vicknair\",\"Suffix\":\"Sr.\",\"Company_Name\":\"xyz\",\"Sales_Person\":\"adventure-works\\\\shu0\",\"Email_Address\":\"yvonne1@adventure-works.com\",\"Phone\":\"996-555-0196\",\"Password_Hash\":\"zrx5RFbPSu5mxzlRdxbDIH2wSdDS9S9O�\",\"Password_Salt\":\"zmqmiAQ=\",\"rowguid\":\"ffe5de91-a621-482d-bd73-7d6b86db�\",\"Modified_Date\":\"2024-07-15\"},\"nullCount\":{\"Customer_ID\":0,\"Name_Style\":0,\"Title\":7,\"First_Name\":0,\"Middle_Name\":343,\"Last_Name\":0,\"Suffix\":799,\"Company_Name\":0,\"Sales_Person\":0,\"Email_Address\":0,\"Phone\":0,\"Password_Hash\":0,\"Password_Salt\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1721063319000000","MIN_INSERTION_TIME":"1721063319000000","MAX_INSERTION_TIME":"1721063319000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-1f61cd0f-fea3-4d22-9c2a-542a015df15c.c000.snappy.parquet","deletionTimestamp":1721063319288,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":89574,"tags":{"INSERTION_TIME":"1720993415000000","MIN_INSERTION_TIME":"1720993415000000","MAX_INSERTION_TIME":"1720993415000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Customer/_delta_log/00000000000000000004.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1721066686721,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"870726067990048","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_68595b3b-eabb-408f-a8f2-1440bd06f58f","runId":"428767336825282","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3283779565200853"},"clusterId":"0714-153644-9703069z","readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"849","numOutputBytes":"89857"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"2e39f893-8244-4fcc-b951-04d909667999"}} 2 | {"add":{"path":"part-00000-6930ca08-5aca-4652-828c-891d05f81ec0.c000.snappy.parquet","partitionValues":{},"size":89857,"modificationTime":1721066686000,"dataChange":true,"stats":"{\"numRecords\":849,\"minValues\":{\"Customer_ID\":1,\"Title\":\"Mr.\",\"First_Name\":\"A.\",\"Middle_Name\":\"A.\",\"Last_Name\":\"Abel\",\"Suffix\":\"II\",\"Company_Name\":\"A Bike Store\",\"Sales_Person\":\"adventure-works\\\\John\",\"Email_Address\":\"a0@adventure-works.com\",\"Phone\":\"1 (11) 500 555-0110\",\"Password_Hash\":\"+CFHIbYlE+nsmSbCpJOTLZXChbtYA9Jz\",\"Password_Salt\":\"+Yo0P+0=\",\"rowguid\":\"009f9cfc-23a3-4b66-92be-bacce909\",\"Modified_Date\":\"2005-07-01\"},\"maxValues\":{\"Customer_ID\":595959,\"Title\":\"Sra.\",\"First_Name\":\"Yvonne\",\"Middle_Name\":\"Z.\",\"Last_Name\":\"Vicknair\",\"Suffix\":\"Sr.\",\"Company_Name\":\"xyz\",\"Sales_Person\":\"adventure-works\\\\shu0\",\"Email_Address\":\"yvonne1@adventure-works.com\",\"Phone\":\"996-555-0196\",\"Password_Hash\":\"zrx5RFbPSu5mxzlRdxbDIH2wSdDS9S9O�\",\"Password_Salt\":\"zmqmiAQ=\",\"rowguid\":\"ffe5de91-a621-482d-bd73-7d6b86db�\",\"Modified_Date\":\"2024-07-15\"},\"nullCount\":{\"Customer_ID\":0,\"Name_Style\":0,\"Title\":7,\"First_Name\":0,\"Middle_Name\":343,\"Last_Name\":0,\"Suffix\":799,\"Company_Name\":0,\"Sales_Person\":0,\"Email_Address\":0,\"Phone\":0,\"Password_Hash\":0,\"Password_Salt\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1721066686000000","MIN_INSERTION_TIME":"1721066686000000","MAX_INSERTION_TIME":"1721066686000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-7adc454d-a91d-4ee7-b714-6ea2e091c15d.c000.snappy.parquet","deletionTimestamp":1721066686720,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":89857,"tags":{"INSERTION_TIME":"1721063319000000","MIN_INSERTION_TIME":"1721063319000000","MAX_INSERTION_TIME":"1721063319000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Customer/part-00000-03ef8173-49f3-444e-afd7-ec7f6bbbbe7f.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Customer/part-00000-03ef8173-49f3-444e-afd7-ec7f6bbbbe7f.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Customer/part-00000-1f61cd0f-fea3-4d22-9c2a-542a015df15c.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Customer/part-00000-1f61cd0f-fea3-4d22-9c2a-542a015df15c.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Customer/part-00000-6930ca08-5aca-4652-828c-891d05f81ec0.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Customer/part-00000-6930ca08-5aca-4652-828c-891d05f81ec0.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Customer/part-00000-7adc454d-a91d-4ee7-b714-6ea2e091c15d.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Customer/part-00000-7adc454d-a91d-4ee7-b714-6ea2e091c15d.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Customer/part-00000-e10500d5-3de1-4224-93a6-e783afbac4d7.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Customer/part-00000-e10500d5-3de1-4224-93a6-e783afbac4d7.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/CustomerAddress/_delta_log/00000000000000000000.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1720992043004,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"417","numOutputBytes":"20673"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"6e89770b-d181-4e7f-8385-a4b3d061c323"}} 2 | {"metaData":{"id":"166a5a32-d550-484c-ab65-7aded6edac9d","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"Customer_ID\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Address_ID\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Address_Type\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowguid\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Modified_Date\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1720992042508}} 3 | {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} 4 | {"add":{"path":"part-00000-6e3b853b-2217-46ae-967f-d3f83819de4e.c000.snappy.parquet","partitionValues":{},"size":20673,"modificationTime":1720992042000,"dataChange":true,"stats":"{\"numRecords\":417,\"minValues\":{\"Customer_ID\":29485,\"Address_ID\":9,\"Address_Type\":\"Main Office\",\"rowguid\":\"00390f39-ad50-4852-94c9-8cc153fc\",\"Modified_Date\":\"2005-07-01\"},\"maxValues\":{\"Customer_ID\":30118,\"Address_ID\":11382,\"Address_Type\":\"Shipping\",\"rowguid\":\"fff86e21-3f2f-4dee-a66f-e750aceb�\",\"Modified_Date\":\"2008-06-01\"},\"nullCount\":{\"Customer_ID\":0,\"Address_ID\":0,\"Address_Type\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1720992042000000","MIN_INSERTION_TIME":"1720992042000000","MAX_INSERTION_TIME":"1720992042000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 5 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/CustomerAddress/_delta_log/00000000000000000001.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1720993286056,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"824972020226640","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_f67c78c6-ca25-4cf6-91ed-a7486846071c","runId":"534322008629160","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"417","numOutputBytes":"20673"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"0a25ef04-7968-42db-9c4b-2130cbeb6892"}} 2 | {"add":{"path":"part-00000-03370ca9-9a27-4999-abca-9dd24e1c6cd6.c000.snappy.parquet","partitionValues":{},"size":20673,"modificationTime":1720993285000,"dataChange":true,"stats":"{\"numRecords\":417,\"minValues\":{\"Customer_ID\":29485,\"Address_ID\":9,\"Address_Type\":\"Main Office\",\"rowguid\":\"00390f39-ad50-4852-94c9-8cc153fc\",\"Modified_Date\":\"2005-07-01\"},\"maxValues\":{\"Customer_ID\":30118,\"Address_ID\":11382,\"Address_Type\":\"Shipping\",\"rowguid\":\"fff86e21-3f2f-4dee-a66f-e750aceb�\",\"Modified_Date\":\"2008-06-01\"},\"nullCount\":{\"Customer_ID\":0,\"Address_ID\":0,\"Address_Type\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1720993285000000","MIN_INSERTION_TIME":"1720993285000000","MAX_INSERTION_TIME":"1720993285000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-6e3b853b-2217-46ae-967f-d3f83819de4e.c000.snappy.parquet","deletionTimestamp":1720993286055,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":20673,"tags":{"INSERTION_TIME":"1720992042000000","MIN_INSERTION_TIME":"1720992042000000","MAX_INSERTION_TIME":"1720992042000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/CustomerAddress/_delta_log/00000000000000000002.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1720993417172,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"874354820143471","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_bf79d0e2-e145-472c-bbe5-fc4d39ab2505","runId":"117196242773545","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":1,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"417","numOutputBytes":"20673"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"45a94fb2-c5d2-43dc-94dc-3fe4e2a61b02"}} 2 | {"add":{"path":"part-00000-3b750be7-380a-4729-b83d-8b822ae6d0fb.c000.snappy.parquet","partitionValues":{},"size":20673,"modificationTime":1720993417000,"dataChange":true,"stats":"{\"numRecords\":417,\"minValues\":{\"Customer_ID\":29485,\"Address_ID\":9,\"Address_Type\":\"Main Office\",\"rowguid\":\"00390f39-ad50-4852-94c9-8cc153fc\",\"Modified_Date\":\"2005-07-01\"},\"maxValues\":{\"Customer_ID\":30118,\"Address_ID\":11382,\"Address_Type\":\"Shipping\",\"rowguid\":\"fff86e21-3f2f-4dee-a66f-e750aceb�\",\"Modified_Date\":\"2008-06-01\"},\"nullCount\":{\"Customer_ID\":0,\"Address_ID\":0,\"Address_Type\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1720993417000000","MIN_INSERTION_TIME":"1720993417000000","MAX_INSERTION_TIME":"1720993417000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-03370ca9-9a27-4999-abca-9dd24e1c6cd6.c000.snappy.parquet","deletionTimestamp":1720993417171,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":20673,"tags":{"INSERTION_TIME":"1720993285000000","MIN_INSERTION_TIME":"1720993285000000","MAX_INSERTION_TIME":"1720993285000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/CustomerAddress/_delta_log/00000000000000000003.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1721063321180,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"470636195993389","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_1b032853-22d4-4da9-a571-4799af77050e","runId":"397864626697191","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":2,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"417","numOutputBytes":"20673"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"02870fb1-c57f-418c-acbe-cb3c70e7f724"}} 2 | {"add":{"path":"part-00000-5abcc85d-9e37-4f48-872c-524b343a8e3b.c000.snappy.parquet","partitionValues":{},"size":20673,"modificationTime":1721063321000,"dataChange":true,"stats":"{\"numRecords\":417,\"minValues\":{\"Customer_ID\":29485,\"Address_ID\":9,\"Address_Type\":\"Main Office\",\"rowguid\":\"00390f39-ad50-4852-94c9-8cc153fc\",\"Modified_Date\":\"2005-07-01\"},\"maxValues\":{\"Customer_ID\":30118,\"Address_ID\":11382,\"Address_Type\":\"Shipping\",\"rowguid\":\"fff86e21-3f2f-4dee-a66f-e750aceb�\",\"Modified_Date\":\"2008-06-01\"},\"nullCount\":{\"Customer_ID\":0,\"Address_ID\":0,\"Address_Type\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1721063321000000","MIN_INSERTION_TIME":"1721063321000000","MAX_INSERTION_TIME":"1721063321000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-3b750be7-380a-4729-b83d-8b822ae6d0fb.c000.snappy.parquet","deletionTimestamp":1721063321179,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":20673,"tags":{"INSERTION_TIME":"1720993417000000","MIN_INSERTION_TIME":"1720993417000000","MAX_INSERTION_TIME":"1720993417000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/CustomerAddress/_delta_log/00000000000000000004.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1721066687972,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"870726067990048","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_68595b3b-eabb-408f-a8f2-1440bd06f58f","runId":"428767336825282","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3283779565200853"},"clusterId":"0714-153644-9703069z","readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"417","numOutputBytes":"20673"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"f03e428b-e5ed-4ddf-b29a-d2449cc8d1e0"}} 2 | {"add":{"path":"part-00000-b45f9d2d-af60-4f0b-8be7-886037161274.c000.snappy.parquet","partitionValues":{},"size":20673,"modificationTime":1721066687000,"dataChange":true,"stats":"{\"numRecords\":417,\"minValues\":{\"Customer_ID\":29485,\"Address_ID\":9,\"Address_Type\":\"Main Office\",\"rowguid\":\"00390f39-ad50-4852-94c9-8cc153fc\",\"Modified_Date\":\"2005-07-01\"},\"maxValues\":{\"Customer_ID\":30118,\"Address_ID\":11382,\"Address_Type\":\"Shipping\",\"rowguid\":\"fff86e21-3f2f-4dee-a66f-e750aceb�\",\"Modified_Date\":\"2008-06-01\"},\"nullCount\":{\"Customer_ID\":0,\"Address_ID\":0,\"Address_Type\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1721066687000000","MIN_INSERTION_TIME":"1721066687000000","MAX_INSERTION_TIME":"1721066687000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-5abcc85d-9e37-4f48-872c-524b343a8e3b.c000.snappy.parquet","deletionTimestamp":1721066687972,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":20673,"tags":{"INSERTION_TIME":"1721063321000000","MIN_INSERTION_TIME":"1721063321000000","MAX_INSERTION_TIME":"1721063321000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/CustomerAddress/part-00000-03370ca9-9a27-4999-abca-9dd24e1c6cd6.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/CustomerAddress/part-00000-03370ca9-9a27-4999-abca-9dd24e1c6cd6.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/CustomerAddress/part-00000-3b750be7-380a-4729-b83d-8b822ae6d0fb.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/CustomerAddress/part-00000-3b750be7-380a-4729-b83d-8b822ae6d0fb.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/CustomerAddress/part-00000-5abcc85d-9e37-4f48-872c-524b343a8e3b.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/CustomerAddress/part-00000-5abcc85d-9e37-4f48-872c-524b343a8e3b.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/CustomerAddress/part-00000-6e3b853b-2217-46ae-967f-d3f83819de4e.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/CustomerAddress/part-00000-6e3b853b-2217-46ae-967f-d3f83819de4e.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/CustomerAddress/part-00000-b45f9d2d-af60-4f0b-8be7-886037161274.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/CustomerAddress/part-00000-b45f9d2d-af60-4f0b-8be7-886037161274.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Product/part-00000-1eef65d4-6707-41e6-ae25-f738de1da9cf.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Product/part-00000-1eef65d4-6707-41e6-ae25-f738de1da9cf.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Product/part-00000-2fe88964-31b6-4b59-bd8d-5b691b5a05b8.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Product/part-00000-2fe88964-31b6-4b59-bd8d-5b691b5a05b8.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Product/part-00000-7c5c647a-3926-47f9-bcab-c4e6eb3d9a13.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Product/part-00000-7c5c647a-3926-47f9-bcab-c4e6eb3d9a13.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Product/part-00000-cfa1dbcf-7515-4b3e-beef-e904fd54c45f.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Product/part-00000-cfa1dbcf-7515-4b3e-beef-e904fd54c45f.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/Product/part-00000-f96f920b-09b6-4f60-824c-a6b68042970b.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/Product/part-00000-f96f920b-09b6-4f60-824c-a6b68042970b.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductCategory/_delta_log/00000000000000000000.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1720992046467,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"41","numOutputBytes":"3655"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"e08a5c75-8843-4827-a353-a48a0b5c1679"}} 2 | {"metaData":{"id":"7dd7ea78-78c2-4958-b060-510778233f6f","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"Product_Category_ID\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Parent_Product_Category_ID\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowguid\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Modified_Date\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1720992045840}} 3 | {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} 4 | {"add":{"path":"part-00000-7591c294-967e-459b-9cc7-7ce18cfced34.c000.snappy.parquet","partitionValues":{},"size":3655,"modificationTime":1720992046000,"dataChange":true,"stats":"{\"numRecords\":41,\"minValues\":{\"Product_Category_ID\":1,\"Parent_Product_Category_ID\":1,\"Name\":\"Accessories\",\"rowguid\":\"000310c0-bcc8-42c4-b0c3-45ae611a\",\"Modified_Date\":\"2002-06-01\"},\"maxValues\":{\"Product_Category_ID\":41,\"Parent_Product_Category_ID\":4,\"Name\":\"Wheels\",\"rowguid\":\"fe4d46f2-c87c-48c5-a4a1-3f55712d�\",\"Modified_Date\":\"2002-06-01\"},\"nullCount\":{\"Product_Category_ID\":0,\"Parent_Product_Category_ID\":4,\"Name\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1720992046000000","MIN_INSERTION_TIME":"1720992046000000","MAX_INSERTION_TIME":"1720992046000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 5 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductCategory/_delta_log/00000000000000000001.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1720993289699,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"824972020226640","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_f67c78c6-ca25-4cf6-91ed-a7486846071c","runId":"534322008629160","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"41","numOutputBytes":"3655"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"1903fb79-53b6-40ad-a748-88227bc637e3"}} 2 | {"add":{"path":"part-00000-6cb09020-98e0-4140-bbff-495204e1f9be.c000.snappy.parquet","partitionValues":{},"size":3655,"modificationTime":1720993289000,"dataChange":true,"stats":"{\"numRecords\":41,\"minValues\":{\"Product_Category_ID\":1,\"Parent_Product_Category_ID\":1,\"Name\":\"Accessories\",\"rowguid\":\"000310c0-bcc8-42c4-b0c3-45ae611a\",\"Modified_Date\":\"2002-06-01\"},\"maxValues\":{\"Product_Category_ID\":41,\"Parent_Product_Category_ID\":4,\"Name\":\"Wheels\",\"rowguid\":\"fe4d46f2-c87c-48c5-a4a1-3f55712d�\",\"Modified_Date\":\"2002-06-01\"},\"nullCount\":{\"Product_Category_ID\":0,\"Parent_Product_Category_ID\":4,\"Name\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1720993289000000","MIN_INSERTION_TIME":"1720993289000000","MAX_INSERTION_TIME":"1720993289000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-7591c294-967e-459b-9cc7-7ce18cfced34.c000.snappy.parquet","deletionTimestamp":1720993289698,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":3655,"tags":{"INSERTION_TIME":"1720992046000000","MIN_INSERTION_TIME":"1720992046000000","MAX_INSERTION_TIME":"1720992046000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductCategory/_delta_log/00000000000000000002.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1720993420459,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"874354820143471","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_bf79d0e2-e145-472c-bbe5-fc4d39ab2505","runId":"117196242773545","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":1,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"41","numOutputBytes":"3655"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"004deef5-45ca-42ef-adcc-abbb096a6d63"}} 2 | {"add":{"path":"part-00000-25274960-344c-48e7-8562-7e542f540cc0.c000.snappy.parquet","partitionValues":{},"size":3655,"modificationTime":1720993420000,"dataChange":true,"stats":"{\"numRecords\":41,\"minValues\":{\"Product_Category_ID\":1,\"Parent_Product_Category_ID\":1,\"Name\":\"Accessories\",\"rowguid\":\"000310c0-bcc8-42c4-b0c3-45ae611a\",\"Modified_Date\":\"2002-06-01\"},\"maxValues\":{\"Product_Category_ID\":41,\"Parent_Product_Category_ID\":4,\"Name\":\"Wheels\",\"rowguid\":\"fe4d46f2-c87c-48c5-a4a1-3f55712d�\",\"Modified_Date\":\"2002-06-01\"},\"nullCount\":{\"Product_Category_ID\":0,\"Parent_Product_Category_ID\":4,\"Name\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1720993420000000","MIN_INSERTION_TIME":"1720993420000000","MAX_INSERTION_TIME":"1720993420000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-6cb09020-98e0-4140-bbff-495204e1f9be.c000.snappy.parquet","deletionTimestamp":1720993420458,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":3655,"tags":{"INSERTION_TIME":"1720993289000000","MIN_INSERTION_TIME":"1720993289000000","MAX_INSERTION_TIME":"1720993289000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductCategory/_delta_log/00000000000000000003.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1721063324564,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"470636195993389","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_1b032853-22d4-4da9-a571-4799af77050e","runId":"397864626697191","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":2,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"41","numOutputBytes":"3655"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"1896bce2-4ce2-456a-b14d-58d3ae256420"}} 2 | {"add":{"path":"part-00000-1f78343d-da82-4095-8bf2-6d7cba6cabe9.c000.snappy.parquet","partitionValues":{},"size":3655,"modificationTime":1721063324000,"dataChange":true,"stats":"{\"numRecords\":41,\"minValues\":{\"Product_Category_ID\":1,\"Parent_Product_Category_ID\":1,\"Name\":\"Accessories\",\"rowguid\":\"000310c0-bcc8-42c4-b0c3-45ae611a\",\"Modified_Date\":\"2002-06-01\"},\"maxValues\":{\"Product_Category_ID\":41,\"Parent_Product_Category_ID\":4,\"Name\":\"Wheels\",\"rowguid\":\"fe4d46f2-c87c-48c5-a4a1-3f55712d�\",\"Modified_Date\":\"2002-06-01\"},\"nullCount\":{\"Product_Category_ID\":0,\"Parent_Product_Category_ID\":4,\"Name\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1721063324000000","MIN_INSERTION_TIME":"1721063324000000","MAX_INSERTION_TIME":"1721063324000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-25274960-344c-48e7-8562-7e542f540cc0.c000.snappy.parquet","deletionTimestamp":1721063324563,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":3655,"tags":{"INSERTION_TIME":"1720993420000000","MIN_INSERTION_TIME":"1720993420000000","MAX_INSERTION_TIME":"1720993420000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductCategory/_delta_log/00000000000000000004.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1721066691109,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"870726067990048","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_68595b3b-eabb-408f-a8f2-1440bd06f58f","runId":"428767336825282","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3283779565200853"},"clusterId":"0714-153644-9703069z","readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"41","numOutputBytes":"3655"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"53f37068-ee2f-485a-a275-dbe11208fb0f"}} 2 | {"add":{"path":"part-00000-606aaa81-2085-4b7e-b1d4-f3199ad62efe.c000.snappy.parquet","partitionValues":{},"size":3655,"modificationTime":1721066690000,"dataChange":true,"stats":"{\"numRecords\":41,\"minValues\":{\"Product_Category_ID\":1,\"Parent_Product_Category_ID\":1,\"Name\":\"Accessories\",\"rowguid\":\"000310c0-bcc8-42c4-b0c3-45ae611a\",\"Modified_Date\":\"2002-06-01\"},\"maxValues\":{\"Product_Category_ID\":41,\"Parent_Product_Category_ID\":4,\"Name\":\"Wheels\",\"rowguid\":\"fe4d46f2-c87c-48c5-a4a1-3f55712d�\",\"Modified_Date\":\"2002-06-01\"},\"nullCount\":{\"Product_Category_ID\":0,\"Parent_Product_Category_ID\":4,\"Name\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1721066690000000","MIN_INSERTION_TIME":"1721066690000000","MAX_INSERTION_TIME":"1721066690000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-1f78343d-da82-4095-8bf2-6d7cba6cabe9.c000.snappy.parquet","deletionTimestamp":1721066691108,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":3655,"tags":{"INSERTION_TIME":"1721063324000000","MIN_INSERTION_TIME":"1721063324000000","MAX_INSERTION_TIME":"1721063324000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductCategory/part-00000-1f78343d-da82-4095-8bf2-6d7cba6cabe9.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/ProductCategory/part-00000-1f78343d-da82-4095-8bf2-6d7cba6cabe9.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductCategory/part-00000-25274960-344c-48e7-8562-7e542f540cc0.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/ProductCategory/part-00000-25274960-344c-48e7-8562-7e542f540cc0.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductCategory/part-00000-606aaa81-2085-4b7e-b1d4-f3199ad62efe.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/ProductCategory/part-00000-606aaa81-2085-4b7e-b1d4-f3199ad62efe.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductCategory/part-00000-6cb09020-98e0-4140-bbff-495204e1f9be.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/ProductCategory/part-00000-6cb09020-98e0-4140-bbff-495204e1f9be.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductCategory/part-00000-7591c294-967e-459b-9cc7-7ce18cfced34.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/ProductCategory/part-00000-7591c294-967e-459b-9cc7-7ce18cfced34.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductDescription/_delta_log/00000000000000000000.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1720992048482,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"762","numOutputBytes":"54180"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"280db6cb-40a7-4f82-b58b-0e0a0f559ebf"}} 2 | {"metaData":{"id":"613b4888-ccc3-42c2-ba76-dd8b52831b3e","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"Product_Description_ID\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Description\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowguid\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Modified_Date\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1720992047931}} 3 | {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} 4 | {"add":{"path":"part-00000-44a4b557-d31b-4272-9fba-24a51a85641d.c000.snappy.parquet","partitionValues":{},"size":54180,"modificationTime":1720992048000,"dataChange":true,"stats":"{\"numRecords\":762,\"minValues\":{\"Product_Description_ID\":3,\"Description\":\"10-speed aluminum derailleur wit\",\"rowguid\":\"00ffdfac-0207-4df0-8051-7d3c8848\",\"Modified_Date\":\"2007-06-01\"},\"maxValues\":{\"Product_Description_ID\":2010,\"Description\":\"Équipé de la même technologie qu�\",\"rowguid\":\"ffc3d488-a805-49f9-956c-e6f2f6e1�\",\"Modified_Date\":\"2008-03-11\"},\"nullCount\":{\"Product_Description_ID\":0,\"Description\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1720992048000000","MIN_INSERTION_TIME":"1720992048000000","MAX_INSERTION_TIME":"1720992048000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 5 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductDescription/_delta_log/00000000000000000001.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1720993291081,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"824972020226640","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_f67c78c6-ca25-4cf6-91ed-a7486846071c","runId":"534322008629160","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"762","numOutputBytes":"54180"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"6dd14b46-c1db-456e-b28c-555d2128c180"}} 2 | {"add":{"path":"part-00000-8ca9f841-3b00-4e90-8e15-c849db38b4ad.c000.snappy.parquet","partitionValues":{},"size":54180,"modificationTime":1720993291000,"dataChange":true,"stats":"{\"numRecords\":762,\"minValues\":{\"Product_Description_ID\":3,\"Description\":\"10-speed aluminum derailleur wit\",\"rowguid\":\"00ffdfac-0207-4df0-8051-7d3c8848\",\"Modified_Date\":\"2007-06-01\"},\"maxValues\":{\"Product_Description_ID\":2010,\"Description\":\"Équipé de la même technologie qu�\",\"rowguid\":\"ffc3d488-a805-49f9-956c-e6f2f6e1�\",\"Modified_Date\":\"2008-03-11\"},\"nullCount\":{\"Product_Description_ID\":0,\"Description\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1720993291000000","MIN_INSERTION_TIME":"1720993291000000","MAX_INSERTION_TIME":"1720993291000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-44a4b557-d31b-4272-9fba-24a51a85641d.c000.snappy.parquet","deletionTimestamp":1720993291079,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":54180,"tags":{"INSERTION_TIME":"1720992048000000","MIN_INSERTION_TIME":"1720992048000000","MAX_INSERTION_TIME":"1720992048000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductDescription/_delta_log/00000000000000000002.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1720993421781,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"874354820143471","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_bf79d0e2-e145-472c-bbe5-fc4d39ab2505","runId":"117196242773545","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":1,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"762","numOutputBytes":"54180"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"572cf4a1-5c45-4fe4-92f1-9cf95d3384d7"}} 2 | {"add":{"path":"part-00000-d35b56c6-d9c2-4fa1-8448-8905394eea6f.c000.snappy.parquet","partitionValues":{},"size":54180,"modificationTime":1720993421000,"dataChange":true,"stats":"{\"numRecords\":762,\"minValues\":{\"Product_Description_ID\":3,\"Description\":\"10-speed aluminum derailleur wit\",\"rowguid\":\"00ffdfac-0207-4df0-8051-7d3c8848\",\"Modified_Date\":\"2007-06-01\"},\"maxValues\":{\"Product_Description_ID\":2010,\"Description\":\"Équipé de la même technologie qu�\",\"rowguid\":\"ffc3d488-a805-49f9-956c-e6f2f6e1�\",\"Modified_Date\":\"2008-03-11\"},\"nullCount\":{\"Product_Description_ID\":0,\"Description\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1720993421000000","MIN_INSERTION_TIME":"1720993421000000","MAX_INSERTION_TIME":"1720993421000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-8ca9f841-3b00-4e90-8e15-c849db38b4ad.c000.snappy.parquet","deletionTimestamp":1720993421771,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":54180,"tags":{"INSERTION_TIME":"1720993291000000","MIN_INSERTION_TIME":"1720993291000000","MAX_INSERTION_TIME":"1720993291000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductDescription/_delta_log/00000000000000000003.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1721063326694,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"470636195993389","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_1b032853-22d4-4da9-a571-4799af77050e","runId":"397864626697191","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","readVersion":2,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"762","numOutputBytes":"54180"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"31d1b956-02b7-4474-b6a9-a9dfbab565fd"}} 2 | {"add":{"path":"part-00000-0ed176d7-e043-4567-9fa3-7b4f43aa5073.c000.snappy.parquet","partitionValues":{},"size":54180,"modificationTime":1721063326000,"dataChange":true,"stats":"{\"numRecords\":762,\"minValues\":{\"Product_Description_ID\":3,\"Description\":\"10-speed aluminum derailleur wit\",\"rowguid\":\"00ffdfac-0207-4df0-8051-7d3c8848\",\"Modified_Date\":\"2007-06-01\"},\"maxValues\":{\"Product_Description_ID\":2010,\"Description\":\"Équipé de la même technologie qu�\",\"rowguid\":\"ffc3d488-a805-49f9-956c-e6f2f6e1�\",\"Modified_Date\":\"2008-03-11\"},\"nullCount\":{\"Product_Description_ID\":0,\"Description\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1721063326000000","MIN_INSERTION_TIME":"1721063326000000","MAX_INSERTION_TIME":"1721063326000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-d35b56c6-d9c2-4fa1-8448-8905394eea6f.c000.snappy.parquet","deletionTimestamp":1721063326693,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":54180,"tags":{"INSERTION_TIME":"1720993421000000","MIN_INSERTION_TIME":"1720993421000000","MAX_INSERTION_TIME":"1720993421000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductDescription/_delta_log/00000000000000000004.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1721066692355,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"job":{"jobId":"870726067990048","jobName":"ADF_adf-e2e-de_copy_all_tables_Silver to Gold_68595b3b-eabb-408f-a8f2-1440bd06f58f","runId":"428767336825282","jobOwnerId":"4650962323305322","triggerType":"manual"},"notebook":{"notebookId":"3283779565200853"},"clusterId":"0714-153644-9703069z","readVersion":3,"isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"762","numOutputBytes":"54180"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"9193111d-89c8-41b2-a0a2-c375fbfc7598"}} 2 | {"add":{"path":"part-00000-fe3f0965-0bd4-432b-93ca-41e4aa6f9083.c000.snappy.parquet","partitionValues":{},"size":54180,"modificationTime":1721066692000,"dataChange":true,"stats":"{\"numRecords\":762,\"minValues\":{\"Product_Description_ID\":3,\"Description\":\"10-speed aluminum derailleur wit\",\"rowguid\":\"00ffdfac-0207-4df0-8051-7d3c8848\",\"Modified_Date\":\"2007-06-01\"},\"maxValues\":{\"Product_Description_ID\":2010,\"Description\":\"Équipé de la même technologie qu�\",\"rowguid\":\"ffc3d488-a805-49f9-956c-e6f2f6e1�\",\"Modified_Date\":\"2008-03-11\"},\"nullCount\":{\"Product_Description_ID\":0,\"Description\":0,\"rowguid\":0,\"Modified_Date\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1721066692000000","MIN_INSERTION_TIME":"1721066692000000","MAX_INSERTION_TIME":"1721066692000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 3 | {"remove":{"path":"part-00000-0ed176d7-e043-4567-9fa3-7b4f43aa5073.c000.snappy.parquet","deletionTimestamp":1721066692353,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":54180,"tags":{"INSERTION_TIME":"1721063326000000","MIN_INSERTION_TIME":"1721063326000000","MAX_INSERTION_TIME":"1721063326000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} 4 | -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductDescription/part-00000-0ed176d7-e043-4567-9fa3-7b4f43aa5073.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/ProductDescription/part-00000-0ed176d7-e043-4567-9fa3-7b4f43aa5073.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductDescription/part-00000-44a4b557-d31b-4272-9fba-24a51a85641d.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/ProductDescription/part-00000-44a4b557-d31b-4272-9fba-24a51a85641d.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductDescription/part-00000-8ca9f841-3b00-4e90-8e15-c849db38b4ad.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/ProductDescription/part-00000-8ca9f841-3b00-4e90-8e15-c849db38b4ad.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductDescription/part-00000-d35b56c6-d9c2-4fa1-8448-8905394eea6f.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/ProductDescription/part-00000-d35b56c6-d9c2-4fa1-8448-8905394eea6f.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductDescription/part-00000-fe3f0965-0bd4-432b-93ca-41e4aa6f9083.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/data-lake-storage-gen2/gold/SalesLT/ProductDescription/part-00000-fe3f0965-0bd4-432b-93ca-41e4aa6f9083.c000.snappy.parquet -------------------------------------------------------------------------------- /data-lake-storage-gen2/gold/SalesLT/ProductModel/_delta_log/00000000000000000000.json: -------------------------------------------------------------------------------- 1 | {"commitInfo":{"timestamp":1720992050080,"userId":"4650962323305322","userName":"alexanderbean6601@gmail.com","operation":"WRITE","operationParameters":{"mode":"Overwrite","statsOnLoad":false,"partitionBy":"[]"},"notebook":{"notebookId":"3539605995336538"},"clusterId":"0714-153644-9703069z","isolationLevel":"WriteSerializable","isBlindAppend":false,"operationMetrics":{"numFiles":"1","numOutputRows":"128","numOutputBytes":"20265"},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/14.3.x-photon-scala2.12","txnId":"f80a1d5d-d842-4b41-8264-61ee7423188a"}} 2 | {"metaData":{"id":"d232fca8-53e8-4258-933a-8f94ce7eafa9","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"Product_Model_ID\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Catalog_Description\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowguid\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"Modified_Date\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.enableDeletionVectors":"true"},"createdTime":1720992049609}} 3 | {"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors"]}} 4 | {"add":{"path":"part-00000-6390ead3-2d99-406f-b886-3ec6311252aa.c000.snappy.parquet","partitionValues":{},"size":20265,"modificationTime":1720992049000,"dataChange":true,"stats":"{\"numRecords\":128,\"minValues\":{\"Product_Model_ID\":1,\"Name\":\"All-Purpose Bike Stand\",\"Catalog_Description\":\" to the source URI of your mount point. 10 | dbutils.fs.mount( 11 | source = "abfss://bronze@gen2e2ede.dfs.core.windows.net/", 12 | mount_point = "/mnt/bronze", 13 | extra_configs = configs) 14 | 15 | # COMMAND ---------- 16 | 17 | # Mount Azure Data Lake Gen2 Storage (Silver layer) to Databricks File System (DBFS) using credential passthrough 18 | 19 | configs = { 20 | "fs.azure.account.auth.type": "CustomAccessToken", 21 | "fs.azure.account.custom.token.provider.class": spark.conf.get("spark.databricks.passthrough.adls.gen2.tokenProviderClassName") 22 | } 23 | 24 | # Optionally, you can add to the source URI of your mount point. 25 | dbutils.fs.mount( 26 | source = "abfss://silver@gen2e2ede.dfs.core.windows.net/", 27 | mount_point = "/mnt/silver", 28 | extra_configs = configs) 29 | 30 | # COMMAND ---------- 31 | 32 | # Mount Azure Data Lake Gen2 Storage (Gold layer) to Databricks File System (DBFS) using credential passthrough 33 | 34 | configs = { 35 | "fs.azure.account.auth.type": "CustomAccessToken", 36 | "fs.azure.account.custom.token.provider.class": spark.conf.get("spark.databricks.passthrough.adls.gen2.tokenProviderClassName") 37 | } 38 | 39 | # Optionally, you can add to the source URI of your mount point. 40 | dbutils.fs.mount( 41 | source = "abfss://gold@gen2e2ede.dfs.core.windows.net/", 42 | mount_point = "/mnt/gold", 43 | extra_configs = configs) 44 | 45 | # COMMAND ---------- 46 | 47 | dbutils.fs.ls("/mnt/bronze/SalesLT/") 48 | 49 | # COMMAND ---------- 50 | 51 | 52 | -------------------------------------------------------------------------------- /key-vault/README.md: -------------------------------------------------------------------------------- 1 | # Azure Key Vault - (kv-e2e-de) 2 | 3 | I used key vault to store the login information for the on-prem SQL Server Database and for my Databricks workspace token. 4 | 5 | ![keyvault](https://github.com/user-attachments/assets/b71ef9e8-56b6-4e5e-9851-02f62641f992) 6 | -------------------------------------------------------------------------------- /power-bi-dashboard/README.md: -------------------------------------------------------------------------------- 1 | # Power BI - (pbi-e2e-de) 2 | 3 | This results from the ETL pipeline and offers an interactive reporting and analytics experience. 4 | 5 | Power BI Desktop loads the data from the "gold_db" table created in Azure Synapse Analytics. 6 | 7 | Why use the Azure Synapse database as the source VS the Gold container in the Data Lake? 8 | - It's an established best practice to use a database as the serving layer for Power BI reporting. 9 | - The Synapse database offers greater scalability and easier maintenance than reporting from the Data Lake. 10 | 11 | The Desktop version is included in this folder to download and experiment with the visualizations. The dashboard is also published to the Power BI Service (and Microsoft Fabric) but cannot be accessed by users outside my organization. 12 | 13 | ![pbi-dash](https://github.com/user-attachments/assets/571f0eb7-e412-4fb0-a0ef-22acb5b2f13e) 14 | -------------------------------------------------------------------------------- /power-bi-dashboard/pbi-e2e-de-dashboard.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexanderbean/E2E-Data-Engineering-in-Azure/18eb9786e187b830f942843f1df99e4679436b39/power-bi-dashboard/pbi-e2e-de-dashboard.pbix -------------------------------------------------------------------------------- /resource-group/ExportedTemplate-rg-e2e-de/parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "vulnerabilityAssessments_Default_storageContainerPath": { 6 | "value": null 7 | }, 8 | "vaults_kv_e2e_de_name": { 9 | "value": null 10 | }, 11 | "workspaces_synw_e2e_de_name": { 12 | "value": null 13 | }, 14 | "workspaces_dbw_e2e_de_name": { 15 | "value": null 16 | }, 17 | "storageAccounts_gen2e2ede_name": { 18 | "value": null 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /resource-group/README.md: -------------------------------------------------------------------------------- 1 | # Resource Group - rg-e2e-de 2 | 3 | This resource group hosted all of the services used in this project. 4 | 5 | The "ExportedTemplate" folder contains the ARM template for automation and easy deployment. In the next project, I intend to use Terraform as IaC. 6 | 7 | ![resource-group](https://github.com/user-attachments/assets/7c5bc29c-12ac-4ccc-b9d9-e77098e0216f) 8 | -------------------------------------------------------------------------------- /synapse-analytics/README.md: -------------------------------------------------------------------------------- 1 | # Azure Synapse Analytics - synw-e2e-de 2 | 3 | This is the final piece of the ETL pipeline in Azure. The "load" piece of the pipeline is done in Synapse. 4 | 5 | The pipeline below copies the transformed data from the Gold container and loads it into a serverless SQL database. 6 | 7 | ![synw-pipeline](https://github.com/user-attachments/assets/cd419b05-ab65-4def-a32f-169b18c4ed58) 8 | -------------------------------------------------------------------------------- /synapse-analytics/credential/WorkspaceSystemIdentity.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "WorkspaceSystemIdentity", 3 | "properties": { 4 | "type": "ManagedIdentity" 5 | } 6 | } -------------------------------------------------------------------------------- /synapse-analytics/dataset/goldtables.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "goldtables", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "synw-e2e-de-WorkspaceDefaultStorage", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Binary", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "folderPath": "SalesLT", 14 | "fileSystem": "gold" 15 | } 16 | } 17 | }, 18 | "type": "Microsoft.Synapse/workspaces/datasets" 19 | } -------------------------------------------------------------------------------- /synapse-analytics/integrationRuntime/AutoResolveIntegrationRuntime.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AutoResolveIntegrationRuntime", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "AutoResolve", 8 | "dataFlowProperties": { 9 | "computeType": "General", 10 | "coreCount": 8, 11 | "timeToLive": 0 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /synapse-analytics/linkedService/serverlessSQLdb.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "serverlessSQLdb", 3 | "type": "Microsoft.Synapse/workspaces/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureSqlDatabase", 7 | "typeProperties": { 8 | "connectionString": "Integrated Security=False;Encrypt=True;Connection Timeout=30;Data Source=synw-e2e-de-ondemand.sql.azuresynapse.net;Initial Catalog=gold_db" 9 | }, 10 | "connectVia": { 11 | "referenceName": "AutoResolveIntegrationRuntime", 12 | "type": "IntegrationRuntimeReference" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /synapse-analytics/linkedService/synw-e2e-de-WorkspaceDefaultSqlServer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "synw-e2e-de-WorkspaceDefaultSqlServer", 3 | "type": "Microsoft.Synapse/workspaces/linkedservices", 4 | "properties": { 5 | "typeProperties": { 6 | "connectionString": "Data Source=tcp:synw-e2e-de.sql.azuresynapse.net,1433;Initial Catalog=@{linkedService().DBName}" 7 | }, 8 | "parameters": { 9 | "DBName": { 10 | "type": "String" 11 | } 12 | }, 13 | "type": "AzureSqlDW", 14 | "connectVia": { 15 | "referenceName": "AutoResolveIntegrationRuntime", 16 | "type": "IntegrationRuntimeReference" 17 | }, 18 | "annotations": [] 19 | } 20 | } -------------------------------------------------------------------------------- /synapse-analytics/linkedService/synw-e2e-de-WorkspaceDefaultStorage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "synw-e2e-de-WorkspaceDefaultStorage", 3 | "type": "Microsoft.Synapse/workspaces/linkedservices", 4 | "properties": { 5 | "typeProperties": { 6 | "url": "https://gen2e2ede.dfs.core.windows.net" 7 | }, 8 | "type": "AzureBlobFS", 9 | "connectVia": { 10 | "referenceName": "AutoResolveIntegrationRuntime", 11 | "type": "IntegrationRuntimeReference" 12 | }, 13 | "annotations": [] 14 | } 15 | } -------------------------------------------------------------------------------- /synapse-analytics/pipeline/Create view.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Create view", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Get Tablenames", 7 | "type": "GetMetadata", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "0.12:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataset": { 19 | "referenceName": "goldtables", 20 | "type": "DatasetReference" 21 | }, 22 | "fieldList": [ 23 | "childItems" 24 | ], 25 | "storeSettings": { 26 | "type": "AzureBlobFSReadSettings", 27 | "recursive": true, 28 | "enablePartitionDiscovery": false 29 | }, 30 | "formatSettings": { 31 | "type": "BinaryReadSettings" 32 | } 33 | } 34 | }, 35 | { 36 | "name": "For each table name", 37 | "type": "ForEach", 38 | "dependsOn": [ 39 | { 40 | "activity": "Get Tablenames", 41 | "dependencyConditions": [ 42 | "Succeeded" 43 | ] 44 | } 45 | ], 46 | "userProperties": [], 47 | "typeProperties": { 48 | "items": { 49 | "value": "@activity('Get Tablenames').output.childItems", 50 | "type": "Expression" 51 | }, 52 | "activities": [ 53 | { 54 | "name": "Stored procedure1", 55 | "type": "SqlServerStoredProcedure", 56 | "dependsOn": [], 57 | "policy": { 58 | "timeout": "0.12:00:00", 59 | "retry": 0, 60 | "retryIntervalInSeconds": 30, 61 | "secureOutput": false, 62 | "secureInput": false 63 | }, 64 | "userProperties": [], 65 | "typeProperties": { 66 | "storedProcedureName": "[dbo].[CreateSQLServerlessView_gold]", 67 | "storedProcedureParameters": { 68 | "ViewName": { 69 | "value": { 70 | "value": "@item().name", 71 | "type": "Expression" 72 | }, 73 | "type": "String" 74 | } 75 | } 76 | }, 77 | "linkedServiceName": { 78 | "referenceName": "serverlessSQLdb", 79 | "type": "LinkedServiceReference" 80 | } 81 | } 82 | ] 83 | } 84 | } 85 | ], 86 | "annotations": [], 87 | "lastPublishTime": "2024-07-15T15:27:55Z" 88 | }, 89 | "type": "Microsoft.Synapse/workspaces/pipelines" 90 | } -------------------------------------------------------------------------------- /synapse-analytics/publish_config.json: -------------------------------------------------------------------------------- 1 | {"publishBranch":"workspace_publish","enableGitComment":true} -------------------------------------------------------------------------------- /synapse-analytics/sqlscript/sp_CreateSQLServerlessView_gold.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sp_CreateSQLServerlessView_gold", 3 | "properties": { 4 | "content": { 5 | "query": "USE gold_db\nGO\n\nCREATE OR ALTER PROC CreateSQLServerlessView_gold @ViewName nvarchar(100)\nAS \nBEGIN\n\nDECLARE @statement VARCHAR(MAX)\n\n SET @statement = N'CREATE OR ALTER VIEW ' + @ViewName + ' AS\n SELECT *\n FROM\n OPENROWSET(\n BULK ''https://gen2e2ede.dfs.core.windows.net/gold/SalesLT/' + @ViewName + '/'',\n FORMAT = ''DELTA''\n ) as [result]\n '\n\n\nEXEC (@statement)\n\nEND\nGO", 6 | "metadata": { 7 | "language": "sql" 8 | }, 9 | "currentConnection": { 10 | "databaseName": "gold_db", 11 | "poolName": "Built-in" 12 | }, 13 | "resultLimit": 5000 14 | }, 15 | "type": "SqlQuery" 16 | } 17 | } --------------------------------------------------------------------------------