├── Setup └── Files │ ├── Workspace │ └── AWDataPipelines.txt │ ├── Warehouse │ └── DWTableOrder.txt │ ├── AW │ ├── Scenario.csv │ ├── AccountRange.xlsx │ ├── AccountRangeRules.xlsx │ ├── DepartmentGroup.csv │ ├── Organization.csv │ └── Currency.csv │ ├── Lakehouse │ ├── AW_Backup.zip │ ├── AW_landing.zip │ ├── AW_tables.zip │ └── Pipelines.parquet │ └── SQLDatabase │ └── MetaTableOrder.txt ├── Workspaces └── DWA │ ├── DW.Warehouse │ ├── aw │ │ ├── aw.sql │ │ ├── Tables │ │ │ ├── ReportAccountMap.sql │ │ │ ├── DimScenario.sql │ │ │ ├── DimDepartmentGroup.sql │ │ │ ├── DimCurrency.sql │ │ │ ├── FactFinance.sql │ │ │ ├── DimOrganization.sql │ │ │ ├── DimAccount.sql │ │ │ └── DimDate.sql │ │ └── StoredProcedures │ │ │ └── usp_AccountRangeMapCreate.sql │ ├── dwa │ │ ├── dwa.sql │ │ └── StoredProcedures │ │ │ ├── usp_TableDrop.sql │ │ │ ├── usp_PipelinePostExecute.sql │ │ │ ├── usp_TableLoad_PrestageView.sql │ │ │ ├── usp_TableSwap.sql │ │ │ ├── usp_TableLoad_Update.sql │ │ │ ├── usp_TableLoad_CTAS.sql │ │ │ ├── usp_TableLoad_Delete.sql │ │ │ ├── usp_TableLoad_Insert.sql │ │ │ ├── usp_TableLoad_CheckSchemaDrift.sql │ │ │ ├── usp_TableLoad_DeploySchemaDrift.sql │ │ │ ├── usp_TableCreate.sql │ │ │ └── usp_TableCreateColumns.sql │ ├── int │ │ ├── int.sql │ │ └── StoredProcedures │ │ │ ├── usp_SettingEndDate.sql │ │ │ └── usp_LoadNumbers.sql │ ├── stg │ │ ├── stg.sql │ │ └── Tables │ │ │ └── DimAccount_staging.sql │ ├── aw_int │ │ ├── aw_int.sql │ │ ├── Tables │ │ │ └── Account_Staging.sql │ │ └── Views │ │ │ ├── AccountRangeRules.sql │ │ │ ├── Scenario.sql │ │ │ ├── DepartmentGroup.sql │ │ │ ├── AccountRange.sql │ │ │ ├── Currency.sql │ │ │ ├── Finance.sql │ │ │ ├── Organization.sql │ │ │ ├── Date.sql │ │ │ └── Account.sql │ ├── reports │ │ ├── reports.sql │ │ └── Views │ │ │ ├── DimScenario.sql │ │ │ ├── ReportAccountMap.sql │ │ │ ├── FactFinance.sql │ │ │ ├── General Ledger.sql │ │ │ ├── DimDate.sql │ │ │ ├── FinancialReport.sql │ │ │ ├── Department.sql │ │ │ ├── DimDepartmentGroup.sql │ │ │ ├── Account.sql │ │ │ ├── DimAccount.sql │ │ │ └── DimOrganization.sql │ ├── DefaultSemanticModel.txt │ ├── .platform │ ├── DW.sqlproj │ └── dbo │ │ └── StoredProcedures │ │ └── sp_whoisactive.sql │ ├── Readme.md │ ├── LH.Lakehouse │ ├── lakehouse.metadata.json │ ├── .platform │ └── shortcuts.metadata.json │ ├── Finance-GL.SemanticModel │ ├── definition │ │ ├── database.tmdl │ │ ├── cultures │ │ │ └── en-IE.tmdl │ │ ├── model.tmdl │ │ ├── relationships.tmdl │ │ └── tables │ │ │ ├── Scenario.tmdl │ │ │ ├── ReportAccountMap.tmdl │ │ │ ├── Date.tmdl │ │ │ ├── Department.tmdl │ │ │ ├── Account.tmdl │ │ │ └── Organization.tmdl │ ├── definition.pbism │ ├── .pbi │ │ └── editorSettings.json │ └── .platform │ ├── Meta.SQLDatabase │ ├── Security │ │ ├── audit.sql │ │ ├── config.sql │ │ ├── devops.sql │ │ └── bob@prodata.ie.sql │ ├── config │ │ ├── Tables │ │ │ ├── PipelineMetaCache.sql │ │ │ ├── IdentityMethods.sql │ │ │ ├── edwTableLineage.sql │ │ │ ├── PackageGroupTables.sql │ │ │ ├── PackageGroups.sql │ │ │ ├── Datasets.sql │ │ │ ├── PackageGroupPipelines.sql │ │ │ ├── PackageGroupLinks.sql │ │ │ ├── PipelineTables.sql │ │ │ ├── DatasetLineage.sql │ │ │ ├── Configurations.sql │ │ │ ├── Templates.sql │ │ │ ├── edwTableJoins.sql │ │ │ ├── cdcSqlTables.sql │ │ │ ├── PipelineMeta.sql │ │ │ ├── aiPrompts.sql │ │ │ ├── PipelineGroups.sql │ │ │ ├── edwTables.sql │ │ │ └── Pipelines.sql │ │ ├── Views │ │ │ └── Artefacts.sql │ │ └── StoredProcedures │ │ │ ├── usp_GetWorkspaces.sql │ │ │ ├── usp_OpsDatasets.sql │ │ │ ├── usp_cdcSqlCommit.sql │ │ │ ├── usp_cdcSqlTables.sql │ │ │ ├── usp_GetTables.sql │ │ │ ├── usp_cdcSqlBegin.sql │ │ │ ├── usp_TruncateAll.sql │ │ │ ├── usp_PipelineSequence.sql │ │ │ └── usp_PipelineQueue.sql │ ├── .platform │ ├── dbo │ │ ├── Tables │ │ │ └── dict_artefacts.sql │ │ └── Functions │ │ │ └── ufn_BuildChatPayloadJson.sql │ ├── audit │ │ ├── StoredProcedures │ │ │ ├── usp_PipelineEndFailure.sql │ │ │ ├── usp_PipelineEndSuccess.sql │ │ │ ├── usp_PipelineLogCopy.sql │ │ │ └── usp_PipelineStart.sql │ │ └── Tables │ │ │ ├── ReleaseLog.sql │ │ │ └── PipelineLog.sql │ └── Meta.sqlproj │ ├── Finance-GL.Report │ ├── definition.pbir │ ├── .platform │ └── CustomVisuals │ │ └── powerKPIMatrixEB2381CC88A8425FBEB1B07FF57784E6 │ │ └── package.json │ ├── Jobs │ ├── SCH_Ops.DataPipeline │ │ ├── .platform │ │ └── pipeline-content.json │ ├── SCH_ConfigBackup.DataPipeline │ │ ├── .platform │ │ └── pipeline-content.json │ └── SCH_AdventureWorks.DataPipeline │ │ ├── .platform │ │ └── pipeline-content.json │ └── DWA │ ├── Pipeline-Worker.DataPipeline │ └── .platform │ ├── Library │ ├── Extract │ │ ├── Extract-SQL.DataPipeline │ │ │ └── .platform │ │ ├── Extract-CSV.Notebook │ │ │ └── .platform │ │ ├── Extract-XML.Notebook │ │ │ └── .platform │ │ ├── Extract-Excel.Notebook │ │ │ └── .platform │ │ ├── Extract-JSON.Notebook │ │ │ ├── .platform │ │ │ └── notebook-content.py │ │ ├── Extract-O365-API.Notebook │ │ │ └── .platform │ │ ├── Extract-SP-Excel.Notebook │ │ │ └── .platform │ │ ├── Extract-CSV-Pandas.Notebook │ │ │ └── .platform │ │ └── Extract-Parquet-CDC.Notebook │ │ │ └── .platform │ ├── Refresh │ │ ├── Refresh-PBI.DataPipeline │ │ │ ├── .platform │ │ │ └── pipeline-content.json │ │ └── Refresh-Fabric.Notebook │ │ │ ├── .platform │ │ │ └── notebook-content.py │ ├── Ingest │ │ ├── Ingest-SQL-CDC.DataPipeline │ │ │ └── .platform │ │ ├── Copy-SFTP-Prodata.DataPipeline │ │ │ ├── .platform │ │ │ └── pipeline-content.json │ │ ├── Copy-Blob.Notebook │ │ │ └── .platform │ │ ├── Ingest-SP.Notebook │ │ │ └── .platform │ │ └── Ingest-SFTP.Notebook │ │ │ └── .platform │ ├── Export │ │ ├── Export-SP.Notebook │ │ │ └── .platform │ │ ├── Export-CSV.Notebook │ │ │ └── .platform │ │ ├── Export-SQL.Notebook │ │ │ └── .platform │ │ ├── Export-Excel.Notebook │ │ │ ├── .platform │ │ │ └── notebook-content.py │ │ └── Export-Parquet.Notebook │ │ │ └── .platform │ ├── Shared │ │ ├── MD-Sync.Notebook │ │ │ ├── .platform │ │ │ └── notebook-content.py │ │ ├── MD-Sync Python.Notebook │ │ │ ├── .platform │ │ │ └── notebook-content.py │ │ ├── SharePoint-Shared-Functions.Notebook │ │ │ └── .platform │ │ └── SQL-Connection-Shared-Functions.Notebook │ │ │ ├── .platform │ │ │ └── notebook-content.py │ ├── AI │ │ ├── Extract-AI-LH.Notebook │ │ │ └── .platform │ │ ├── Convert-PDF-Markdown.Notebook │ │ │ └── .platform │ │ └── Extract-Markdown-EH.Notebook │ │ │ └── .platform │ └── Transform │ │ └── Transform-LH-DuckDB.Notebook │ │ └── .platform │ ├── Pipeline-Controller.DataPipeline │ └── .platform │ ├── Config │ ├── Backup_ConfigSchema.DataPipeline │ │ └── .platform │ ├── Environment-Variables.DataPipeline │ │ ├── .platform │ │ └── pipeline-content.json │ └── Restore_ConfigSchema.DataPipeline │ │ └── .platform │ ├── ArchiveFiles.Notebook │ ├── .platform │ └── notebook-content.py │ └── Ops │ ├── Extract-Artefacts.Notebook │ ├── .platform │ └── notebook-content.py │ ├── Extract-Dictionary.Notebook │ └── .platform │ └── Extract-Fabric-Logs.Notebook │ ├── .platform │ └── notebook-content.py ├── CODEOWNERS ├── Reports └── Finance-GL.pbix ├── LICENSE └── README.md /Setup/Files/Workspace/AWDataPipelines.txt: -------------------------------------------------------------------------------- 1 | Copy-SFTP-Prodata -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw/aw.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA aw; -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/dwa.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA dwa; -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/int/int.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA int; -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/stg/stg.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA stg; -------------------------------------------------------------------------------- /Setup/Files/Warehouse/DWTableOrder.txt: -------------------------------------------------------------------------------- 1 | [aw_int].[AccountRange] -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw_int/aw_int.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA aw_int; -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/reports/reports.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA reports; -------------------------------------------------------------------------------- /Workspaces/DWA/Readme.md: -------------------------------------------------------------------------------- 1 | This is an auto-created file for /Workspaces/DWA -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/DefaultSemanticModel.txt: -------------------------------------------------------------------------------- 1 | Has default semantic model -------------------------------------------------------------------------------- /Workspaces/DWA/LH.Lakehouse/lakehouse.metadata.json: -------------------------------------------------------------------------------- 1 | {"defaultSchema":"dbo"} -------------------------------------------------------------------------------- /Setup/Files/AW/Scenario.csv: -------------------------------------------------------------------------------- 1 | ScenarioName 2 | Actual 3 | Budget 4 | Forecast 5 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Require review from the Maintainers team for any change 2 | * @your-org/DWA_Owner -------------------------------------------------------------------------------- /Reports/Finance-GL.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProdataSQL/DWA/main/Reports/Finance-GL.pbix -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/definition/database.tmdl: -------------------------------------------------------------------------------- 1 | database 2 | compatibilityLevel: 1567 3 | 4 | -------------------------------------------------------------------------------- /Setup/Files/AW/AccountRange.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProdataSQL/DWA/main/Setup/Files/AW/AccountRange.xlsx -------------------------------------------------------------------------------- /Setup/Files/Lakehouse/AW_Backup.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProdataSQL/DWA/main/Setup/Files/Lakehouse/AW_Backup.zip -------------------------------------------------------------------------------- /Setup/Files/Lakehouse/AW_landing.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProdataSQL/DWA/main/Setup/Files/Lakehouse/AW_landing.zip -------------------------------------------------------------------------------- /Setup/Files/Lakehouse/AW_tables.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProdataSQL/DWA/main/Setup/Files/Lakehouse/AW_tables.zip -------------------------------------------------------------------------------- /Setup/Files/AW/AccountRangeRules.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProdataSQL/DWA/main/Setup/Files/AW/AccountRangeRules.xlsx -------------------------------------------------------------------------------- /Setup/Files/Lakehouse/Pipelines.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProdataSQL/DWA/main/Setup/Files/Lakehouse/Pipelines.parquet -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/Security/audit.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA [audit] 2 | AUTHORIZATION [dbo]; 3 | 4 | 5 | GO 6 | 7 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/Security/config.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA [config] 2 | AUTHORIZATION [dbo]; 3 | 4 | 5 | GO 6 | 7 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/Security/devops.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA [devops] 2 | AUTHORIZATION [dbo]; 3 | 4 | 5 | GO 6 | 7 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/Security/bob@prodata.ie.sql: -------------------------------------------------------------------------------- 1 | CREATE USER [bob@prodata.ie] 2 | WITH SID = 0x727BA36D154EAB45B1FEA7959282523D, TYPE = E; 3 | 4 | 5 | GO 6 | 7 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/PipelineMetaCache.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[PipelineMetaCache] ( 2 | [ChecksumToken] INT NULL 3 | ); 4 | 5 | 6 | GO 7 | 8 | -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.Report/definition.pbir: -------------------------------------------------------------------------------- 1 | { 2 | "version": "4.0", 3 | "datasetReference": { 4 | "byPath": { 5 | "path": "../Finance-GL.SemanticModel" 6 | } 7 | } 8 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw/Tables/ReportAccountMap.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [aw].[ReportAccountMap] ( 2 | 3 | [ReportNo] bigint NULL, 4 | [Report] varchar(8000) NULL, 5 | [AccountKey] varchar(16) NULL 6 | ); -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/definition/cultures/en-IE.tmdl: -------------------------------------------------------------------------------- 1 | cultureInfo en-IE 2 | 3 | linguisticMetadata = 4 | { 5 | "Version": "1.0.0", 6 | "Language": "en-US" 7 | } 8 | contentType: json 9 | 10 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw/Tables/DimScenario.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [aw].[DimScenario] ( 2 | 3 | [ScenarioKey] varchar(16) NOT NULL, 4 | [ScenarioName] varchar(50) NOT NULL, 5 | [FileName] varchar(512) NULL, 6 | [LineageKey] varchar(36) NOT NULL 7 | ); -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/definition.pbism: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/semanticModel/definitionProperties/1.0.0/schema.json", 3 | "version": "4.2", 4 | "settings": { 5 | "qnaEnabled": false 6 | } 7 | } -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/.pbi/editorSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/semanticModel/editorSettings/1.0.0/schema.json", 3 | "parallelQueryLoading": true, 4 | "shouldNotifyUserOfNameConflictResolution": true 5 | } -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Views/Artefacts.sql: -------------------------------------------------------------------------------- 1 | 2 | CREATE View [config].[Artefacts] AS 3 | SELECT display_name as artefact, id as artefact_id, workspace_id , description 4 | FROM [dbo].[dict_artefacts] 5 | WHERE type in ('Notebook', 'DataPipeline') 6 | 7 | GO 8 | 9 | -------------------------------------------------------------------------------- /Setup/Files/AW/DepartmentGroup.csv: -------------------------------------------------------------------------------- 1 | DepartmentGroupName,ParentDepartmentGroupName 2 | Corporate, 3 | Executive General and Administration,Corporate 4 | Inventory Management,Corporate 5 | Manufacturing,Corporate 6 | Quality Assurance,Corporate 7 | Research and Development,Corporate 8 | Sales and Marketing,Corporate 9 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw/Tables/DimDepartmentGroup.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [aw].[DimDepartmentGroup] ( 2 | 3 | [DepartmentGroupKey] varchar(16) NULL, 4 | [ParentDepartmentGroupKey] varchar(16) NULL, 5 | [DepartmentGroupName] varchar(50) NOT NULL, 6 | [FileName] varchar(512) NULL, 7 | [LineageKey] varchar(36) NOT NULL, 8 | [RowChecksum] int NOT NULL 9 | ); -------------------------------------------------------------------------------- /Workspaces/DWA/LH.Lakehouse/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Lakehouse", 5 | "displayName": "LH" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "54a6a75f-8983-ae4c-406d-59d7d58f4f2d" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw/Tables/DimCurrency.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [aw].[DimCurrency] ( 2 | 3 | [CurrencyKey] varchar(16) NULL, 4 | [CurrencyAlternateKey] char(3) NOT NULL, 5 | [CurrencyName] varchar(50) NOT NULL, 6 | [RowVersionNo] bigint NULL, 7 | [RowChecksum] bigint NOT NULL, 8 | [FileName] varchar(512) NULL, 9 | [LineageKey] varchar(36) NOT NULL 10 | ); -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.Report/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Report", 5 | "displayName": "Finance-GL" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "a6140a2b-035d-92ef-42ff-8a475eb8480c" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "SQLDatabase", 5 | "displayName": "Meta" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "77526212-2be8-973a-4f64-af27fe70c606" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/IdentityMethods.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[IdentityMethods] ( 2 | [IdentityMethod] VARCHAR (20) NOT NULL, 3 | [Expression] VARCHAR (8000) NULL, 4 | [Description] VARCHAR (8000) NULL, 5 | CONSTRAINT [PK_IdentityMethods] PRIMARY KEY CLUSTERED ([IdentityMethod] ASC) 6 | ); 7 | 8 | 9 | GO 10 | 11 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/edwTableLineage.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[edwTableLineage] ( 2 | [TableID] INT NULL, 3 | [SourceTable] VARCHAR (128) NULL, 4 | [SourceType] VARCHAR (128) NULL, 5 | [SourceDatabase] VARCHAR (128) NULL, 6 | [PipelineID] INT NULL 7 | ); 8 | 9 | 10 | GO 11 | 12 | -------------------------------------------------------------------------------- /Workspaces/DWA/Jobs/SCH_Ops.DataPipeline/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "DataPipeline", 5 | "displayName": "SCH_Ops" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "11f7b8cb-cf58-a297-4522-5fb8bbef03cc" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "SemanticModel", 5 | "displayName": "Finance-GL" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "18216e7f-4e08-802e-40a7-49ba68e3a1d9" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Pipeline-Worker.DataPipeline/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "DataPipeline", 5 | "displayName": "Pipeline-Worker" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "962001bc-df81-af26-42a2-1f599ef1dad6" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/Jobs/SCH_ConfigBackup.DataPipeline/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "DataPipeline", 5 | "displayName": "SCH_ConfigBackup" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "ec6eb6b4-e10a-87fe-49ea-70bcbd999a6a" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/LH.Lakehouse/shortcuts.metadata.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "unittest", 4 | "path": "/Files", 5 | "target": { 6 | "type": "OneLake", 7 | "oneLake": { 8 | "path": "Files/unittest", 9 | "itemId": "f2f9c5fa-ca0c-41b2-b0e1-3028165b4f6c", 10 | "workspaceId": "9b8a6500-5ccb-49a9-885b-b5b081efed75" 11 | } 12 | } 13 | } 14 | ] -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/dbo/Tables/dict_artefacts.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [dbo].[dict_artefacts] ( 2 | [id] VARCHAR (MAX) NULL, 3 | [display_name] VARCHAR (MAX) NULL, 4 | [description] VARCHAR (MAX) NULL, 5 | [type] VARCHAR (MAX) NULL, 6 | [workspace_id] VARCHAR (MAX) NULL, 7 | [folder_id] VARCHAR (MAX) NULL 8 | ); 9 | 10 | 11 | GO 12 | 13 | -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Extract/Extract-SQL.DataPipeline/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "DataPipeline", 5 | "displayName": "Extract-SQL" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "1ef560ab-7329-b681-413e-531d3e1cfc3c" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Refresh/Refresh-PBI.DataPipeline/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "DataPipeline", 5 | "displayName": "Refresh-PBI" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "af5d26bb-0fd5-ba1c-48ba-b752643ba4a3" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Pipeline-Controller.DataPipeline/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "DataPipeline", 5 | "displayName": "Pipeline-Controller" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "1967f137-61fb-bc8c-4f29-d84704ff05ff" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/Jobs/SCH_AdventureWorks.DataPipeline/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "DataPipeline", 5 | "displayName": "SCH_AdventureWorks" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "32694129-8498-8a48-46c5-bbaeb09d979d" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Config/Backup_ConfigSchema.DataPipeline/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "DataPipeline", 5 | "displayName": "Backup_ConfigSchema" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "4f2f3edf-f172-aca8-4aa2-72a81aefde0f" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Ingest/Ingest-SQL-CDC.DataPipeline/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "DataPipeline", 5 | "displayName": "Ingest-SQL-CDC" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "f63cc462-5a7f-8e09-431a-9b762b6c88f9" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Warehouse", 5 | "displayName": "DW", 6 | "description": "DWA Enterprise Datawarehouse" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "e2ce0200-2243-81be-4799-b337c7786002" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Config/Environment-Variables.DataPipeline/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "DataPipeline", 5 | "displayName": "Environment-Variables" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "0b161fb6-be93-ad8e-48c7-d3f479d6132a" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Config/Restore_ConfigSchema.DataPipeline/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "DataPipeline", 5 | "displayName": "Restore_ConfigSchema" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "19edbb93-106d-b476-419b-4bfa8c505456" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Ingest/Copy-SFTP-Prodata.DataPipeline/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "DataPipeline", 5 | "displayName": "Copy-SFTP-Prodata" 6 | }, 7 | "config": { 8 | "version": "2.0", 9 | "logicalId": "a7b2734a-00db-ad99-4260-66a1aab1d2be" 10 | } 11 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/reports/Views/DimScenario.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) 27414BC6C33945089BCED8DA8B1A9FC9C93B1B09551C07CB65B97DEFC91EC5E6 2 | 3 | 4 | 5 | /* Description: AW DimDepartmentGroup for PowerBI Model 6 | 7 | History: 8 | 21/02/2025 Created 9 | */ 10 | CREATE VIEW [reports].[DimScenario] 11 | AS 12 | SELECT ScenarioKey 13 | ,ScenarioName AS Scenario 14 | FROM aw.DimScenario -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/ArchiveFiles.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "ArchiveFiles", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "083d8859-3005-94e8-423d-00d06f9d2aab" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/reports/Views/ReportAccountMap.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) 0B47FB90D304D753A60855DECE6ED285B50311703E442CE0ABCA95F6E816E4E2 2 | 3 | 4 | 5 | 6 | /* Description: AW DimDepartmentGroup for PowerBI Model 7 | 8 | History: 9 | 21/02/2025 Created 10 | */ 11 | CREATE VIEW [reports].[ReportAccountMap] 12 | AS 13 | SELECT ReportNo, Report, AccountKey 14 | FROM aw.ReportAccountMap -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Export/Export-SP.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Export-SP", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "c9670eca-f19a-aaf3-4b86-8fd6ae9895bd" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Ingest/Copy-Blob.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Copy-Blob", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "a1b3fab4-a32e-b1ab-4922-0f091cc198e8" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Ingest/Ingest-SP.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Ingest-SP", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "293ede4b-e9a9-852d-4c26-49787dd54399" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Shared/MD-Sync.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "MD-Sync", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "7efd9cba-12a1-9d6a-4cb5-906086438946" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/AI/Extract-AI-LH.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-AI-LH", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "75c9fbd5-b62e-9367-4e6c-0a1c0c860360" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Export/Export-CSV.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Export-CSV", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "269a1c80-ee7a-aaf4-4884-e9be6003b87a" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Export/Export-SQL.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Export-SQL", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "fce32c02-c4cd-bf07-474a-8609ff5cf3bc" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Extract/Extract-CSV.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-CSV", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "7c159d39-545c-a974-499e-6f35b2790875" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Extract/Extract-XML.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-XML", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "c904f505-876a-88b7-443a-5bf38ac31689" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Ingest/Ingest-SFTP.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Ingest-SFTP", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "ae2df5c7-02cc-94fb-4f5a-b864357cc2e2" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Ops/Extract-Artefacts.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-Artefacts", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "5a82bb55-610e-aade-4985-34200afe9e50" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Export/Export-Excel.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Export-Excel", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "d8445908-7972-a302-4cf7-33cc7d0acd9b" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Export/Export-Parquet.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Export-Parquet", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "f1793084-7345-ad2c-4812-4cebc0ba42d6" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Extract/Extract-Excel.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-Excel", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "9cb3eb44-3861-82c7-42cc-bdea8ab0a766" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Extract/Extract-JSON.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-JSON", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "782e7673-6a31-8079-4389-42415f032bad" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Shared/MD-Sync Python.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "MD-Sync Python", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "32ebf2c7-8a1b-ad04-4037-b7a2ff65f8b9" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Ops/Extract-Dictionary.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-Dictionary", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "59e69e78-d8a2-ac90-43c3-8ca0aca2d112" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Ops/Extract-Fabric-Logs.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-Fabric-Logs", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "ec39ef46-61f5-b799-4398-a9cc1e0d01cb" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw/Tables/FactFinance.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [aw].[FactFinance] ( 2 | 3 | [DateKey] varchar(8) NOT NULL, 4 | [DepartmentGroupKey] varchar(16) NULL, 5 | [ScenarioKey] varchar(16) NOT NULL, 6 | [OrganizationKey] varchar(16) NOT NULL, 7 | [AccountKey] varchar(16) NULL, 8 | [Date] varchar(8000) NULL, 9 | [Amount] decimal(38,6) NULL, 10 | [FileName] varchar(512) NULL, 11 | [LineageKey] varchar(36) NOT NULL 12 | ); -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Extract/Extract-O365-API.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-O365-API", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "ca71f62b-2dbb-9178-4ad4-51560a6404cf" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Extract/Extract-SP-Excel.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-SP-Excel", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "97ced11a-8d27-8806-4865-77ab8d913015" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Refresh/Refresh-Fabric.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Refresh-Fabric", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "0bb49457-d28f-ac09-481b-15ab2c2551e3" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw/Tables/DimOrganization.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [aw].[DimOrganization] ( 2 | 3 | [OrganizationKey] varchar(16) NOT NULL, 4 | [ParentOrganizationKey] varchar(16) NULL, 5 | [PercentageOfOwnership] varchar(10) NOT NULL, 6 | [OrganizationName] varchar(50) NOT NULL, 7 | [CurrencyKey] varchar(16) NOT NULL, 8 | [RowChecksum] bigint NOT NULL, 9 | [FileName] varchar(512) NULL, 10 | [LineageKey] varchar(36) NOT NULL 11 | ); -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/AI/Convert-PDF-Markdown.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Convert-PDF-Markdown", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "7aff8440-4c95-800c-4346-f16211fe56c0" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/AI/Extract-Markdown-EH.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-Markdown-EH", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "65753566-9d24-9cd0-4920-6c4058a2fd1c" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Extract/Extract-CSV-Pandas.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-CSV-Pandas", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "4ae1547b-05fe-a67b-4ca4-816dccf11d86" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Extract/Extract-Parquet-CDC.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Extract-Parquet-CDC", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "65156325-29a7-b59f-4316-472c26f9e7d3" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Transform/Transform-LH-DuckDB.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "Transform-LH-DuckDB", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "c305da77-db0a-a81b-46f9-15529bdcfb0e" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Shared/SharePoint-Shared-Functions.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "SharePoint-Shared-Functions", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "1a5c04b6-0a85-b496-4e20-b7323329b232" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Shared/SQL-Connection-Shared-Functions.Notebook/.platform: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", 3 | "metadata": { 4 | "type": "Notebook", 5 | "displayName": "SQL-Connection-Shared-Functions", 6 | "description": "New notebook" 7 | }, 8 | "config": { 9 | "version": "2.0", 10 | "logicalId": "4d28b7db-9c96-b5d5-4c35-765dcacecce3" 11 | } 12 | } -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/StoredProcedures/usp_GetWorkspaces.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Return Workspaces 3 | */ 4 | CREATE PROCEDURE [config].[usp_GetWorkspaces] 5 | AS 6 | BEGIN 7 | SET NOCOUNT ON 8 | SELECT DISTINCT JSON_VALUE(lower([ConnectionSettings]), lower('$.workspaceid')) AS WorkspaceID 9 | FROM config.Configurations 10 | WHERE JSON_VALUE(lower([ConnectionSettings]), lower('$.workspaceid')) is not null 11 | END 12 | 13 | GO 14 | 15 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/DW.sqlproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | DW 5 | Microsoft.Data.Tools.Schema.Sql.SqlDwUnifiedDatabaseSchemaProvider 6 | Latin1_General_100_CI_AS_KS_WS_SC_UTF8 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/StoredProcedures/usp_OpsDatasets.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | /* 4 | 5 | */ 6 | CREATE PROCEDURE [config].[usp_OpsDatasets] 7 | AS 8 | BEGIN 9 | SET NOCOUNT ON 10 | 11 | SELECT Dataset, Monitored, MaxAgeHours, PackageGroup, workspace 12 | FROM config.Datasets ds 13 | INNER JOIN Configurations c on c.ConfigurationID =ds.ConfigurationID 14 | CROSS APPLY OPENJSON (c.ConnectionSettings) WITH ( 15 | workspace varchar(128) 16 | ) j 17 | WHERE c.[Enabled]=1 18 | END 19 | 20 | GO 21 | 22 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/StoredProcedures/usp_cdcSqlCommit.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Commit SQL Server CDC 3 | Test: 4 | exec config.usp_cdcSqlCommit 'TableHComment' 5 | History: 15/06/2025 Bob, Created 6 | */ 7 | CREATE PROCEDURE [config].[usp_cdcSqlCommit] 8 | ( 9 | @Table sysname 10 | 11 | ) 12 | AS 13 | BEGIN 14 | SET NOCOUNT ON 15 | 16 | UPDATE [config].[cdcSqlTables] 17 | SET max_lsn=target_lsn, max_datetime=target_datetime 18 | WHERE [Table] =@Table 19 | END 20 | 21 | GO 22 | 23 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/PackageGroupTables.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[PackageGroupTables] ( 2 | [PackageGroup] VARCHAR (50) NOT NULL, 3 | [TableID] SMALLINT NOT NULL, 4 | CONSTRAINT [PK_PackageGroupTables] PRIMARY KEY CLUSTERED ([PackageGroup] ASC, [TableID] ASC), 5 | CONSTRAINT [FK_PackageGroupTables_edwTables] FOREIGN KEY ([TableID]) REFERENCES [config].[edwTables] ([TableID]), 6 | CONSTRAINT [FK_PackageGroupTables_PackageGroups] FOREIGN KEY ([PackageGroup]) REFERENCES [config].[PackageGroups] ([PackageGroup]) 7 | ); 8 | 9 | 10 | GO 11 | 12 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw_int/Tables/Account_Staging.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [aw_int].[Account_Staging] ( 2 | 3 | [AccountKey] varchar(16) NULL, 4 | [ParentAccountKey] varchar(16) NULL, 5 | [AccountCodeAlternateKey] int NOT NULL, 6 | [ParentAccountCodeAlternateKey] int NULL, 7 | [AccountDescription] varchar(50) NOT NULL, 8 | [AccountType] varchar(50) NULL, 9 | [Operator] varchar(50) NOT NULL, 10 | [CustomMembers] varchar(50) NULL, 11 | [ValueType] varchar(50) NOT NULL, 12 | [CustomMemberOptions] varchar(200) NULL, 13 | [FileName] varchar(512) NULL, 14 | [LineageKey] varchar(36) NOT NULL 15 | ); -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2023 Prodata Business Systems Ltd 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /Setup/Files/SQLDatabase/MetaTableOrder.txt: -------------------------------------------------------------------------------- 1 | [dbo].[dict_artefacts] 2 | [config].[Configurations] 3 | [config].[IdentityMethods] 4 | [config].[Datasets] 5 | [config].[PackageGroups] 6 | [config].[edwTables] 7 | [config].[edwTableJoins] 8 | [config].[edwTableLineage] 9 | [config].[Templates] 10 | [config].[PackageGroupLinks] 11 | [config].[PipelineGroups] 12 | [config].[Pipelines] 13 | [config].[PipelineTables] 14 | [config].[PackageGroupPipelines] 15 | [config].[PackageGroupTables] 16 | [config].[DatasetLineage] 17 | [config].[cdcSqlTables] 18 | [audit].[PipelineLog] 19 | [audit].[ReleaseLog] 20 | [config].[PipelineMeta] 21 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/PackageGroups.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[PackageGroups] ( 2 | [PackageGroup] VARCHAR (50) NOT NULL, 3 | [Monitored] BIT NULL, 4 | [SortOrder] VARCHAR (50) NULL, 5 | CONSTRAINT [PK_PackageGroups] PRIMARY KEY NONCLUSTERED ([PackageGroup] ASC) 6 | ); 7 | 8 | 9 | GO 10 | 11 | 12 | 13 | CREATE TRIGGER [config].TR_PackageGroups 14 | ON [config].[PackageGroups] 15 | AFTER INSERT,DELETE,UPDATE 16 | AS 17 | BEGIN 18 | SET NOCOUNT ON; 19 | exec [config].[usp_PipelineBuildMetaData] 20 | 21 | END 22 | 23 | GO 24 | 25 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw/Tables/DimAccount.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [aw].[DimAccount] ( 2 | 3 | [AccountKey] varchar(16) NULL, 4 | [ParentAccountKey] varchar(16) NULL, 5 | [AccountCodeAlternateKey] int NOT NULL, 6 | [ParentAccountCodeAlternateKey] int NULL, 7 | [AccountDescription] varchar(50) NOT NULL, 8 | [AccountType] varchar(50) NULL, 9 | [Operator] varchar(50) NOT NULL, 10 | [CustomMembers] varchar(50) NULL, 11 | [ValueType] varchar(50) NOT NULL, 12 | [CustomMemberOptions] varchar(200) NULL, 13 | [RowChecksum] int NOT NULL, 14 | [FileName] varchar(512) NULL, 15 | [LineageKey] varchar(36) NOT NULL 16 | ); -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/Datasets.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[Datasets] ( 2 | [Dataset] VARCHAR (50) NOT NULL, 3 | [Monitored] BIT NULL, 4 | [MaxAgeHours] INT NULL, 5 | [PackageGroup] VARCHAR (50) NULL, 6 | [ConfigurationID] INT NULL, 7 | [DataOwner] VARCHAR (255) NULL, 8 | CONSTRAINT [PK_Datasets] PRIMARY KEY CLUSTERED ([Dataset] ASC), 9 | CONSTRAINT [FK_Datasets_Configurations] FOREIGN KEY ([ConfigurationID]) REFERENCES [config].[Configurations] ([ConfigurationID]) 10 | ); 11 | 12 | 13 | GO 14 | 15 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/PackageGroupPipelines.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[PackageGroupPipelines] ( 2 | [PackageGroup] VARCHAR (50) NOT NULL, 3 | [PipelineID] INT NOT NULL, 4 | CONSTRAINT [PK_PackageGroupPipelines] PRIMARY KEY CLUSTERED ([PackageGroup] ASC, [PipelineID] ASC), 5 | CONSTRAINT [FK_PackageGroupPipelines_PackageGroups] FOREIGN KEY ([PackageGroup]) REFERENCES [config].[PackageGroups] ([PackageGroup]), 6 | CONSTRAINT [FK_PackageGroupPipelines_Pipelines] FOREIGN KEY ([PipelineID]) REFERENCES [config].[Pipelines] ([PipelineID]) 7 | ); 8 | 9 | 10 | GO 11 | 12 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw_int/Views/AccountRangeRules.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) AEDD8CF50C356AD90C241D33C360C178B3596E958D85FB7C17AE49659262C0B3 2 | 3 | 4 | /* Description: AW Staging AccountRangeRules 5 | Example: EXEC dwa.usp_TableLoad NULL,4,NULL 6 | History: 7 | 19/02/2025 Created 8 | */ 9 | CREATE VIEW [aw_int].[AccountRangeRules] AS 10 | SELECT [ReportNo] 11 | ,[ReportSection] AS [ReportSection] 12 | ,[FromAccountNo] 13 | ,[ToAccountNo] 14 | ,[FileName] AS FileName 15 | ,ISNULL(CONVERT(VARCHAR(36),LineageKey),0) AS LineageKey 16 | FROM LH.aw_stg.[accountrangerules] -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/stg/Tables/DimAccount_staging.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [stg].[DimAccount_staging] ( 2 | 3 | [AccountKey] varchar(16) NULL, 4 | [ParentAccountKey] varchar(16) NULL, 5 | [AccountCodeAlternateKey] int NOT NULL, 6 | [ParentAccountCodeAlternateKey] int NULL, 7 | [AccountDescription] varchar(50) NOT NULL, 8 | [AccountType] varchar(50) NULL, 9 | [Operator] varchar(50) NOT NULL, 10 | [CustomMembers] varchar(50) NULL, 11 | [ValueType] varchar(50) NOT NULL, 12 | [CustomMemberOptions] varchar(200) NULL, 13 | [RowChecksum] int NOT NULL, 14 | [FileName] varchar(512) NULL, 15 | [LineageKey] varchar(36) NOT NULL 16 | ); -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/PackageGroupLinks.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[PackageGroupLinks] ( 2 | [PackageGroup] VARCHAR (50) NOT NULL, 3 | [ChildPackageGroup] VARCHAR (50) NOT NULL, 4 | CONSTRAINT [PK_PackageGroupLinks] PRIMARY KEY NONCLUSTERED ([PackageGroup] ASC, [ChildPackageGroup] ASC), 5 | CONSTRAINT [FK_PackageGroupLinks_PackageGroups] FOREIGN KEY ([PackageGroup]) REFERENCES [config].[PackageGroups] ([PackageGroup]), 6 | CONSTRAINT [FK_PackageGroupLinks_PackageGroups1] FOREIGN KEY ([ChildPackageGroup]) REFERENCES [config].[PackageGroups] ([PackageGroup]) 7 | ); 8 | 9 | 10 | GO 11 | 12 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw_int/Views/Scenario.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) 1C87B76C169AA25EDF6158BE6476F96E5DE06828D78A56F9FB2CC1F86EAE9F3B 2 | 3 | 4 | 5 | /* Description: AW Dimension Scenario 6 | Example: EXEC dwa.usp_TableLoad NULL,4,NULL 7 | History: 8 | 19/02/2025 Created 9 | */ 10 | CREATE VIEW [aw_int].[Scenario] AS 11 | SELECT ISNULL(CONVERT(VARCHAR(16),HASHBYTES('MD5', s.ScenarioName),2),'') AS ScenarioKey 12 | ,ISNULL(CONVERT(varchar(50),s.ScenarioName),'') AS ScenarioName 13 | , CONVERT(VARCHAR(512), s.[FileName]) AS FileName 14 | , ISNULL(CONVERT(VARCHAR(36), s.LineageKey), 0) AS LineageKey 15 | FROM LH.aw_stg.scenario s; -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw_int/Views/DepartmentGroup.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) 6CE16FD9F1944562989E5F6042F5D1043A1B173C762203F402DF3C5B9DEFF110 2 | 3 | 4 | 5 | 6 | CREATE VIEW [aw_int].[DepartmentGroup] 7 | AS 8 | SELECT CONVERT(VARCHAR(16),HASHBYTES('MD5', p.DepartmentGroupName),2) AS DepartmentGroupKey 9 | , CONVERT(VARCHAR(16),HASHBYTES('MD5', p.ParentDepartmentGroupName),2) AS ParentDepartmentGroupKey 10 | , ISNULL(CONVERT(VARCHAR(50), p.DepartmentGroupName ), '') AS DepartmentGroupName 11 | , CONVERT(VARCHAR(512), p.[FileName] ) AS FileName 12 | , ISNULL(CONVERT(VARCHAR(36),LineageKey ),0) AS LineageKey 13 | FROM LH.aw_stg.departmentgroup p -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/reports/Views/FactFinance.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) A4F44FC40CF68B5A9C4888A0A20C9C54B3026E6598F2F6AF63937934C1F594D2 2 | 3 | 4 | 5 | /* Description: AW DimDepartmentGroup for PowerBI Model 6 | 7 | History: 8 | 21/02/2025 Created 9 | */ 10 | CREATE VIEW [reports].[FactFinance] 11 | AS 12 | SELECT DateKey 13 | ,DepartmentGroupKey 14 | ,ScenarioKey 15 | ,OrganizationKey 16 | ,CONVERT(MONEY, Amount * CASE WHEN a.AccountType IN ('Expenditures', 'Liabilities') THEN -1 ELSE 1 END) AS BaseAmount 17 | ,f.AccountKey 18 | ,Date 19 | FROM aw.FactFinance f 20 | INNER JOIN aw.DimAccount a 21 | ON a.AccountKey = f.AccountKey -------------------------------------------------------------------------------- /Workspaces/DWA/Jobs/SCH_Ops.DataPipeline/pipeline-content.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": { 3 | "activities": [ 4 | { 5 | "type": "TridentNotebook", 6 | "typeProperties": { 7 | "notebookId": "5a82bb55-610e-aade-4985-34200afe9e50", 8 | "workspaceId": "00000000-0000-0000-0000-000000000000", 9 | "sessionTag": "ETL" 10 | }, 11 | "policy": { 12 | "timeout": "0.12:00:00", 13 | "retry": 0, 14 | "retryIntervalInSeconds": 30, 15 | "secureInput": false, 16 | "secureOutput": false 17 | }, 18 | "name": "Extract-Artefacts", 19 | "dependsOn": [] 20 | } 21 | ] 22 | } 23 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/reports/Views/General Ledger.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) A4F44FC40CF68B5A9C4888A0A20C9C54B3026E6598F2F6AF63937934C1F594D2 2 | 3 | 4 | 5 | 6 | /* Description: AW General Ledger for PowerBI Direct Lake Model 7 | 8 | History: 9 | 21/02/2025 Created 10 | */ 11 | CREATE VIEW [reports].[General Ledger] 12 | AS 13 | SELECT DateKey 14 | ,DepartmentGroupKey 15 | ,ScenarioKey 16 | ,OrganizationKey 17 | ,CONVERT(MONEY, Amount * CASE WHEN a.AccountType IN ('Expenditures', 'Liabilities') THEN -1 ELSE 1 END) AS BaseAmount 18 | ,f.AccountKey 19 | ,Date 20 | FROM aw.FactFinance f 21 | INNER JOIN aw.DimAccount a 22 | ON a.AccountKey = f.AccountKey -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/PipelineTables.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[PipelineTables] ( 2 | [PipelineID] INT NOT NULL, 3 | [TableID] SMALLINT NOT NULL, 4 | [TargetSchema] VARCHAR (128) NULL, 5 | [TargetTable] VARCHAR (128) NULL, 6 | [TargetDatabase] VARCHAR (128) NULL, 7 | CONSTRAINT [PK_PipelineTables] PRIMARY KEY NONCLUSTERED ([PipelineID] ASC, [TableID] ASC), 8 | CONSTRAINT [FK_PipelineTables_edwTables] FOREIGN KEY ([TableID]) REFERENCES [config].[edwTables] ([TableID]), 9 | CONSTRAINT [FK_PipelineTables_Pipelines] FOREIGN KEY ([PipelineID]) REFERENCES [config].[Pipelines] ([PipelineID]) 10 | ); 11 | 12 | 13 | GO 14 | 15 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/StoredProcedures/usp_TableDrop.sql: -------------------------------------------------------------------------------- 1 | /* Drop a Star Schema Table 2 | Example:[dwa].[usp_TableDrop] 'aw.DimDate' 3 | History: 4 | 18/08/2023 Deepak, Created 5 | */ 6 | CREATE PROCEDURE [dwa].[usp_TableDrop] @Table [sysname] 7 | AS 8 | BEGIN 9 | DECLARE @object_id INT 10 | DECLARE @sql NVARCHAR(4000) 11 | DECLARE @is_external BIT 12 | 13 | SELECT @object_id = object_id, @is_external = is_external 14 | FROM sys.tables 15 | WHERE object_id = object_id(@Table) 16 | 17 | IF @object_id IS NOT NULL 18 | BEGIN 19 | SET @sql = 'DROP ' + CASE WHEN @is_external = 1 THEN 'EXTERNAL ' ELSE '' END + 'TABLE ' + @Table + ';' 20 | PRINT @sql 21 | EXEC (@sql) 22 | END 23 | END -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/DatasetLineage.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[DatasetLineage] ( 2 | [Dataset] VARCHAR (50) NOT NULL, 3 | [DatasetTableName] VARCHAR (128) NOT NULL, 4 | [SourceDatabase] VARCHAR (128) NULL, 5 | [SourceObject] VARCHAR (255) NULL, 6 | [SourceType] VARCHAR (50) NULL, 7 | [SourceConnection] VARCHAR (255) NULL, 8 | [PipelineID] INT NULL, 9 | CONSTRAINT [FK_DatasetLineage_Datasets] FOREIGN KEY ([Dataset]) REFERENCES [config].[Datasets] ([Dataset]), 10 | CONSTRAINT [FK_DatasetLineage_Pipelines] FOREIGN KEY ([PipelineID]) REFERENCES [config].[Pipelines] ([PipelineID]) 11 | ); 12 | 13 | 14 | GO 15 | 16 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/reports/Views/DimDate.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) 886B20EE65965E88EA0F7A2ACE4F2448BC0050530018A009D29871EAE931728B 2 | 3 | 4 | 5 | /* Description: AW DimDepartmentGroup for PowerBI Model 6 | 7 | History: 8 | 21/02/2025 Created 9 | */ 10 | CREATE VIEW [reports].[DimDate] 11 | AS 12 | SELECT DateKey 13 | ,FullDateAlternateKey AS Date 14 | ,MonthName AS Month 15 | ,FiscalQuarter AS [Fiscal Quarter] 16 | ,FiscalYear AS [Fiscal Year] 17 | ,MonthNumberOfYear AS [Month No] 18 | ,FiscalYear * 100 + MonthNumberOfYear as [Fiscal Period No] 19 | ,CONVERT (varchar(4), FiscalYear) + '-' + format(MonthNumberOfYear,'00') as [Fiscal Period] 20 | FROM aw.DimDate -------------------------------------------------------------------------------- /Setup/Files/AW/Organization.csv: -------------------------------------------------------------------------------- 1 | OrganizationName,ParentOrganizationName,PercentageOfOwnership,CurrencyCode 2 | AdventureWorks Cycle,,1,USD 3 | North America Operations,AdventureWorks Cycle,1,USD 4 | Northeast Division,USA Operations,1,USD 5 | Northwest Division,USA Operations,1,USD 6 | Central Division,USA Operations,1,USD 7 | Southeast Division,USA Operations,1,USD 8 | Southwest Division,USA Operations,1,USD 9 | Canadian Division,North America Operations,0.75,CAD 10 | European Operations,AdventureWorks Cycle,1,EUR 11 | Pacific Operations,AdventureWorks Cycle,0.75,AUD 12 | France,European Operations,0.5,EUR 13 | Germany,European Operations,0.25,EUR 14 | Australia,Pacific Operations,0.5,AUD 15 | USA Operations,North America Operations,1,USD 16 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/reports/Views/FinancialReport.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) B657F59F0F5A0E30FD6BB145732616630A968CCA73A651989934E1B37DE99677 2 | 3 | 4 | 5 | 6 | 7 | /* Description: AW DimDepartmentGroup for PowerBI Model 8 | 9 | History: 10 | 21/02/2025 Created 11 | */ 12 | CREATE VIEW [reports].[FinancialReport] 13 | AS 14 | SELECT ReportNo AS [Report No] 15 | ,ReportHeading AS [Report Heading] 16 | ,ReportSection AS [Report Section] 17 | ,[LineNo] 18 | ,LinkLineNo 19 | ,LinkReport 20 | ,LinkHeading 21 | 22 | ,ReportHeadingNo 23 | ,COALESCE(LinkSignage, 1) AS [Link Signage] 24 | ,Operator 25 | ,Report 26 | ,Calc1 27 | FROM aw_int.AccountRange -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/Configurations.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[Configurations] ( 2 | [ConfigurationID] INT NOT NULL, 3 | [ConfigurationName] VARCHAR (8000) NULL, 4 | [ConnectionSettings] VARCHAR (8000) NULL, 5 | [Enabled] INT CONSTRAINT [DF_Configurations_Enabled] DEFAULT ((1)) NOT NULL, 6 | CONSTRAINT [PK_Configurations] PRIMARY KEY CLUSTERED ([ConfigurationID] ASC) 7 | ); 8 | 9 | 10 | GO 11 | 12 | 13 | CREATE TRIGGER [config].TR_Configurations 14 | ON [config].[Configurations] 15 | AFTER INSERT,DELETE,UPDATE 16 | AS 17 | BEGIN 18 | SET NOCOUNT ON; 19 | exec [config].[usp_PipelineBuildMetaData] 20 | 21 | END 22 | 23 | GO 24 | 25 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/Templates.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[Templates] ( 2 | [Template] VARCHAR (200) NOT NULL, 3 | [TemplateDescription] VARCHAR (500) NULL, 4 | [SourceSettings] VARCHAR (8000) NULL, 5 | [TargetSettings] VARCHAR (8000) NULL, 6 | [ArtefactType] VARCHAR (100) NOT NULL, 7 | [ConfigurationID] INT NOT NULL, 8 | CONSTRAINT [PK_Templates] PRIMARY KEY NONCLUSTERED ([Template] ASC) 9 | ); 10 | 11 | 12 | GO 13 | 14 | CREATE TRIGGER [config].TR_Templates 15 | ON config.Templates 16 | AFTER INSERT,DELETE,UPDATE 17 | AS 18 | BEGIN 19 | SET NOCOUNT ON; 20 | exec [config].[usp_PipelineBuildMetaData] 21 | 22 | END 23 | 24 | GO 25 | 26 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw/Tables/DimDate.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [aw].[DimDate] ( 2 | 3 | [DateKey] varchar(8) NOT NULL, 4 | [FullDateAlternateKey] date NOT NULL, 5 | [DayNumberOfWeek] smallint NOT NULL, 6 | [DayOfWeek] varchar(50) NOT NULL, 7 | [DayNumberOfMonth] smallint NOT NULL, 8 | [DayNumberOfYear] int NOT NULL, 9 | [WeekNumberOfYear] int NOT NULL, 10 | [MonthName] varchar(50) NOT NULL, 11 | [MonthNumberOfYear] smallint NOT NULL, 12 | [CalendarQuarter] smallint NOT NULL, 13 | [CalendarYear] smallint NOT NULL, 14 | [CalendarSemester] smallint NOT NULL, 15 | [FiscalQuarter] smallint NOT NULL, 16 | [FiscalYear] smallint NOT NULL, 17 | [FiscalSemester] smallint NOT NULL, 18 | [FileName] varchar(512) NULL, 19 | [LineageKey] varchar(36) NOT NULL 20 | ); -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.Report/CustomVisuals/powerKPIMatrixEB2381CC88A8425FBEB1B07FF57784E6/package.json: -------------------------------------------------------------------------------- 1 | {"version":"3.1.1","author":{"name":"Microsoft","email":"pbicvsupport@microsoft.com"},"resources":[{"resourceId":"rId0","sourceType":5,"file":"resources/powerKPIMatrixEB2381CC88A8425FBEB1B07FF57784E6.pbiviz.json"}],"visual":{"guid":"powerKPIMatrixEB2381CC88A8425FBEB1B07FF57784E6","name":"PowerKPIMatrix","displayName":"Power KPI Matrix 3.1.1","version":"3.1.1","description":"Monitor balanced scorecards and unlimited number of metrics and KPIs in a single, compact, easy to read list.","supportUrl":"https://aka.ms/customvisualscommunity","gitHubUrl":"https://github.com/Microsoft/PowerBI-visuals-PowerKPIMatrix","visualClassName":"PowerKPIMatrix"},"metadata":{"pbivizjson":{"resourceId":"rId0"}}} -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw_int/Views/AccountRange.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) EB08FB7937C620BB5B20F7C40807B1BA836CF0F66CE7588C4F83FE732E9C845A 2 | 3 | 4 | 5 | /* Description: AW Staging AccountRange 6 | Example: EXEC dwa.usp_TableLoad NULL,4,NULL 7 | History: 8 | 20/02/2025 Created 9 | */ 10 | CREATE VIEW [aw_int].[AccountRange] AS 11 | SELECT [ReportNo] 12 | ,[ReportHeading] 13 | ,[ReportHeadingNo] 14 | ,[ReportSection] 15 | ,[LineNo] 16 | ,[LinkLineNo] 17 | ,[LinkSignage] 18 | ,[LinkHeading] 19 | ,[LinkReport] 20 | ,[Operator] 21 | ,[Report] 22 | ,[Calc1] as Calc1 23 | ,[FileName] 24 | ,ISNULL(CONVERT(VARCHAR(36),LineageKey),0) AS LineageKey 25 | FROM LH.aw_stg.[accountrange] -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/edwTableJoins.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[edwTableJoins] ( 2 | [TableID] SMALLINT NOT NULL, 3 | [RelatedTableID] SMALLINT NOT NULL, 4 | [ShowBK] BIT NULL, 5 | [JoinSQL] VARCHAR (512) NULL, 6 | [AliasPK] VARCHAR (50) NULL, 7 | [JoinOrder] SMALLINT NULL, 8 | [JoinType] VARCHAR (20) NULL, 9 | CONSTRAINT [PK_edwTableJoins] PRIMARY KEY CLUSTERED ([TableID] ASC, [RelatedTableID] ASC), 10 | CONSTRAINT [FK_edwTableJoins_edwTables] FOREIGN KEY ([TableID]) REFERENCES [config].[edwTables] ([TableID]), 11 | CONSTRAINT [FK_edwTableJoins_edwTables1] FOREIGN KEY ([RelatedTableID]) REFERENCES [config].[edwTables] ([TableID]) 12 | ); 13 | 14 | 15 | GO 16 | 17 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/int/StoredProcedures/usp_SettingEndDate.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: Set EndDate for populating DimDate on the basis on max date from Trans table 3 | History: 4 | 20/02/2025 Created 5 | */ 6 | CREATE PROC [int].[usp_SettingEndDate] AS 7 | BEGIN 8 | SET NOCOUNT ON; 9 | 10 | DECLARE @EndDate AS INT; 11 | DECLARE @MaxDate AS DATE; 12 | 13 | SET @MaxDate = ( 14 | SELECT COALESCE(MAX(Date),getdate()) 15 | FROM tst.Finance 16 | ); 17 | SET @EndDate = ( 18 | SELECT CONVERT(VARCHAR(8), DATEADD(yy, DATEDIFF(yy, 0, MAX(DATEADD(d, 7, CONVERT(DATE, @MaxDate)))) + 1, - 1), 112) AS EndDate 19 | ); 20 | 21 | UPDATE config.Settings 22 | SET [Value] = @EndDate 23 | WHERE Entity = 'EDW' 24 | AND Attribute = 'EndDate'; 25 | END -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/definition/model.tmdl: -------------------------------------------------------------------------------- 1 | model Model 2 | culture: en-IE 3 | defaultPowerBIDataSourceVersion: powerBI_V3 4 | sourceQueryCulture: en-IE 5 | dataAccessOptions 6 | legacyRedirects 7 | returnErrorValuesAsNull 8 | 9 | queryGroup 'Finance GL' 10 | 11 | annotation PBI_QueryGroupOrder = 1 12 | 13 | annotation PBI_QueryOrder = ["General Ledger","Account","Department","Scenario","Organization","Date","Finanical Report","ReportAccountMap"] 14 | 15 | annotation __PBI_TimeIntelligenceEnabled = 0 16 | 17 | annotation PBIDesktopVersion = 2.138.1203.0 (24.11) 18 | 19 | ref table Account 20 | ref table 'General Ledger' 21 | ref table Department 22 | ref table Scenario 23 | ref table Organization 24 | ref table Date 25 | ref table 'Finanical Report' 26 | ref table ReportAccountMap 27 | 28 | ref cultureInfo en-IE 29 | 30 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/StoredProcedures/usp_PipelinePostExecute.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: Log Pipline Post Execute Worker 3 | Used By: ADF Pipeline-Worker 4 | EXEC [dwa].[usp_PipelinePostExecute] 'AB839FF1-D84D-4D78-AA11-56108971B03E' ,2 5 | History: 6 | 19/09/2023 Deepak, Created 7 | 04/06/2025 Kristan, Removed code stuffing 8 | */ 9 | CREATE PROC [dwa].[usp_PipelinePostExecute] @RunID [char](36), @PipelineID [int] AS 10 | BEGIN 11 | SET NOCOUNT ON; 12 | DECLARE @PostExecuteSQL VARCHAR(4000) 13 | 14 | SELECT @PostExecuteSQL = p.PostExecuteSQL 15 | FROM audit.PipelineLog p 16 | WHERE p.PipelineID = @PipelineID AND p.RunID = @RunID 17 | GROUP BY p.PostExecuteSQL 18 | 19 | IF @PostExecuteSQL IS NOT NULL 20 | BEGIN 21 | PRINT ('/*Executing Post Execute*/' + char(13) + @PostExecuteSQL) 22 | EXEC(@PostExecuteSQL ) 23 | END 24 | END -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/StoredProcedures/usp_TableLoad_PrestageView.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description:Materialize VIEWs into _staging table. This is performance tweak if the view is expensive. 3 | Example: exec dwa.[usp_TableLoad_PrestageView] 'aw_int.Organization',null 4 | History: 14/08/2023 Deepak, Created 5 | */ 6 | CREATE PROC [dwa].[usp_TableLoad_PrestageView] @SourceObject sysname, @TargetObject [sysname] AS 7 | BEGIN 8 | SET NOCOUNT ON 9 | DECLARE @sql nvarchar(4000) ='' 10 | IF @TargetObject is null 11 | set @TargetObject = @SourceObject+ '_Prestage' 12 | SET @sql = 'DROP TABLE IF EXISTS ' + @TargetObject + ';' + CHAR(13) 13 | SET @sql = @sql + 'CREATE TABLE ' + @TargetObject + ' AS ' + CHAR(13) 14 | SET @sql = @sql + 'SELECT * FROM ' + @SourceObject + ';' 15 | PRINT '/* Prestage View. PrestageSourceFlag=1 */' 16 | PRINT @sql 17 | exec (@sql) 18 | 19 | END -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/audit/StoredProcedures/usp_PipelineEndFailure.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: Log Error for a Pipeline 3 | Used by: ADF Pipeline-Worker 4 | Example: 5 | exec [audit].[usp_PipelineEndFailure] 'FED89103-A6A6-4DE9-95C8-72F7FFEB029F','ErrorCode','ErrorMessage' 6 | 7 | History: 8 | 22/08/2021 Bob, Created for Fabric DWA 9 | 03/06/2025 Kristan, added to Pipeline-Worker 10 | 11 | */ 12 | CREATE PROC [audit].[usp_PipelineEndFailure] @LineageKey uniqueidentifier,@ErrorCode [varchar](max),@ErrorMessage [varchar](max) AS 13 | BEGIN 14 | SET NOCOUNT ON 15 | UPDATE [audit].PipelineLog 16 | SET Status='Failed', ErrorCode=coalesce(@ErrorCode,'Inner Activity'), ErrorMessage=coalesce(@ErrorMessage,'Inner Activiy Failed. Check Monitoring Hub '+ MonitoringUrl) 17 | WHERE LineageKey=@LineageKey and Status <> 'Failed' 18 | 19 | 20 | END 21 | 22 | GO 23 | 24 | -------------------------------------------------------------------------------- /Workspaces/DWA/Jobs/SCH_ConfigBackup.DataPipeline/pipeline-content.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": { 3 | "activities": [ 4 | { 5 | "type": "InvokePipeline", 6 | "typeProperties": { 7 | "parameters": {}, 8 | "waitOnCompletion": true, 9 | "workspaceId": "00000000-0000-0000-0000-000000000000", 10 | "pipelineId": "4f2f3edf-f172-aca8-4aa2-72a81aefde0f", 11 | "operationType": "InvokeFabricPipeline" 12 | }, 13 | "externalReferences": { 14 | "connection": "8bb55f8d-a23b-4d7a-9b1e-f75030d20acb" 15 | }, 16 | "policy": { 17 | "timeout": "0.12:00:00", 18 | "retry": 0, 19 | "retryIntervalInSeconds": 30, 20 | "secureInput": false, 21 | "secureOutput": false 22 | }, 23 | "name": "TR_ConfigBackup", 24 | "dependsOn": [] 25 | } 26 | ] 27 | } 28 | } -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/cdcSqlTables.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[cdcSqlTables] ( 2 | [ConfigurationID] INT NOT NULL, 3 | [LakehouseName] VARCHAR (128) NULL, 4 | [Database] VARCHAR (128) NOT NULL, 5 | [Table] VARCHAR (512) NOT NULL, 6 | [PrimaryKeys] VARCHAR (500) NULL, 7 | [max_lsn] BINARY (10) NULL, 8 | [max_datetime] DATETIME2 (7) NULL, 9 | [target_lsn] BINARY (10) NULL, 10 | [target_datetime] DATETIME2 (7) NULL, 11 | [Enabled] BIT CONSTRAINT [DF_cdcSqlTables_Enabled] DEFAULT ((1)) NOT NULL, 12 | CONSTRAINT [PK_cdcSqlTables] PRIMARY KEY CLUSTERED ([ConfigurationID] ASC, [Database] ASC, [Table] ASC), 13 | CONSTRAINT [FK_cdcSqlTables_Configurations] FOREIGN KEY ([ConfigurationID]) REFERENCES [config].[Configurations] ([ConfigurationID]) 14 | ); 15 | 16 | 17 | GO 18 | 19 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/Meta.sqlproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Meta 6 | {00000000-0000-0000-0000-000000000000} 7 | Microsoft.Data.Tools.Schema.Sql.SqlDbFabricDatabaseSchemaProvider 8 | 1033, CI 9 | 10 | 11 | 12 | False 13 | master 14 | 170.0.0 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/StoredProcedures/usp_cdcSqlTables.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Return a List of Tables for CDC and their Primary Keys 3 | Used By: Extract-Parquet-CDC Notebook 4 | Test: 5 | exec [config].[usp_cdcSqlTables] @LakehouseName='LH' 6 | History: 15/06/2025 Bob, Created 7 | 29/07/2025 Kristan, added SqlTables param 8 | 9 | */ 10 | CREATE PROCEDURE [config].[usp_cdcSqlTables] 11 | ( 12 | @LakehouseName sysname = 'FabricLH', 13 | @SqlTables sysname = 'config.cdcSqlTables' 14 | ) 15 | AS 16 | BEGIN 17 | SET NOCOUNT ON; 18 | 19 | DECLARE @sql nvarchar(max) 20 | 21 | SET @sql = 'SELECT [Table], PrimaryKeys 22 | FROM ' + @SqlTables + ' 23 | WHERE LakehouseName = @LakehouseName AND Enabled = 1' 24 | 25 | EXEC sp_executesql @sql, 26 | N'@LakehouseName sysname', 27 | @LakehouseName = @LakehouseName 28 | END 29 | 30 | GO 31 | 32 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/audit/Tables/ReleaseLog.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [audit].[ReleaseLog] ( 2 | [ReleaseNo] INT NOT NULL, 3 | [SourceWorkspaceId] UNIQUEIDENTIFIER NOT NULL, 4 | [TargetWorkspaceId] UNIQUEIDENTIFIER NOT NULL, 5 | [SourceWorkspace] VARCHAR (50) NOT NULL, 6 | [TargetWorkspace] VARCHAR (50) NOT NULL, 7 | [ArtefactsJson] VARCHAR (8000) NULL, 8 | [SqlScripts] VARCHAR (8000) NULL, 9 | [ReleaseFolder] VARCHAR (8000) NOT NULL, 10 | [CreatedBy] VARCHAR (8000) NULL, 11 | [CreatedDateTime] DATETIME2 (6) NOT NULL, 12 | [Status] VARCHAR (20) NOT NULL, 13 | [ErrorMessage] VARCHAR (8000) NULL, 14 | [StartedTime] DATETIME2 (6) NULL, 15 | [CompletedTime] DATETIME2 (6) NULL, 16 | [ReleaseNotes] VARCHAR (8000) NULL, 17 | [ReleaseNotesFile] VARCHAR (500) NULL, 18 | PRIMARY KEY CLUSTERED ([ReleaseNo] ASC) 19 | ); 20 | 21 | 22 | GO 23 | 24 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw_int/Views/Currency.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) 3ECCC8478CD6EC2DD8FE82A5A0E6D31D54C47E752B680C2872653DB38C86E2BE 2 | 3 | 4 | 5 | /****** Object: View [aw_int].[Currency] Script Date: 06/11/2024 19:39:49 ******/ 6 | 7 | /* Description: AW DimCurrency 8 | Example: EXEC dwa.usp_TableLoad NULL,2,NULL 9 | History: 10 | 19/02/2025 Deepak Created 11 | */ 12 | CREATE VIEW [aw_int].[Currency] 13 | AS 14 | SELECT CONVERT(VARCHAR(16),HASHBYTES('MD5', [CurrencyCode]),2) AS CurrencyKey 15 | , ISNULL(CONVERT(CHAR(3), [CurrencyCode]), '') AS CurrencyAlternateKey 16 | , ISNULL(CONVERT(VARCHAR(50), [CurrencyName]), '') AS CurrencyName 17 | , ROW_NUMBER() OVER (PARTITION BY [CurrencyCode],[CurrencyName] ORDER BY [CurrencyCode] DESC) AS RowVersionNo 18 | , ISNULL(CONVERT(bigint,BINARY_CHECKSUM([CurrencyName],[CurrencyCode])),0) AS RowChecksum 19 | , CONVERT(VARCHAR(512), [FileName]) AS FileName 20 | ,ISNULL(CONVERT(VARCHAR(36),LineageKey),0) AS LineageKey 21 | FROM LH.aw_stg.[currency]; -------------------------------------------------------------------------------- /Workspaces/DWA/Jobs/SCH_AdventureWorks.DataPipeline/pipeline-content.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": { 3 | "activities": [ 4 | { 5 | "type": "InvokePipeline", 6 | "typeProperties": { 7 | "parameters": { 8 | "PackageGroup": "AW", 9 | "Stage": "ALL", 10 | "PipelineID": "-1" 11 | }, 12 | "waitOnCompletion": true, 13 | "workspaceId": "00000000-0000-0000-0000-000000000000", 14 | "pipelineId": "1967f137-61fb-bc8c-4f29-d84704ff05ff", 15 | "operationType": "InvokeFabricPipeline" 16 | }, 17 | "externalReferences": { 18 | "connection": "8bb55f8d-a23b-4d7a-9b1e-f75030d20acb" 19 | }, 20 | "policy": { 21 | "timeout": "0.12:00:00", 22 | "retry": 0, 23 | "retryIntervalInSeconds": 30, 24 | "secureInput": false, 25 | "secureOutput": false 26 | }, 27 | "name": "TR_AventureWorks", 28 | "dependsOn": [] 29 | } 30 | ] 31 | } 32 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw_int/Views/Finance.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) 21AA21A82C92B96AEC9FC877E72D97086DD9A28077D57C33B4BA0CB9BBC18D67 2 | 3 | /* Description: AW FactFinance 4 | Example: EXEC dwa.usp_TableLoad NULL,7,NULL 5 | History: 6 | 19/02/2025 Created 7 | */ 8 | CREATE VIEW [aw_int].[Finance] AS 9 | SELECT CONVERT(VARCHAR(8),CONVERT(DATE,CONVERT(VARCHAR(10),f.[AccountDate])),112) AS DateKey 10 | , f.[AccountCode] AS [AccountCodeAlternateKey] 11 | , f.[AccountDate] as [Date] 12 | , CONVERT(VARCHAR(50), f.[OrganizationName]) AS OrganizationName 13 | , CONVERT(VARCHAR(50), f.[DepartmentGroupName]) AS DepartmentGroupName 14 | , CONVERT(VARCHAR(50), f.ScenarioName) AS ScenarioName 15 | , SUM(CONVERT(DECIMAL(20, 6), isnull(f.[Amount], 0))) AS Amount 16 | , CONVERT(VARCHAR(512), f.[FileName]) AS FileName 17 | , ISNULL(CONVERT(VARCHAR(36), f.LineageKey), 0) AS LineageKey 18 | FROM LH.aw_stg.transactions f 19 | GROUP BY [AccountDate],[AccountCode],[OrganizationName],[DepartmentGroupName],ScenarioName,f.LineageKey, f.FileName -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/int/StoredProcedures/usp_LoadNumbers.sql: -------------------------------------------------------------------------------- 1 | /* Description: Create Numbers Tables 2 | History: 3 | 20/02/2025 Created 4 | */ 5 | CREATE PROC [int].[usp_LoadNumbers] AS 6 | BEGIN 7 | SET NOCOUNT ON; 8 | 9 | IF OBJECT_ID('int.Numbers') IS NOT NULL 10 | BEGIN 11 | DROP TABLE int.Numbers 12 | END 13 | 14 | BEGIN 15 | CREATE TABLE int.Numbers( 16 | [n] [bigint] NULL 17 | ) 18 | END 19 | 20 | ;WITH L0 AS (SELECT 1 AS n UNION ALL SELECT 1), 21 | L1 AS (SELECT 1 AS n FROM L0 AS a CROSS JOIN L0 AS b), 22 | L2 AS (SELECT 1 AS n FROM L1 AS a CROSS JOIN L1 AS b), 23 | L3 AS (SELECT 1 AS n FROM L2 AS a CROSS JOIN L2 AS b), 24 | L4 AS (SELECT 1 AS n FROM L3 AS a CROSS JOIN L3 AS b), 25 | L5 AS (SELECT 1 AS n FROM L4 AS a CROSS JOIN L4 AS b), 26 | Nums AS (SELECT ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) AS n FROM L5) 27 | 28 | 29 | 30 | INSERT INTO int.Numbers(n) 31 | SELECT TOP(10958) n FROM Nums ORDER BY n; 32 | END -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/definition/relationships.tmdl: -------------------------------------------------------------------------------- 1 | relationship f4aa0e09-ce9e-4d49-9822-a57fdc5363e2 2 | fromColumn: 'General Ledger'.OrganizationKey 3 | toColumn: Organization.OrganizationKey 4 | 5 | relationship 296c0dac-acaa-487b-ace8-4d6118073ab8 6 | fromColumn: 'General Ledger'.DateKey 7 | toColumn: Date.DateKey 8 | 9 | relationship bc931702-a639-4b45-92fb-fee72bd82ea6 10 | fromColumn: 'General Ledger'.AccountKey 11 | toColumn: Account.AccountKey 12 | 13 | relationship f0eefeab-44bf-4fc2-b20a-96f646abd14d 14 | fromColumn: 'General Ledger'.DepartmentGroupKey 15 | toColumn: Department.DepartmentGroupKey 16 | 17 | relationship fbf24dff-3cf2-45ed-9c8c-7af158d9c802 18 | fromColumn: 'General Ledger'.ScenarioKey 19 | toColumn: Scenario.ScenarioKey 20 | 21 | relationship 96e19940-038d-458c-a4eb-fdf054f0a7d7 22 | crossFilteringBehavior: bothDirections 23 | fromColumn: ReportAccountMap.AccountKey 24 | toColumn: Account.AccountKey 25 | 26 | relationship 04e3da1c-ad13-466e-8435-7e97d963a265 27 | fromColumn: ReportAccountMap.ReportNo 28 | toColumn: 'Finanical Report'.'Report No' 29 | 30 | -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/definition/tables/Scenario.tmdl: -------------------------------------------------------------------------------- 1 | table Scenario 2 | lineageTag: aa55d8a1-9a57-4815-8fe3-48acd590fb88 3 | 4 | column ScenarioKey 5 | dataType: string 6 | isHidden 7 | lineageTag: f23aa67a-f3b8-48a0-b7b0-2957a5a9e16f 8 | summarizeBy: none 9 | sourceColumn: ScenarioKey 10 | 11 | changedProperty = IsHidden 12 | 13 | annotation SummarizationSetBy = Automatic 14 | 15 | column Scenario 16 | dataType: string 17 | lineageTag: 101bc198-8ddd-40df-aa80-7af260039f2c 18 | summarizeBy: none 19 | sourceColumn: Scenario 20 | 21 | annotation SummarizationSetBy = Automatic 22 | 23 | partition Scenario-180ddbc1-ecb6-47cf-a0a3-a068bda0bc9c = m 24 | mode: import 25 | queryGroup: 'Finance GL' 26 | source = 27 | let 28 | dbo_DimScenario = Sql.Database("fkm4vwf6l6zebg4lqrhbtdcmsq-yctecwmyrr4u3mdfun4j5hqjma.datawarehouse.fabric.microsoft.com", "DW", [Query="SELECT ScenarioKey, Scenario FROM reports.DimScenario"]) 29 | in 30 | dbo_DimScenario 31 | 32 | annotation PBI_ResultType = Table 33 | 34 | annotation PBI_NavigationStepName = Navigation 35 | 36 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/reports/Views/Department.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) A71F75AABF4A8511B842E551A008A8C60FB84054DB5AE22412D8DA94DEDF8CB4 2 | 3 | 4 | 5 | 6 | /* Description: AW DepartmentGroup for PowerBI Direct Lake Model 7 | 8 | History: 9 | 21/02/2025 Created 10 | */ 11 | CREATE VIEW [reports].[Department] 12 | AS 13 | SELECT DepartmentGroupKey 14 | ,ParentDepartmentGroupKey 15 | ,DepartmentGroupName AS [Department Name] 16 | ,CASE WHEN CHARINDEX('|', DepartmentGroupPath) = 0 THEN DepartmentGroupPath ELSE SUBSTRING(DepartmentGroupPath, 1, CHARINDEX('|', DepartmentGroupPath) -1) END AS [Department Group] 17 | ,CASE WHEN CHARINDEX('|', DepartmentGroupPath) = 0 THEN '' ELSE SUBSTRING(DepartmentGroupPath, CHARINDEX('|', DepartmentGroupPath) + 1, LEN(DepartmentGroupPath) - CHARINDEX('|', DepartmentGroupPath) + 1) END AS Department 18 | FROM ( 19 | SELECT 20 | TRIM('|' FROM COALESCE(d2.DepartmentGroupName, '') + '|' + COALESCE(d1.DepartmentGroupName, '')) AS DepartmentGroupPath, 21 | d1.* 22 | FROM aw.DimDepartmentGroup d1 23 | LEFT JOIN 24 | aw.DimDepartmentGroup d2 ON d1.ParentDepartmentGroupKey = d2.DepartmentGroupKey 25 | ) d -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/reports/Views/DimDepartmentGroup.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) A71F75AABF4A8511B842E551A008A8C60FB84054DB5AE22412D8DA94DEDF8CB4 2 | 3 | 4 | 5 | /* Description: AW DimDepartmentGroup for PowerBI Model 6 | 7 | History: 8 | 21/02/2025 Created 9 | */ 10 | CREATE VIEW [reports].[DimDepartmentGroup] 11 | AS 12 | SELECT DepartmentGroupKey 13 | ,ParentDepartmentGroupKey 14 | ,DepartmentGroupName AS [Department Name] 15 | ,CASE WHEN CHARINDEX('|', DepartmentGroupPath) = 0 THEN DepartmentGroupPath ELSE SUBSTRING(DepartmentGroupPath, 1, CHARINDEX('|', DepartmentGroupPath) -1) END AS [Department Group] 16 | ,CASE WHEN CHARINDEX('|', DepartmentGroupPath) = 0 THEN '' ELSE SUBSTRING(DepartmentGroupPath, CHARINDEX('|', DepartmentGroupPath) + 1, LEN(DepartmentGroupPath) - CHARINDEX('|', DepartmentGroupPath) + 1) END AS Department 17 | FROM ( 18 | SELECT 19 | TRIM('|' FROM COALESCE(d2.DepartmentGroupName, '') + '|' + COALESCE(d1.DepartmentGroupName, '')) AS DepartmentGroupPath, 20 | d1.* 21 | FROM aw.DimDepartmentGroup d1 22 | LEFT JOIN 23 | aw.DimDepartmentGroup d2 ON d1.ParentDepartmentGroupKey = d2.DepartmentGroupKey 24 | ) d -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw_int/Views/Organization.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) F2AC694AB10BC94E381E64BD97B30BD1A7468C0D8998436EF1A14C54FD7457E9 2 | 3 | 4 | /* Description: AW Dimension Organization 5 | Example: EXEC dwa.usp_TableLoad NULL,5,NULL 6 | History: 7 | 19/02/2025 Created 8 | */ 9 | CREATE VIEW [aw_int].[Organization] AS 10 | SELECT ISNULL(CONVERT(VARCHAR(16),HASHBYTES('MD5', p.OrganizationName),2),'') AS OrganizationKey 11 | , CONVERT(VARCHAR(16), c.ParentOrganizationKey) AS ParentOrganizationKey 12 | , ISNULL(CONVERT(VARCHAR(10), PercentageOfOwnership), 0) AS PercentageOfOwnership 13 | , ISNULL(CONVERT(VARCHAR(50), p.OrganizationName), '') AS OrganizationName 14 | , ISNULL(CONVERT(VARCHAR(16), cu.CurrencyKey), 0) AS CurrencyKey 15 | , CONVERT(VARCHAR(512), p.[FileName]) AS FileName 16 | ,ISNULL(CONVERT(VARCHAR(36),p.LineageKey),0) AS LineageKey 17 | FROM LH.aw_stg.organization p 18 | LEFT JOIN (SELECT CONVERT(VARCHAR(16),HASHBYTES('MD5', OrganizationName),2) AS ParentOrganizationKey, OrganizationName FROM LH.aw_stg.organization) c 19 | ON c.OrganizationName=p.ParentOrganizationName 20 | INNER JOIN aw.DimCurrency cu ON cu.CurrencyAlternateKey=p.CurrencyCode COLLATE Latin1_General_100_CI_AS_KS_WS_SC_UTF8; -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/PipelineMeta.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[PipelineMeta] ( 2 | [PipelineID] INT NULL, 3 | [Pipeline] VARCHAR (200) NULL, 4 | [Template] VARCHAR (128) NULL, 5 | [SourceConnectionSettings] VARCHAR (8000) NULL, 6 | [TargetConnectionSettings] VARCHAR (8000) NULL, 7 | [SourceSettings] VARCHAR (8000) NULL, 8 | [TargetSettings] VARCHAR (8000) NULL, 9 | [ActivitySettings] VARCHAR (8000) NULL, 10 | [PipelineSequence] INT NOT NULL, 11 | [PackageGroup] VARCHAR (8000) NULL, 12 | [PreExecuteSQL] VARCHAR (8000) NULL, 13 | [PostExecuteSQL] VARCHAR (8000) NULL, 14 | [Enabled] BIT NOT NULL, 15 | [Stage] VARCHAR (50) NULL, 16 | [TemplateType] VARCHAR (50) NULL, 17 | [TemplateID] UNIQUEIDENTIFIER NULL, 18 | [TableID] INT NULL, 19 | [SessionTag] VARCHAR (50) NULL, 20 | [TemplateWorkspaceID] UNIQUEIDENTIFIER NULL 21 | ); 22 | 23 | 24 | GO 25 | 26 | -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Shared/SQL-Connection-Shared-Functions.Notebook/notebook-content.py: -------------------------------------------------------------------------------- 1 | # Fabric notebook source 2 | 3 | # METADATA ******************** 4 | 5 | # META { 6 | # META "kernel_info": { 7 | # META "name": "synapse_pyspark" 8 | # META }, 9 | # META "dependencies": {} 10 | # META } 11 | 12 | # MARKDOWN ******************** 13 | 14 | # #### SQL-Connection-Shared-Functions 15 | # 16 | # This notebook is designed to establish a secure connection to a Microsoft Fabric or Power BI-backed SQL endpoint using a short-lived Azure Active Directory (AAD) token 17 | 18 | # CELL ******************** 19 | 20 | import struct 21 | import sqlalchemy 22 | import pyodbc 23 | from notebookutils import mssparkutils 24 | def create_engine(connection_string : str): 25 | token = mssparkutils.credentials.getToken('https://analysis.windows.net/powerbi/api').encode("UTF-16-LE") 26 | token_struct = struct.pack(f' 'sysdiagrams' and SCHEMA_NAME(schema_id) = 'config' 13 | UNION ALL 14 | SELECT cte_2.lvl + 1 AS Expr1, t.object_id, t.name 15 | FROM cte AS cte_2 INNER JOIN 16 | sys.tables AS t ON EXISTS 17 | (SELECT NULL AS Expr1 18 | FROM sys.foreign_keys AS fk 19 | WHERE (parent_object_id = t.object_id) AND (referenced_object_id = cte_2.object_id)) AND t.object_id <> cte_2.object_id AND cte_2.lvl < 30 20 | WHERE (t.type_desc = 'USER_TABLE') AND (t.is_ms_shipped = 0) and SCHEMA_NAME(schema_id) = 'config') 21 | SELECT TOP (100) PERCENT name, MAX(lvl) AS dependency_level 22 | FROM cte AS cte_1 23 | GROUP BY name 24 | ORDER BY dependency_level, name 25 | 26 | END 27 | 28 | GO 29 | 30 | -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/definition/tables/ReportAccountMap.tmdl: -------------------------------------------------------------------------------- 1 | table ReportAccountMap 2 | isHidden 3 | lineageTag: ee45c0d5-cc35-4daf-aea3-ea0a13419568 4 | 5 | column ReportNo 6 | dataType: int64 7 | isHidden 8 | formatString: 0 9 | lineageTag: 2a4b4412-4b9d-494e-a22f-8b63da0321aa 10 | summarizeBy: none 11 | sourceColumn: ReportNo 12 | 13 | changedProperty = IsHidden 14 | 15 | annotation SummarizationSetBy = User 16 | 17 | column Report 18 | dataType: string 19 | isHidden 20 | lineageTag: 17591d0a-3e58-40ab-b34d-aade1f2b14d2 21 | summarizeBy: none 22 | sourceColumn: Report 23 | 24 | changedProperty = IsHidden 25 | 26 | annotation SummarizationSetBy = Automatic 27 | 28 | column AccountKey 29 | dataType: string 30 | isHidden 31 | lineageTag: 4116a320-6a73-48a0-9f44-9d2ecb7784b7 32 | summarizeBy: none 33 | sourceColumn: AccountKey 34 | 35 | changedProperty = IsHidden 36 | 37 | annotation SummarizationSetBy = Automatic 38 | 39 | partition ReportAccountMap-7705b288-2899-4e91-b62d-418d02cf2e28 = m 40 | mode: import 41 | queryGroup: 'Finance GL' 42 | source = 43 | let 44 | Source= Sql.Database("fkm4vwf6l6zebg4lqrhbtdcmsq-yctecwmyrr4u3mdfun4j5hqjma.datawarehouse.fabric.microsoft.com", "DW", [Query="SELECT ReportNo, Report, AccountKey FROM [reports].[ReportAccountMap]"]) 45 | in 46 | Source 47 | 48 | changedProperty = IsHidden 49 | 50 | annotation PBI_ResultType = Table 51 | 52 | annotation PBI_NavigationStepName = Navigation 53 | 54 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/audit/Tables/PipelineLog.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [audit].[PipelineLog] ( 2 | [LineageKey] UNIQUEIDENTIFIER NOT NULL, 3 | [RunID] UNIQUEIDENTIFIER NULL, 4 | [PipelineID] INT NOT NULL, 5 | [PackageGroup] VARCHAR (50) NULL, 6 | [PipelineName] VARCHAR (512) NULL, 7 | [Status] VARCHAR (100) NOT NULL, 8 | [StartDate] DATE NOT NULL, 9 | [StartDateTime] DATETIME2 (6) NOT NULL, 10 | [EndDateTime] DATETIME2 (6) NULL, 11 | [Template] VARCHAR (200) NULL, 12 | [StartedBy] VARCHAR (512) NULL, 13 | [PipelineSequence] SMALLINT NULL, 14 | [ErrorCode] VARCHAR (4000) NULL, 15 | [ErrorMessage] VARCHAR (4000) NULL, 16 | [Stage] VARCHAR (20) NULL, 17 | [Pipeline] UNIQUEIDENTIFIER NULL, 18 | [ParentRunID] UNIQUEIDENTIFIER NULL, 19 | [ParentGroupID] UNIQUEIDENTIFIER NULL, 20 | [WorkspaceID] UNIQUEIDENTIFIER NULL, 21 | [MonitoringUrl] VARCHAR (512) NULL, 22 | [RowsRead] BIGINT NULL, 23 | [CopyDuration] INT NULL, 24 | [DataRead] BIGINT NULL, 25 | [DataWritten] BIGINT NULL, 26 | [ExtendedLog] VARCHAR (8000) NULL, 27 | [DebugOutput] VARCHAR (8000) NULL, 28 | CONSTRAINT [PK_PipelineLog] PRIMARY KEY CLUSTERED ([LineageKey] ASC) 29 | ); 30 | 31 | 32 | GO 33 | 34 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw_int/Views/Date.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) 1FBD0A69F7FD430A9FD834BAC5C4E8DDB6006B8F01F70EDA41D10DDCD3B4BC51 2 | 3 | /****** Object: View [aw_int].[Date] Script Date: 06/11/2024 19:37:48 ******/ 4 | /* Source: Calculated 5 | Unit Test: dwa.usp_TableLoad @TableID=1 6 | History: 19/02/2025 Created 7 | */ 8 | CREATE VIEW [aw_int].[Date] 9 | AS 10 | SELECT ISNULL(CONVERT(VARCHAR(8),CONVERT(DATE,CONVERT(VARCHAR(10),Date),103),112),0) AS DateKey 11 | ,ISNULL(CONVERT(DATE,CONVERT(VARCHAR(10),Date),103),'') AS FullDateAlternateKey 12 | ,ISNULL(CONVERT(smallint,DayNumberOfWeek),0) AS DayNumberOfWeek 13 | ,ISNULL(CONVERT(varchar(50),EnglishDayNameOfWeek),'') AS DayOfWeek 14 | ,ISNULL(CONVERT(smallint,DayNumberOfMonth),0) AS DayNumberOfMonth 15 | ,ISNULL(CONVERT(int,DayNumberOfYear),0) AS DayNumberOfYear 16 | ,ISNULL(CONVERT(int,WeekNumberOfYear),0) AS WeekNumberOfYear 17 | ,ISNULL(CONVERT(varchar(50),EnglishMonthName),'') AS MonthName 18 | ,ISNULL(CONVERT(smallint,MonthNumberOfYear),0) AS MonthNumberOfYear 19 | ,ISNULL(CONVERT(smallint,CalendarQuarter),0) AS CalendarQuarter 20 | ,ISNULL(CONVERT(smallint,CalendarYear),0) AS CalendarYear 21 | ,ISNULL(CONVERT(smallint,CalendarSemester),0) AS CalendarSemester 22 | ,ISNULL(CONVERT(smallint,FiscalQuarter),0) AS FiscalQuarter 23 | ,ISNULL(CONVERT(smallint,FiscalYear),0) AS FiscalYear 24 | ,ISNULL(CONVERT(smallint,FiscalSemester),0) AS FiscalSemester 25 | ,CONVERT(VARCHAR(512), [FileName]) AS FileName 26 | ,ISNULL(CONVERT(VARCHAR(36),LineageKey),0) AS LineageKey 27 | FROM [LH].[aw_stg].[date]; -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/audit/StoredProcedures/usp_PipelineEndSuccess.sql: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | Description: Complete a Pipeline 4 | Used By: ADF Pipeline-Worker 5 | 6 | -- Example 7 | exec [audit].[usp_PipelineStart] 37, '7EEFA88E-D9F9-4204-AC47-97F2FC3362CE', '1087FDF2-90FB-437B-A3CB-D2F94E8C4D82', 2 8 | SELECT * FROM audit.PipelineLog ORDER BY StartDateTime DESC 9 | SELECT * FROM audit.LineageLog ORDER BY LineageKey DESC 10 | exec [audit].[usp_PipelineStart] 1,'7EEFA88E-D9F9-4204-AC47-97F2FC3362CE' 11 | 12 | History: 13 | 30/04/2024 Bob, Migrated to Fabric 14 | 04/06/2024 Aidan, Added Backoff Algorithm 15 | 04/01/2025 Aidan, Migrated to META 16 | 03/06/2025 Kristan, Added to Pipeline-Worker 17 | */ 18 | 19 | CREATE PROC [audit].[usp_PipelineEndSuccess] 20 | @LineageKey uniqueidentifier 21 | AS 22 | BEGIN 23 | SET NOCOUNT ON; 24 | 25 | DECLARE @PostExecuteSQL varchar(8000) 26 | DECLARE @DateTime datetime 27 | 28 | SELECT 29 | @PostExecuteSQL = COALESCE(p.PostExecuteSQL, pg.PostExecuteSQL), 30 | @DateTime = GETDATE() 31 | FROM [config].[Pipelines] p 32 | INNER JOIN [config].[PipelineGroups] pg ON pg.PipelineGroupID = p.PipelineGroupID 33 | 34 | UPDATE [audit].PipelineLog 35 | SET 36 | Status = 'Completed', 37 | EndDateTime = @DateTime 38 | WHERE LineageKey = @LineageKey 39 | AND EndDateTime IS NULL 40 | 41 | IF @PostExecuteSQL IS NOT NULL 42 | BEGIN 43 | PRINT ('/*Executing Post Execute*/' + CHAR(13) + @PostExecuteSQL) 44 | EXEC(@PostExecuteSQL) 45 | END 46 | END 47 | 48 | GO 49 | 50 | -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Refresh/Refresh-PBI.DataPipeline/pipeline-content.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": { 3 | "activities": [ 4 | { 5 | "name": "Semantic model refresh", 6 | "type": "PBISemanticModelRefresh", 7 | "dependsOn": [], 8 | "policy": { 9 | "timeout": "0.12:00:00", 10 | "retry": 0, 11 | "retryIntervalInSeconds": 30, 12 | "secureOutput": false, 13 | "secureInput": false 14 | }, 15 | "typeProperties": { 16 | "method": "post", 17 | "waitOnCompletion": true, 18 | "commitMode": "Transactional", 19 | "operationType": "SemanticModelRefresh", 20 | "datasetId": "@json(string(pipeline().parameters.SourceConnectionSettings)).DatasetID", 21 | "groupId": "@json(string(pipeline().parameters.SourceConnectionSettings)).WorkspaceID" 22 | }, 23 | "externalReferences": { 24 | "connection": "25dc7bd7-61f3-4683-adf1-9c6141e45385" 25 | } 26 | } 27 | ], 28 | "parameters": { 29 | "SourceConnectionSettings": { 30 | "type": "string", 31 | "defaultValue": { 32 | "WorkspaceID": "9b8a6500-5ccb-49a9-885b-b5b081efed75", 33 | "DatasetID": "b385ab81-ec52-44da-911a-a09e07f4e56f" 34 | } 35 | }, 36 | "SourceSettings": { 37 | "type": "string" 38 | }, 39 | "LineageKey": { 40 | "type": "string" 41 | }, 42 | "TargetSettings": { 43 | "type": "string" 44 | }, 45 | "TargetConnectionSettings": { 46 | "type": "string" 47 | }, 48 | "ActivitySettings": { 49 | "type": "string" 50 | } 51 | } 52 | } 53 | } -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw_int/Views/Account.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) EA147841DFF905BE86CB6B0C0B5CB27470E39EDC45FE607472A2CF6C94BD31EE 2 | 3 | 4 | 5 | /* Description: AW Dim Account 6 | Example: EXEC dwa.usp_TableLoad @TableID=6 7 | History: 8 | 19/02/2025 Shruti Created 9 | */ 10 | CREATE VIEW [aw_int].[Account] AS 11 | SELECT CONVERT(VARCHAR(16),HASHBYTES('MD5', CONVERT(VARCHAR(4),a.AccountCode)),2) AS AccountKey 12 | , CONVERT(VARCHAR(16), c.ParentAccountKey) AS ParentAccountKey 13 | , ISNULL(CONVERT(INT, a.AccountCode), 0) AS AccountCodeAlternateKey 14 | , CONVERT(INT, a.ParentAccountCode) AS ParentAccountCodeAlternateKey 15 | , ISNULL(CONVERT(VARCHAR(50), a.AccountDescription COLLATE Latin1_General_100_CI_AS_KS_WS_SC_UTF8), '') AS AccountDescription 16 | , CONVERT(VARCHAR(50), a.AccountType COLLATE Latin1_General_100_CI_AS_KS_WS_SC_UTF8) AS AccountType 17 | , ISNULL(CONVERT(VARCHAR(50), a.Operator COLLATE Latin1_General_100_CI_AS_KS_WS_SC_UTF8), '') AS Operator 18 | , CONVERT(VARCHAR(50), a.CustomMembers COLLATE Latin1_General_100_CI_AS_KS_WS_SC_UTF8) AS CustomMembers 19 | , ISNULL(CONVERT(VARCHAR(50), a.ValueType COLLATE Latin1_General_100_CI_AS_KS_WS_SC_UTF8), '') AS ValueType 20 | , CONVERT(VARCHAR(200), a.CustomMemberOptions COLLATE Latin1_General_100_CI_AS_KS_WS_SC_UTF8) AS CustomMemberOptions 21 | , CONVERT(VARCHAR(512), a.[FileName] COLLATE Latin1_General_100_CI_AS_KS_WS_SC_UTF8) AS FileName 22 | ,ISNULL(CONVERT(VARCHAR(36),LineageKey COLLATE Latin1_General_100_CI_AS_KS_WS_SC_UTF8),0) AS LineageKey 23 | FROM LH.aw_stg.[account] a 24 | LEFT JOIN (SELECT CONVERT(VARCHAR(16),HASHBYTES('MD5', CONVERT(VARCHAR(4),ParentAccountCode)),2) AS ParentAccountKey, AccountCode FROM LH.aw_stg.account) c ON c.AccountCode = a.ParentAccountCode; -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/aw/StoredProcedures/usp_AccountRangeMapCreate.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: Generate ReportAccountMap table for Financial Modelling 3 | Used By: Table load for ReportAccountMap 4 | 5 | -- Example 6 | exec [aw].[usp_AccountRangeMapCreate] 7 | SELECT * FROM aw.ReportAccountMap 8 | 9 | History: 10 | 20/02/2025 Created 11 | */ 12 | 13 | CREATE PROC [aw].[usp_AccountRangeMapCreate] AS 14 | BEGIN 15 | SET NOCOUNT ON; 16 | 17 | IF OBJECT_ID('aw.ReportAccountMap') IS NOT NULL 18 | DROP TABLE aw.ReportAccountMap; 19 | 20 | SELECT ar.ReportNo 21 | , ar.Report COLLATE Latin1_General_100_CI_AS_KS_WS_SC_UTF8 AS Report 22 | , a.AccountKey COLLATE Latin1_General_100_CI_AS_KS_WS_SC_UTF8 AS AccountKey 23 | INTO aw.ReportAccountMap 24 | FROM aw.DimAccount a 25 | INNER JOIN aw_int.AccountRangeRules arr 26 | ON a.AccountCodeAlternateKey BETWEEN arr.FromAccountNo AND arr.ToAccountNo 27 | INNER JOIN aw_int.AccountRange ar 28 | ON ar.ReportNo = arr.ReportNo 29 | WHERE ar.Operator = 'Sum' 30 | 31 | INSERT INTO aw.ReportAccountMap (ReportNo, Report, AccountKey) 32 | SELECT ar.ReportNo, ar.Report, m.AccountKey 33 | FROM aw_int.AccountRange ar 34 | INNER JOIN aw_int.AccountRange ar2 35 | ON ar2.[LineNo] BETWEEN 1 AND ar.[LineNo] AND ar.Report = ar2.Report 36 | INNER JOIN aw.ReportAccountMap m 37 | ON ar2.ReportNo = m.ReportNo 38 | WHERE ar.Operator = 'Running Sum' AND ar2.Operator = 'Sum' 39 | 40 | INSERT INTO aw.ReportAccountMap (ReportNo, Report, AccountKey) 41 | SELECT ar2.ReportNo, ar2.Report, m.AccountKey 42 | FROM aw.ReportAccountMap m 43 | INNER JOIN aw_int.AccountRange ar 44 | ON m.ReportNo = ar.ReportNo 45 | INNER JOIN aw_int.AccountRange ar2 46 | ON ar.Calc1 = ar2.ReportHeading 47 | WHERE ar.Operator = 'Sum' AND ar.Calc1 IS NOT NULL 48 | 49 | END -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/StoredProcedures/usp_TableSwap.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: Swap 2 table. useful for Green-Blue style deployments 3 | Assumes a schema called "tmp" exists 4 | Example: exec [dwa].[usp_TableSwap] 'aw_int.DimOrganization','aw.DimOrganization' 5 | History: 03/08/2025 Bob, Created 6 | */ 7 | CREATE PROC [dwa].[usp_TableSwap] @SourceTable sysname, @TargetTable sysname 8 | AS 9 | BEGIN 10 | SET NOCOUNT ON 11 | DECLARE @sql nvarchar(max) ='' 12 | ,@SourceSchema sysname 13 | ,@TargetSchema sysname 14 | ,@TempTable sysname 15 | 16 | SELECT @SourceSchema = coalesce(PARSENAME (@SourceTable,2),'dbo') 17 | ,@TargetSchema = coalesce(PARSENAME (@TargetTable,2),'dbo') 18 | ,@TempTable = 'tmp' + '.' + PARSENAME (@TargetTable,1) 19 | 20 | SET @sql ='/* dwa.usp_TableSwap ''' + @SourceTable + ''',''' + @TargetTable + ''' */' + CHAR(13) 21 | SET @sql =@sql + 'BEGIN TRANSACTION;' + CHAR(13) 22 | SET @sql = @sql + 'BEGIN TRY'+ CHAR(13) 23 | IF OBJECT_ID(@SourceTable) is not null 24 | BEGIN 25 | SET @sql = @sql +CHAR(9) + 'DROP TABLE IF EXISTS ' + @TempTable + ';' + CHAR(13) 26 | SET @sql=@sql + CHAR(9) +'ALTER SCHEMA tmp TRANSFER ' + @TargetTable + ';' + CHAR(13) 27 | END 28 | ELSE 29 | IF OBJECT_ID(@TempTable) is null 30 | return 31 | 32 | SET @sql=@sql + CHAR(9) +'ALTER SCHEMA ' + @TargetSchema + ' TRANSFER ' + @SourceTable + ';' + CHAR(13) 33 | SET @sql = @sql +CHAR(9) + 'DROP TABLE IF EXISTS ' + @TempTable + ';' + CHAR(13) 34 | SET @sql = @sql + CHAR(9) +'COMMIT TRANSACTION;'+ CHAR(13) 35 | SET @sql = @sql + 'END TRY'+ CHAR(13) 36 | SET @sql = @sql + 'BEGIN CATCH'+ CHAR(13) 37 | SET @sql = @sql + CHAR(9) +'ROLLBACK TRANSACTION;'+ CHAR(13) 38 | SET @sql = @sql + CHAR(9) +'THROW'+ CHAR(13) 39 | SET @sql = @sql + 'END CATCH;'+ CHAR(13) 40 | print @sql 41 | exec (@sql) 42 | 43 | END -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/aiPrompts.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[aiPrompts] ( 2 | [PromptID] INT NOT NULL, 3 | [prompt_name] VARCHAR (8000) NULL, 4 | [description] VARCHAR (8000) NULL, 5 | [instructions] VARCHAR (MAX) NULL, 6 | [input] VARCHAR (MAX) NULL, 7 | [model] VARCHAR (8000) NULL, 8 | [json_schema] VARCHAR (MAX) NULL, 9 | [ai_endpoint] VARCHAR (8000) NULL, 10 | [api_family] VARCHAR (8000) CONSTRAINT [DF_aiPrompts_api_family] DEFAULT ('responses') NOT NULL, 11 | [api_provider] VARCHAR (8000) CONSTRAINT [DF_aiPrompts_api_provider] DEFAULT ('azure_openai') NOT NULL, 12 | [temperature] NUMERIC (3, 2) CONSTRAINT [DF_aiPrompts_temperature] DEFAULT ((0.3)) NULL, 13 | [SourceConnnectionID] INT NULL, 14 | [ExtraSettings] VARCHAR (MAX) NULL, 15 | [ActivitySettings] VARCHAR (MAX) NULL, 16 | CONSTRAINT [PK_aiPrompts] PRIMARY KEY CLUSTERED ([PromptID] ASC) 17 | ); 18 | 19 | 20 | GO 21 | 22 | /* 23 | Update Activiy Settings for AI Prompts 24 | 25 | */ 26 | CREATE TRIGGER [config].[TR_aiPrompt_Update] 27 | ON [config].[aiPrompts] 28 | AFTER UPDATE, INSERT 29 | AS 30 | BEGIN 31 | SET NOCOUNT ON; 32 | 33 | UPDATE p 34 | SET p.ActivitySettings = dbo.ufn_BuildChatPayloadJson ( 35 | i.model, i.api_family, i.instructions, i.input, 36 | i.json_schema, i.temperature, i.ExtraSettings 37 | ) 38 | FROM config.aiPrompts AS p 39 | JOIN inserted AS i 40 | ON i.PromptID = p.PromptID; 41 | 42 | UPDATE p 43 | set p.ActivitySettings= ai.ActivitySettings 44 | FROM config.Pipelines p 45 | INNER JOIN inserted i on i.PromptID = p.PromptID 46 | INNER JOIN config.aiPrompts ai on ai.PromptID=i.PromptiD 47 | 48 | 49 | END 50 | 51 | GO 52 | 53 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/PipelineGroups.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[PipelineGroups] ( 2 | [PipelineGroupID] INT NOT NULL, 3 | [PipelineGroupName] VARCHAR (200) NULL, 4 | [PipelineGroupDesc] VARCHAR (500) NULL, 5 | [Template] VARCHAR (200) NULL, 6 | [SourceSettings] VARCHAR (8000) NULL, 7 | [TargetSettings] VARCHAR (8000) NULL, 8 | [Enabled] INT NULL, 9 | [ActivitySettings] VARCHAR (8000) NULL, 10 | [SourceConfigurationID] INT NULL, 11 | [TargetConfigurationID] INT NULL, 12 | [PackageGroup] VARCHAR (50) NULL, 13 | [Stage] VARCHAR (8000) NULL, 14 | [PostExecuteSQL] VARCHAR (8000) NULL, 15 | [PreExecuteSQL] VARCHAR (8000) NULL, 16 | [SourceConnectionSettings] VARCHAR (8000) NULL, 17 | [TargetConnectionSettings] VARCHAR (8000) NULL, 18 | [ContinueOnError] VARCHAR (200) NULL, 19 | [SessionTag] VARCHAR (50) NULL, 20 | [LakehouseConfigurationID] INT NULL, 21 | CONSTRAINT [PK_PipelineGroups] PRIMARY KEY NONCLUSTERED ([PipelineGroupID] ASC), 22 | CONSTRAINT [FK_PipelineGroups_Configurations] FOREIGN KEY ([SourceConfigurationID]) REFERENCES [config].[Configurations] ([ConfigurationID]), 23 | CONSTRAINT [FK_PipelineGroups_Configurations1] FOREIGN KEY ([TargetConfigurationID]) REFERENCES [config].[Configurations] ([ConfigurationID]), 24 | CONSTRAINT [FK_PipelineGroups_Templates] FOREIGN KEY ([Template]) REFERENCES [config].[Templates] ([Template]) 25 | ); 26 | 27 | 28 | GO 29 | 30 | 31 | 32 | CREATE TRIGGER [config].TR_PipelineGroups 33 | ON [config].[PipelineGroups] 34 | AFTER INSERT,DELETE,UPDATE 35 | AS 36 | BEGIN 37 | SET NOCOUNT ON; 38 | exec [config].[usp_PipelineBuildMetaData] 39 | 40 | END 41 | 42 | GO 43 | 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DWA - Data Warehouse Automation Framework (Fabric Edition) 2 | 3 | ## Welcome 4 | 5 | Welcome to the Prodata Fabric Data Warehouse Automation Framework. DWA is a metadata automation framework for data warehousing and data engineering. We have been developing and improving this framework for 15 years, and it is now available as open source for you to try. 6 | 7 | ## Overview 8 | 9 | ![image](https://github.com/user-attachments/assets/678020fe-ead9-41f9-a77e-597350fa5e45) 10 | 11 | The DWA framework supports the entire enterprise data warehouse lifecycle (and other data engineering projects) by reducing data engineering pipelines and tasks to metadata and enforcing common enterprise features such as logging, lineage, and error handling, along with orchestration and a flexible template library. 12 | 13 | ## Getting Started and Documentation 14 | 15 | You can browse this GitHub repository to use some of the templates for ideas on your project. 16 | 17 | For installation instructions, examples, demos, and deeper documentation, see the project wiki: 18 | 19 | https://github.com/ProdataSQL/DWA/wiki 20 | 21 | ## Engaging with Prodata for a Quick Start 22 | 23 | If you want to try the framework in your Fabric environment, Prodata offers a Fabric Data Warehouse Quick Start. This is a 2–3 week engagement to install the framework, create a sample proof of concept using your actual data, and provide a walkthrough and training to accelerate your adoption of Fabric. 24 | 25 | More details: 26 | https://prodata.ie/fabric-poc/ 27 | 28 | ## Videos 29 | 30 | ### Introduction to DWA in Fabric 31 | [![image](https://github.com/user-attachments/assets/0cce133b-1d61-4cc0-9f58-70eac999de5b)](https://www.youtube.com/watch?v=9hkCDL8TKSQ) 32 | 33 | ### 20-Minute Demo of automated Ingest, Extract, Transform and Load 34 | [![image](https://github.com/user-attachments/assets/47026239-97f7-48a9-81b5-da2d4d070d9f)](https://www.youtube.com/watch?v=Wc_JE8YsT90) 35 | 36 | ### Other Videos and Tutorials 37 | Tutorials Wiki Site: https://github.com/ProdataSQL/DWA/wiki/8.-Tutorials -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/StoredProcedures/usp_TableLoad_Update.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: UPDATE Operation for Table Load framework - Execute main usp_TableLoad proc 3 | Example: exec dwa.[usp_TableLoad] 'aw.FactFinance',NULL 4 | exec dwa.[usp_TableLoad] NULL, 2 ,NULL,NULL 5 | History: 03/08/2023 Deepak, Created 6 | */ 7 | CREATE PROC [dwa].[usp_TableLoad_Update] @TableID [int],@TargetObject [sysname],@SourceObject [sysname],@UpdateColumns [nvarchar](max),@SelectColumns [nvarchar](max),@JoinSQL [nvarchar](max),@SqlWhere [nvarchar](max),@WhereJoinSQL [nvarchar](max),@Exists [bit] , @TablePrefix sysname,@BusinessKeys varchar(4000), @PrestageTargetFlag bit, @SqlWhereOuter varchar(4000) AS 8 | BEGIN 9 | SET NOCOUNT ON 10 | DECLARE @sql nvarchar(max) 11 | 12 | --IF @PrestageTargetFlag = 1 13 | 14 | SET @JoinSQL = ' '+COALESCE(@JoinSQL,'') + ' '+ COALESCE(@SqlWhere,'') + ') '+@TablePrefix+' ON ' + char(13) 15 | SET @SQL = 'UPDATE t ' + char(13) + 'SET ' 16 | SET @SQL = @SQL + @UpdateColumns + char(13) 17 | SET @SQL = @SQL + 'FROM ' + @TargetObject + ' t' + char(13) 18 | 19 | print @JoinSQL 20 | IF @JoinSQL IS NOT NULL and @PrestageTargetFlag =0 21 | BEGIN 22 | SET @SQL =@SQL + 'INNER JOIN (SELECT ' + @SelectColumns + ' FROM ' + @SourceObject +' ' + @TablePrefix + CHAR(13) 23 | SET @SqlWhere = NULL 24 | END 25 | ELSE 26 | SET @SQL = @SQL + 'INNER JOIN ' + @SourceObject + ' ' + @TablePrefix + CHAR(13) + 'ON ' 27 | 28 | if @PrestageTargetFlag=0 SET @SQL= coalesce(@SQL + @JoinSQL ,@SQL) 29 | SET @SQL = @SQL + ( SELECT string_agg (@TablePrefix + '.' + ltrim(bk.value) + '=t.' + ltrim(bk.value) , ' AND ') FROM string_split(@BusinessKeys,',') bk ) 30 | SET @SQL = @SQL + char(13) + 'AND ' + @TablePrefix + '.RowChecksum <> t.RowChecksum' 31 | 32 | IF len(@SqlWhereOuter) > 0 33 | SET @SqlWhere = coalesce (@SqlWhere + ' AND ', 'WHERE ' ) + @SqlWhereOuter 34 | 35 | IF @SqlWhere is not null SET @sql=@sql + char(13) + @SqlWhere 36 | SET @sql =@sql + ';' 37 | PRINT @sql 38 | EXEC (@sql) 39 | END -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/StoredProcedures/usp_TableLoad_CTAS.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: CTAS Operation for Table Load framework - Execute main usp_TableLoad proc 3 | Example: exec dwa.[usp_TableLoad] 'aw.FactFinance',NULL 4 | exec dwa.[usp_TableLoad] NULL, 7 5 | History: 25/08/2022 Deepak, Created 6 | 17/03/2025 Bob, Tuning 7 | */ 8 | CREATE PROC [dwa].[usp_TableLoad_CTAS] @TableID [int],@TargetObject [sysname],@SourceObject [sysname],@SelectColumns [nvarchar](max),@JoinSQL [nvarchar](max),@SqlWhere [nvarchar](max),@PrestageJoinSQL [nvarchar](max),@DeltaJoinSql [nvarchar](max),@RelatedBusinessKeys [nvarchar](max), @SqlWhereOuter varchar(4000), @InsertColumns varchar(4000), @TablePrefix nvarchar(128), @PrestageTargetFlag bit,@DeleteDDL varchar(4000),@RowChecksum bit AS 9 | BEGIN 10 | SET NOCOUNT ON 11 | DECLARE @sql nvarchar(max) 12 | 13 | SET @sql = 'DROP TABLE IF EXISTS ' + @TargetObject + ';' + CHAR(13) 14 | SET @sql = @Sql + 'CREATE TABLE ' + @TargetObject + CHAR(13) + 'AS' + CHAR(13) 15 | if len(@SqlWhereOuter) > 0 SET @sql =@sql + 'SELECT ' + @InsertColumns + ' FROM ' + CHAR(13) + '(' + CHAR(13) 16 | SET @sql = @sql + 'SELECT ' + @SelectColumns + CHAR(13) 17 | SET @sql = @sql + 'FROM ' + @SourceObject + ' ' + @TablePrefix 18 | 19 | IF coalesce(@RelatedBusinessKeys, '') > '' 20 | BEGIN 21 | SET @sql = @sql + coalesce(CHAR(13) + @JoinSQL, '') 22 | END 23 | 24 | IF @PrestageJoinSQL IS NOT NULL AND @PrestageTargetFlag = 1 25 | SET @sql = @sql + coalesce(CHAR(13) + @PrestageJoinSQL, '') 26 | 27 | IF @DeltaJoinSql IS NOT NULL 28 | SET @sql = @sql + coalesce(CHAR(13) + @DeltaJoinSQL, '') 29 | 30 | IF @PrestageTargetFlag = 0 AND @DeleteDDL IS NOT NULL 31 | SET @SqlWhere = coalesce(@SqlWhere + CHAR(13) + CHAR(9) + ' AND ', 'WHERE ') + 'NOT (' + @TablePrefix + '.' + @DeleteDDL + ')' 32 | 33 | IF @sqlWhere IS NOT NULL 34 | SET @sql = @sql + CHAR(13) + @sqlWhere; 35 | 36 | IF LEN(@SqlWhereOuter ) > 0 SET @sql = @sql + CHAR(13) + ') f' + CHAR(13) + 'WHERE ' + @SqlWhereOuter 37 | 38 | SET @sql=@sql + ';' + char(13) 39 | PRINT @sql 40 | EXEC (@sql) 41 | 42 | END -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/StoredProcedures/usp_TableLoad_Delete.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: DELETE Operation for Table Load framework - Execute main usp_TableLoad proc 3 | Example: exec dwa.[usp_TableLoad] 'aw.FactFinance',NULL 4 | exec dwa.[usp_TableLoad] NULL, 7 5 | History: 26/08/2022 Deepak, Created 6 | */ 7 | CREATE PROC [dwa].[usp_TableLoad_Delete] @TableID int,@SelectColumns nvarchar(max),@JoinSQL nvarchar(max) AS 8 | BEGIN 9 | --SET XACT_ABORT ON 10 | BEGIN 11 | SET NOCOUNT ON 12 | DECLARE @SQL nvarchar(max) 13 | , @SourceObject sysname 14 | , @TargetObject [sysname] 15 | , @TablePrefix nvarchar(max) 16 | , @BusinessKeys nvarchar(max) 17 | , @DeleteDDL nvarchar(4000) 18 | 19 | SELECT @TargetObject = t.SchemaName + '.' + t.TableName, @TablePrefix= coalesce(t.TablePrefix, lower(left(replace(t.TableName, 'Dim',''),1))) 20 | , @SourceObject =t.SourceObject 21 | , @BusinessKeys =t.BusinessKeys 22 | , @DeleteDDL=t.DeleteDDL 23 | FROM Meta.config.edwTables t 24 | WHERE t.TableID = @TableID 25 | 26 | IF @TargetObject IS NULL 27 | BEGIN 28 | RAISERROR ('No Rule found in edwTables for %s', 16, 1, @SourceObject) 29 | END 30 | 31 | SET @SQL ='DELETE t ' + char(13) + 'FROM ' + @TargetObject + ' t' + char(13) 32 | IF @JoinSQL IS NOT NULL 33 | BEGIN 34 | SET @SQL =@SQL + 'INNER JOIN ( SELECT ' + @SelectColumns + char(13) + 'FROM ' + @SourceObject +' ' + @TablePrefix 35 | SET @SQL= coalesce(@SQL + char(13)+ @JoinSQL ,@SQL) 36 | SET @SQL = @SQL + char(13)+' WHERE '+@TablePrefix +'.'+ @DeleteDDL + char(13)+ ') s ON ' 37 | SET @SQL = @SQL + ( SELECT string_agg ('s.' + ltrim(bk.value) + '=t.' + ltrim(bk.value) , ' AND ') 38 | FROM string_split(@BusinessKeys,',') bk ) 39 | END 40 | ELSE 41 | BEGIN 42 | SET @sql = @sql + 'INNER JOIN ' + @SourceObject + ' s ON ' 43 | SET @sql = @sql + ( SELECT string_agg ('s.' + ltrim(bk.value) + '=t.' + ltrim(bk.value) , ' AND ') 44 | FROM string_split(@BusinessKeys,',') bk ) + char(13) + 'AND s.' + @DeleteDDL 45 | END 46 | PRINT @SQL + char(13) 47 | EXEC (@SQL) 48 | 49 | END 50 | END -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/StoredProcedures/usp_cdcSqlBegin.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Begin SQL Server CDC and Return a List of Tables for CDC 3 | Log Target LSN amd Traget Datetime 4 | Test: 5 | exec config.usp_cdcSqlBegin 6 | History: 15/06/2025 Bob, Created 7 | 10/07/2025 Kristan, Added ConfigurationID 8 | 9 | */ 10 | CREATE PROCEDURE [config].[usp_cdcSqlBegin] 11 | ( 12 | @connectionId varchar(50) = '8a10eff2-cdcc-4b00-a2ef-35bae56d7ef2', 13 | @database sysname = 'HICPStoreOne', 14 | @target_lsn varchar(50) = '0x0005E0CA0006EDD90040', 15 | @target_datetime varchar(50) = '2025-06-15T20:21:40.757Z', 16 | @table sysname = null -- Optional if Just One Table 17 | ) 18 | AS 19 | BEGIN 20 | SET NOCOUNT ON; 21 | 22 | DECLARE @target_datetime2 datetime2(7); 23 | DECLARE @target_lsn_bin binary(10); 24 | DECLARE @ConfigurationID int; 25 | 26 | SELECT 27 | @target_datetime2 = CONVERT(datetime2(2), @target_datetime), 28 | @target_lsn_bin = CONVERT(binary(10), @target_lsn, 1); 29 | 30 | -- Get ConfigurationID from config.Configuration using ConnectionID 31 | SELECT @ConfigurationID = ConfigurationID 32 | FROM config.Configurations 33 | WHERE JSON_VALUE(ConnectionSettings, '$.connectionId') = @connectionId; 34 | 35 | 36 | UPDATE config.cdcSqlTables 37 | SET target_lsn = @target_lsn_bin, 38 | target_datetime = @target_datetime2 39 | WHERE ConfigurationID = @ConfigurationID 40 | AND [Database] = @database 41 | AND (@table IS NULL OR [table] = @table) 42 | AND Enabled = 1; 43 | 44 | SELECT 45 | [Database], 46 | [Table], 47 | CONVERT(varchar(50), max_lsn, 1) AS from_lsn, 48 | max_datetime AS from_datetime, 49 | CONVERT(varchar(50), target_lsn, 1) AS to_lsn, 50 | target_datetime AS to_datetime, 51 | PrimaryKeys 52 | FROM config.cdcSqlTables 53 | WHERE ConfigurationID = @ConfigurationID 54 | AND [Database] = @database 55 | AND (@table IS NULL OR [Table] = @table) 56 | AND Enabled = 1; 57 | END; 58 | 59 | GO 60 | 61 | -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Config/Environment-Variables.DataPipeline/pipeline-content.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": { 3 | "activities": [ 4 | { 5 | "name": "Set Environment Values", 6 | "type": "SetVariable", 7 | "dependsOn": [], 8 | "policy": { 9 | "secureOutput": false, 10 | "secureInput": false 11 | }, 12 | "typeProperties": { 13 | "variableName": "pipelineReturnValue", 14 | "value": [ 15 | { 16 | "key": "MetaServerName", 17 | "value": { 18 | "type": "String", 19 | "content": "fkm4vwf6l6zebg4lqrhbtdcmsq-yctecwmyrr4u3mdfun4j5hqjma.database.fabric.microsoft.com" 20 | } 21 | }, 22 | { 23 | "key": "MetaDatabaseName", 24 | "value": { 25 | "type": "String", 26 | "content": "Meta-fe70c606-af27-4f64-973a-2be877526212" 27 | } 28 | }, 29 | { 30 | "key": "WorkspaceID", 31 | "value": { 32 | "type": "String", 33 | "content": "5941a6c0-8c98-4d79-b065-a3789e9e0960" 34 | } 35 | }, 36 | { 37 | "key": "DWServerName", 38 | "value": { 39 | "type": "String", 40 | "content": "fkm4vwf6l6zebg4lqrhbtdcmsq-yctecwmyrr4u3mdfun4j5hqjma.datawarehouse.fabric.microsoft.com" 41 | } 42 | }, 43 | { 44 | "key": "DWDatabaseName", 45 | "value": { 46 | "type": "String", 47 | "content": "DW" 48 | } 49 | }, 50 | { 51 | "key": "LakehouseID", 52 | "value": { 53 | "type": "String", 54 | "content": "d58f4f2d-59d7-406d-ae4c-898354a6a75f" 55 | } 56 | }, 57 | { 58 | "key": "MetaDatabaseID", 59 | "value": { 60 | "type": "String", 61 | "content": "fe70c606-af27-4f64-973a-2be877526212" 62 | } 63 | } 64 | ], 65 | "setSystemVariable": true 66 | } 67 | } 68 | ] 69 | } 70 | } -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/edwTables.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[edwTables] ( 2 | [TableID] SMALLINT NOT NULL, 3 | [SchemaName] VARCHAR (128) NULL, 4 | [TableName] VARCHAR (128) NULL, 5 | [LoadDescription] VARCHAR (512) NULL, 6 | [TableType] VARCHAR (20) NULL, 7 | [SourceObject] VARCHAR (8000) NULL, 8 | [SourceType] VARCHAR (20) NULL, 9 | [BusinessKeys] VARCHAR (512) NULL, 10 | [PrimaryKey] VARCHAR (512) NULL, 11 | [DefaultPackageGroup] VARCHAR (50) NULL, 12 | [TablePrefix] VARCHAR (10) NULL, 13 | [Enabled] BIT NULL, 14 | [InsertFlag] BIT NULL, 15 | [UpdateFlag] BIT NULL, 16 | [DeleteFlag] BIT NULL, 17 | [DedupeFlag] BIT NULL, 18 | [DedupeOrderBy] VARCHAR (4000) NULL, 19 | [AutoDrop] BIT NULL, 20 | [CTAS] BIT NULL, 21 | [AutoTruncate] BIT NULL, 22 | [DeleteDDL] VARCHAR (8000) NULL, 23 | [SkipSqlCondition] VARCHAR (8000) NULL, 24 | [PrestageSourceFlag] BIT NULL, 25 | [PrestageTargetFlag] BIT NULL, 26 | [PrestageSchema] VARCHAR (8000) NULL, 27 | [TableSwapFlag] BIT NULL, 28 | [DeltaTargetObject] VARCHAR (8000) NULL, 29 | [PreLoadSQL] VARCHAR (8000) NULL, 30 | [PostLoadSQL] VARCHAR (8000) NULL, 31 | [WhereSQL] VARCHAR (8000) NULL, 32 | [JoinType] VARCHAR (8000) NULL, 33 | [Language] VARCHAR (8000) NULL, 34 | [SchemaDrift] BIT NULL, 35 | [IsIncremental] BIT NULL, 36 | [SCD] BIT NULL, 37 | [Identity] BIT NULL, 38 | [IdentityMethod] VARCHAR (20) NULL, 39 | [RowChecksum] BIT NULL, 40 | CONSTRAINT [PK_edwTables] PRIMARY KEY CLUSTERED ([TableID] ASC), 41 | CONSTRAINT [FK_edwTables_PackageGroups] FOREIGN KEY ([DefaultPackageGroup]) REFERENCES [config].[PackageGroups] ([PackageGroup]) 42 | ); 43 | 44 | 45 | GO 46 | 47 | -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/definition/tables/Date.tmdl: -------------------------------------------------------------------------------- 1 | table Date 2 | lineageTag: 08015f70-7d70-4eda-80f2-1a924d8de049 3 | 4 | column DateKey 5 | dataType: string 6 | isHidden 7 | lineageTag: 452d8412-eb71-4501-b1ec-22a6d8f7b4fc 8 | summarizeBy: none 9 | sourceColumn: DateKey 10 | 11 | changedProperty = IsHidden 12 | 13 | annotation SummarizationSetBy = Automatic 14 | 15 | column Date 16 | dataType: dateTime 17 | formatString: General Date 18 | lineageTag: 0a13bfa2-1189-4d81-8608-65ae9b78f216 19 | summarizeBy: none 20 | sourceColumn: Date 21 | 22 | annotation SummarizationSetBy = Automatic 23 | 24 | column Month 25 | dataType: string 26 | lineageTag: 28ef5be9-3dfc-4503-8cb9-539a296a336f 27 | summarizeBy: none 28 | sourceColumn: Month 29 | 30 | annotation SummarizationSetBy = Automatic 31 | 32 | column 'Fiscal Quarter' 33 | dataType: int64 34 | formatString: 0 35 | lineageTag: db86bae6-7d8c-42a3-b206-61401570b019 36 | summarizeBy: none 37 | sourceColumn: Fiscal Quarter 38 | 39 | annotation SummarizationSetBy = Automatic 40 | 41 | column 'Fiscal Year' 42 | dataType: int64 43 | formatString: 0 44 | lineageTag: 9dda6ff0-3ae1-4a2d-872e-39fe7f726cb8 45 | summarizeBy: none 46 | sourceColumn: Fiscal Year 47 | 48 | annotation SummarizationSetBy = Automatic 49 | 50 | column 'Fiscal Period No' 51 | dataType: int64 52 | formatString: 0 53 | lineageTag: 8895cdea-fcab-46de-aba1-5f70a2ab4f87 54 | summarizeBy: none 55 | sourceColumn: Fiscal Period No 56 | 57 | annotation SummarizationSetBy = Automatic 58 | 59 | column 'Fiscal Period' 60 | dataType: string 61 | lineageTag: 6f0912d9-5ce9-42f7-8db8-c183743322a2 62 | summarizeBy: none 63 | sourceColumn: Fiscal Period 64 | 65 | annotation SummarizationSetBy = Automatic 66 | 67 | partition Date-09ab7aeb-ed6f-4f1a-ae77-5e5a9261c99b = m 68 | mode: import 69 | queryGroup: 'Finance GL' 70 | source = 71 | let 72 | Source= Sql.Database("fkm4vwf6l6zebg4lqrhbtdcmsq-yctecwmyrr4u3mdfun4j5hqjma.datawarehouse.fabric.microsoft.com", "DW", [Query="SELECT DateKey, Date, Month, [Fiscal Quarter], [Fiscal Year], [Fiscal Period No], [Fiscal Period] FROM reports.DimDate"]) 73 | in 74 | Source 75 | 76 | annotation PBI_ResultType = Table 77 | 78 | annotation PBI_NavigationStepName = Navigation 79 | 80 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/reports/Views/Account.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) 62926774E06E2413D78F6491BEAB8E6D6F7FFD54237DC5D1D73D6B7D212965D4 2 | 3 | 4 | /* Description: AW Account for PowerBI Direct Lake Model 5 | 6 | History: 7 | 21/02/2025 Created 8 | */ 9 | CREATE VIEW [reports].[Account] 10 | AS 11 | WITH AccPath AS 12 | ( 13 | SELECT TRIM('|' FROM COALESCE(a3.AccountDescription, '') + '|' + COALESCE(a2.AccountDescription, '') + '|' + COALESCE(a1.AccountDescription, '')) AS AccountPath 14 | ,a1.AccountKey, a1.ParentAccountKey, a1.AccountCodeAlternateKey, a1.ParentAccountCodeAlternateKey, a1.AccountDescription, a1.AccountType, a1.Operator, a1.CustomMembers, a1.ValueType, a1.CustomMemberOptions 15 | FROM aw.DimAccount a1 16 | LEFT JOIN aw.DimAccount a2 17 | ON a1.ParentAccountKey = a2.AccountKey 18 | LEFT JOIN aw.DimAccount a3 19 | ON a2.ParentAccountKey = a3.AccountKey 20 | ), 21 | DelPos AS 22 | ( 23 | SELECT CHARINDEX('|', AccountPath) AS Delimiter1 24 | ,CHARINDEX('|', AccountPath, CHARINDEX('|', AccountPath) + 1) AS Delimiter2 25 | ,CHARINDEX('|', AccountPath, CHARINDEX('|', AccountPath, CHARINDEX('|', AccountPath) + 1) + 1) AS Delimiter3 26 | ,AccountPath, AccountKey, ParentAccountKey, AccountCodeAlternateKey, ParentAccountCodeAlternateKey, AccountDescription, AccountType, Operator, CustomMembers, ValueType, CustomMemberOptions 27 | FROM AccPath 28 | ) 29 | SELECT AccountKey, ParentAccountKey 30 | ,AccountCodeAlternateKey AS [Account Code] 31 | ,AccountDescription AS Account 32 | ,AccountType AS [Account Type], Operator 33 | ,ValueType AS [Value Type] 34 | ,CASE WHEN Delimiter1 = 0 THEN AccountPath 35 | ELSE SUBSTRING(AccountPath, 1, Delimiter1 - 1) 36 | END AS Report 37 | ,CASE WHEN Delimiter1 = 0 THEN '' 38 | WHEN Delimiter2 = 0 THEN SUBSTRING(AccountPath, Delimiter1 + 1, LEN(AccountPath) - Delimiter1 ) 39 | ELSE SUBSTRING(AccountPath, Delimiter1 + 1, Delimiter2 - Delimiter1 -1) 40 | END AS [Account L2] 41 | ,CASE WHEN Delimiter1 = 0 OR Delimiter2 = 0 THEN '' 42 | WHEN Delimiter3 = 0 THEN SUBSTRING(AccountPath, Delimiter2 + 1, LEN(AccountPath) - Delimiter2) 43 | ELSE SUBSTRING(AccountPath, Delimiter2 + 1, Delimiter3 - Delimiter2 - 1) 44 | END [Account L3] 45 | FROM DelPos -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/reports/Views/DimAccount.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) 62926774E06E2413D78F6491BEAB8E6D6F7FFD54237DC5D1D73D6B7D212965D4 2 | 3 | 4 | 5 | 6 | /* Description: AW DimDepartmentGroup for PowerBI Model 7 | 8 | History: 9 | 21/02/2025 Created 10 | */ 11 | CREATE VIEW [reports].[DimAccount] 12 | AS 13 | WITH AccPath AS 14 | ( 15 | SELECT TRIM('|' FROM COALESCE(a3.AccountDescription, '') + '|' + COALESCE(a2.AccountDescription, '') + '|' + COALESCE(a1.AccountDescription, '')) AS AccountPath 16 | ,a1.AccountKey, a1.ParentAccountKey, a1.AccountCodeAlternateKey, a1.ParentAccountCodeAlternateKey, a1.AccountDescription, a1.AccountType, a1.Operator, a1.CustomMembers, a1.ValueType, a1.CustomMemberOptions 17 | FROM aw.DimAccount a1 18 | LEFT JOIN aw.DimAccount a2 19 | ON a1.ParentAccountKey = a2.AccountKey 20 | LEFT JOIN aw.DimAccount a3 21 | ON a2.ParentAccountKey = a3.AccountKey 22 | ), 23 | DelPos AS 24 | ( 25 | SELECT CHARINDEX('|', AccountPath) AS Delimiter1 26 | ,CHARINDEX('|', AccountPath, CHARINDEX('|', AccountPath) + 1) AS Delimiter2 27 | ,CHARINDEX('|', AccountPath, CHARINDEX('|', AccountPath, CHARINDEX('|', AccountPath) + 1) + 1) AS Delimiter3 28 | ,AccountPath, AccountKey, ParentAccountKey, AccountCodeAlternateKey, ParentAccountCodeAlternateKey, AccountDescription, AccountType, Operator, CustomMembers, ValueType, CustomMemberOptions 29 | FROM AccPath 30 | ) 31 | SELECT AccountKey, ParentAccountKey 32 | ,AccountCodeAlternateKey AS [Account Code] 33 | ,AccountDescription AS Account 34 | ,AccountType AS [Account Type], Operator 35 | ,ValueType AS [Value Type] 36 | ,CASE WHEN Delimiter1 = 0 THEN AccountPath 37 | ELSE SUBSTRING(AccountPath, 1, Delimiter1 - 1) 38 | END AS Report 39 | ,CASE WHEN Delimiter1 = 0 THEN '' 40 | WHEN Delimiter2 = 0 THEN SUBSTRING(AccountPath, Delimiter1 + 1, LEN(AccountPath) - Delimiter1 ) 41 | ELSE SUBSTRING(AccountPath, Delimiter1 + 1, Delimiter2 - Delimiter1 -1) 42 | END AS [Account L2] 43 | ,CASE WHEN Delimiter1 = 0 OR Delimiter2 = 0 THEN '' 44 | WHEN Delimiter3 = 0 THEN SUBSTRING(AccountPath, Delimiter2 + 1, LEN(AccountPath) - Delimiter2) 45 | ELSE SUBSTRING(AccountPath, Delimiter2 + 1, Delimiter3 - Delimiter2 - 1) 46 | END [Account L3] 47 | FROM DelPos -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/definition/tables/Department.tmdl: -------------------------------------------------------------------------------- 1 | table Department 2 | lineageTag: 5336f1dc-c705-4e46-b65d-20ae702fa13c 3 | 4 | column DepartmentGroupKey 5 | dataType: string 6 | isHidden 7 | lineageTag: 76789a93-3b7a-4ff5-87d4-96546a37f2b9 8 | summarizeBy: none 9 | sourceColumn: DepartmentGroupKey 10 | 11 | changedProperty = IsHidden 12 | 13 | annotation SummarizationSetBy = Automatic 14 | 15 | column ParentDepartmentGroupKey 16 | dataType: string 17 | isHidden 18 | lineageTag: 19cfe0cd-f4d3-45a7-a957-cc6376de79cb 19 | summarizeBy: none 20 | sourceColumn: ParentDepartmentGroupKey 21 | 22 | changedProperty = IsHidden 23 | 24 | annotation SummarizationSetBy = Automatic 25 | 26 | column 'Department Name' 27 | dataType: string 28 | lineageTag: 5b490be6-7334-464f-b58e-c31220d40d97 29 | summarizeBy: none 30 | sourceColumn: Department Name 31 | 32 | annotation SummarizationSetBy = Automatic 33 | 34 | column 'Department Group' 35 | dataType: string 36 | lineageTag: ae714ca4-53a0-4a1d-bb1c-4317cb7250ac 37 | summarizeBy: none 38 | sourceColumn: Department Group 39 | 40 | annotation SummarizationSetBy = Automatic 41 | 42 | column Department 43 | dataType: string 44 | lineageTag: 73853b1e-d096-4fa3-aff8-3031df99b56e 45 | summarizeBy: none 46 | sourceColumn: Department 47 | 48 | annotation SummarizationSetBy = Automatic 49 | 50 | hierarchy 'Department Hierarchy' 51 | lineageTag: 81e19cf3-d056-4e73-8e6e-4901f8de099e 52 | 53 | level 'Department Group' 54 | lineageTag: 5d23b741-b189-4070-9f67-55c89750504f 55 | column: 'Department Group' 56 | 57 | level 'Sub Department' 58 | lineageTag: 79cd2fd8-d8d4-49cd-9cf4-8ae2cdcb406f 59 | column: Department 60 | 61 | partition Department-d6337107-b576-462b-b572-d473a86f9f30 = m 62 | mode: import 63 | queryGroup: 'Finance GL' 64 | source = 65 | let 66 | dbo_DimDepartmentGroup = Sql.Database("fkm4vwf6l6zebg4lqrhbtdcmsq-yctecwmyrr4u3mdfun4j5hqjma.datawarehouse.fabric.microsoft.com", "DW", [Query="SELECT DepartmentGroupKey, ParentDepartmentGroupKey, [Department Name], [Department Group], Department FROM reports.DimDepartmentGroup"]) 67 | in 68 | dbo_DimDepartmentGroup 69 | 70 | annotation PBI_ResultType = Table 71 | 72 | annotation LinkedQueryName = Department 73 | 74 | annotation PBI_NavigationStepName = Navigation 75 | 76 | -------------------------------------------------------------------------------- /Setup/Files/AW/Currency.csv: -------------------------------------------------------------------------------- 1 | CurrencyCode,CurrencyName 2 | AFA,Afghani 3 | DZD,Algerian Dinar 4 | ARS,Argentine Peso 5 | AMD,Armenian Dram 6 | AWG,Aruban Guilder 7 | AUD,Australian Dollar 8 | AZM,Azerbaijanian Manat 9 | BSD,Bahamian Dollar 10 | BHD,Bahraini Dinar 11 | THB,Baht 12 | PAB,Balboa 13 | BBD,Barbados Dollar 14 | BEF,Belgian Franc 15 | VEB,Bolivar 16 | BOB,Boliviano 17 | BRL,Brazilian Real 18 | BND,Brunei Dollar 19 | BGN,Bulgarian Lev 20 | CAD,Canadian Dollar 21 | GHC,Cedi 22 | XOF,CFA Franc BCEAO 23 | CLP,Chilean Peso 24 | COP,Colombian Peso 25 | CRC,Costa Rican Colon 26 | HRK,Croatian Kuna 27 | CYP,Cyprus Pound 28 | CZK,Czech Koruna 29 | DKK,Danish Krone 30 | DEM,Deutsche Mark 31 | DOP,Dominican Peso 32 | VND,Dong 33 | GRD,Drachma 34 | EGP,Egyptian Pound 35 | SVC,El Salvador Colon 36 | AED,Emirati Dirham 37 | EUR,EURO 38 | FJD,Fiji Dollar 39 | HUF,Forint 40 | FRF,French Franc 41 | PYG,Guarani 42 | HKD,Hong Kong Dollar 43 | ISK,Iceland Krona 44 | INR,Indian Rupee 45 | IEP,Irish Pound 46 | ITL,Italian Lira 47 | JMD,Jamaican Dollar 48 | JOD,Jordanian Dinar 49 | KES,Kenyan Shilling 50 | EEK,Kroon 51 | KWD,Kuwaiti Dinar 52 | AOA,Kwanza 53 | LVL,Latvian Lats 54 | LBP,Lebanese Pound 55 | ALL,Lek 56 | ROL,Leu 57 | LTL,Lithuanian Litas 58 | MYR,Malaysian Ringgit 59 | MTL,Maltese Lira 60 | FIM,Markka 61 | MUR,Mauritius Rupee 62 | MXN,Mexican Peso 63 | MAD,Moroccan Dirham 64 | NGN,Naira 65 | NAD,Namibia Dollar 66 | NPR,Nepalese Rupee 67 | ANG,Netherlands Antillian Guilder 68 | NLG,Netherlands Guilder 69 | ILS,New Israeli Shekel 70 | TWD,New Taiwan Dollar 71 | NZD,New Zealand Dollar 72 | BTN,Ngultrum 73 | NOK,Norwegian Krone 74 | PEN,Nuevo Sol 75 | OMR,Omani Rial 76 | PKR,Pakistan Rupee 77 | PHP,Philippine Peso 78 | PLZ,Polish Zloty(old) 79 | PTE,Portuguese Escudo 80 | GTQ,Quetzal 81 | ZAR,Rand 82 | MVR,Rufiyaa 83 | IDR,Rupiah 84 | RUB,Russian Ruble 85 | RUR,Russian Ruble(old) 86 | SAR,Saudi Riyal 87 | ATS,Shilling 88 | SGD,Singapore Dollar 89 | SKK,Slovak Koruna 90 | ESP,Spanish Peseta 91 | LKR,Sri Lankan Rupee 92 | SEK,Swedish Krona 93 | CHF,Swiss Franc 94 | BDT,Taka 95 | SIT,Tolar 96 | TTD,Trinidad and Tobago Dollar 97 | TND,Tunisian Dinar 98 | TRL,Turkish Lira 99 | GBP,United Kingdom Pound 100 | UYU,Uruguayan Peso 101 | USD,US Dollar 102 | KRW,Won 103 | JPY,Yen 104 | CNY,Yuan Renminbi 105 | ZWD,Zimbabwe Dollar 106 | PLN,Zloty 107 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/StoredProcedures/usp_TableLoad_Insert.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: INSERT Operation for Table Load framework - Execute main usp_TableLoad proc 3 | Example: exec dwa.[usp_TableLoad] 'aw.FactFinance',NULL,NULL 4 | exec dwa.[usp_TableLoad] NULL, 1,NULL 5 | History: 03/08/2023 Deepak, Created 6 | */ 7 | CREATE PROC [dwa].[usp_TableLoad_Insert] @TableID [int],@TargetObject [sysname],@SourceObject [sysname],@InsertColumns [nvarchar](max),@SelectColumns [nvarchar](max),@JoinSQL [nvarchar](max),@SqlWhere [nvarchar](max),@WhereJoinSQL [nvarchar](max),@Exists [bit], @TablePrefix sysname,@BusinessKeys varchar(4000), @PrestageTargetFlag bit,@PrestageTargetObject sysname, @SqlWhereOuter varchar(4000) AS 8 | BEGIN 9 | SET NOCOUNT ON 10 | DECLARE @sql nvarchar(max) 11 | SET @sqlWhere = COALESCE(@sqlWhere,'') 12 | 13 | SET @SQL ='INSERT INTO ' + @TargetObject + ' (' + @InsertColumns + ')' + char(13) 14 | IF len(@SqlWhereOuter) > 0 or (@BusinessKeys is not null and @Exists =1) SET @SQL =@SQL + 'SELECT ' + @InsertColumns + ' FROM (' + char(13) 15 | IF @PrestageTargetFlag=1 16 | SET @SQL =@SQL + 'SELECT ' + @InsertColumns + char(13) + 'FROM ' + @SourceObject + ' '+ @TablePrefix 17 | ELSE 18 | SET @SQL =@SQL + 'SELECT ' + @SelectColumns + char(13) + 'FROM ' + @SourceObject + ' '+ @TablePrefix 19 | SET @SQL= coalesce(@SQL + char(13)+ @JoinSQL ,@SQL) 20 | 21 | IF len(@SqlWhereOuter) > 0 or (@BusinessKeys is not null and @Exists =1) SET @SQL =@SQL + char(13) + @sqlWhere + ' ) ' + @TablePrefix 22 | SET @JoinSQL =null 23 | IF @BusinessKeys is not null and @Exists =1 24 | BEGIN 25 | SELECT @JoinSQL = coalesce('WHERE ' + @SqlWhereOuter + CHAR(13) + 'AND ' , 'WHERE ' ) + 'NOT EXISTS (SELECT * FROM ' + @TargetObject + ' t WHERE ' 26 | IF @WhereJoinSQL IS NOT NULL 27 | SELECT @JoinSQL = @JoinSQL + (SELECT string_agg(rtrim(bk.value), ' AND ') + ')' FROM string_split(rtrim(@WhereJoinSQl), char(13)) bk) 28 | ELSE 29 | SELECT @JoinSQL = @JoinSQL + ( 30 | SELECT string_agg(@TablePrefix + '.' + ltrim(bk.value) + '=t.' + ltrim(bk.value), ' AND ') + ')' 31 | FROM string_split(@BusinessKeys, ',') bk 32 | ) 33 | IF @JoinSQL is not null SET @sql= @sql + char(13) + @JoinSQL 34 | END 35 | 36 | IF @JoinSQL IS NULL AND @sqlWhere IS NOT NULL SET @sql=@sql + char(13) + @sqlWhere 37 | SET @SQL=@SQL + ';' +char(13) 38 | PRINT @sql 39 | EXEC (@sql) 40 | END -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/StoredProcedures/usp_TruncateAll.sql: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | Truncate ALL Config Tables. 4 | Only run this for deployment or recovery 5 | 15-01-2025 Shruti, Updated Query to return Schema Name within CTE 6 | */ 7 | CREATE PROCEDURE [config].[usp_TruncateAll] 8 | AS 9 | BEGIN 10 | SET NOCOUNT ON 11 | DECLARE @sql nvarchar(max) 12 | 13 | -- DIASBLE Triggers 14 | DROP TABLE IF EXISTS #tr 15 | SELECT t.Name AS TriggerName, o.Name AS TableName 16 | INTO #tr 17 | FROM sys.triggers t 18 | INNER JOIN sys.objects o ON t.parent_id = o.object_id 19 | INNER JOIN sys.schemas s ON o.schema_id = s.schema_id 20 | WHERE s.name = 'config' AND t.type = 'TR' 21 | SELECT @sql = STRING_AGG ('DISABLE TRIGGER [' + TriggerName + '] ON [config].[' + Tablename + ']', ';') 22 | FROM #tr 23 | print @sql 24 | 25 | exec (@sql) 26 | 27 | --DELETE Data from tables 28 | ;WITH cte(lvl, object_id, name, SchemaName) AS (SELECT 1 AS Expr1, object_id, name, SCHEMA_NAME(schema_id) AS SchemaName 29 | FROM sys.tables 30 | WHERE (type_desc = 'USER_TABLE') AND (is_ms_shipped = 0) and name <> 'sysdiagrams' and SCHEMA_NAME(schema_id)='config' 31 | UNION ALL 32 | SELECT cte_2.lvl + 1 AS Expr1, t.object_id, t.name, SCHEMA_NAME(schema_id) AS SchemaName 33 | FROM cte AS cte_2 INNER JOIN 34 | sys.tables AS t ON EXISTS 35 | (SELECT NULL AS Expr1 36 | FROM sys.foreign_keys AS fk 37 | WHERE (parent_object_id = t.object_id) AND (referenced_object_id = cte_2.object_id)) AND t.object_id <> cte_2.object_id AND cte_2.lvl < 30 38 | WHERE (t.type_desc = 'USER_TABLE') AND (t.is_ms_shipped = 0) ) 39 | , t as ( 40 | SELECT TOP (100) PERCENT SchemaName, name, MAX(lvl) AS dependency_level 41 | FROM cte AS cte_1 42 | GROUP BY SchemaName, name 43 | ORDER BY dependency_level, name 44 | ) 45 | SELECT @sql = STRING_AGG ('DELETE FROM ' + SchemaName + '.' + name,';') WITHIN GROUP (ORDER BY dependency_level DESC, name DESC) 46 | FROM t 47 | print @sql 48 | 49 | exec (@sql) 50 | 51 | --ENABLE Triggers after deletion is completed 52 | SELECT @sql = STRING_AGG ('ENABLE TRIGGER [' + TriggerName + '] ON [config].[' + Tablename + ']', ';') 53 | FROM #tr 54 | print @sql 55 | 56 | exec (@sql) 57 | 58 | END 59 | 60 | GO 61 | 62 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/Tables/Pipelines.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [config].[Pipelines] ( 2 | [PipelineID] INT NOT NULL, 3 | [Pipeline] VARCHAR (200) NULL, 4 | [PipelineGroupID] INT NULL, 5 | [PipelineSequence] INT NULL, 6 | [SourceSettings] VARCHAR (8000) NULL, 7 | [TargetSettings] VARCHAR (8000) NULL, 8 | [ActivitySettings] VARCHAR (8000) NULL, 9 | [Enabled] INT CONSTRAINT [DF_Pipelines_Enabled] DEFAULT ((1)) NOT NULL, 10 | [PackageGroup] VARCHAR (50) NULL, 11 | [TableID] SMALLINT NULL, 12 | [Stage] VARCHAR (50) NULL, 13 | [PostExecuteSQL] VARCHAR (8000) NULL, 14 | [Template] VARCHAR (200) NULL, 15 | [PreExecuteSQL] VARCHAR (8000) NULL, 16 | [SourceConnectionSettings] VARCHAR (8000) NULL, 17 | [TargetConnectionSettings] VARCHAR (8000) NULL, 18 | [SourceConfigurationID] INT NULL, 19 | [TargetConfigurationID] INT NULL, 20 | [ContinueOnError] VARCHAR (8000) NULL, 21 | [Comments] VARCHAR (200) NULL, 22 | [SessionTag] VARCHAR (50) NULL, 23 | [LakehouseConfigurationID] INT NULL, 24 | [PromptID] INT NULL, 25 | CONSTRAINT [PK_Pipelines] PRIMARY KEY CLUSTERED ([PipelineID] ASC), 26 | CONSTRAINT [FK_Pipelines_Configurations] FOREIGN KEY ([SourceConfigurationID]) REFERENCES [config].[Configurations] ([ConfigurationID]), 27 | CONSTRAINT [FK_Pipelines_PackageGroups] FOREIGN KEY ([PackageGroup]) REFERENCES [config].[PackageGroups] ([PackageGroup]), 28 | CONSTRAINT [FK_Pipelines_PipelineGroups] FOREIGN KEY ([PipelineGroupID]) REFERENCES [config].[PipelineGroups] ([PipelineGroupID]), 29 | CONSTRAINT [FK_Pipelines_Templates] FOREIGN KEY ([Template]) REFERENCES [config].[Templates] ([Template]) 30 | ); 31 | 32 | 33 | GO 34 | 35 | CREATE TRIGGER [config].[TR_Pipelines] 36 | ON [config].[Pipelines] 37 | AFTER INSERT,DELETE,UPDATE 38 | AS 39 | BEGIN 40 | SET NOCOUNT ON; 41 | 42 | UPDATE p 43 | SET p.ActivitySettings = ai.ActivitySettings 44 | FROM config.Pipelines p 45 | INNER JOIN inserted i on i.PipelineID=p.PipelineID 46 | INNER JOIN config.aiPrompts ai on ai.PromptID = p.PromptID 47 | 48 | exec [config].[usp_PipelineBuildMetaData] 49 | 50 | END 51 | 52 | GO 53 | 54 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/reports/Views/DimOrganization.sql: -------------------------------------------------------------------------------- 1 | -- Auto Generated (Do not modify) F70E5A83F1A2D3583F6C5E522F82F689A99D8F217E02FAF0E470C159B2F3FE2D 2 | 3 | 4 | 5 | 6 | /* Description: AW DimDepartmentGroup for PowerBI Model 7 | 8 | History: 9 | 21/02/2025 Created 10 | */ 11 | CREATE VIEW [reports].[DimOrganization] 12 | AS 13 | WITH OrgPath AS 14 | ( 15 | SELECT TRIM('|' FROM COALESCE(o4.OrganizationName, '') + '|' + COALESCE(o3.OrganizationName, '') + '|' + COALESCE(o2.OrganizationName, '') + '|' + COALESCE(o1.OrganizationName, '')) AS OrganizationPath, 16 | o1.OrganizationKey, o1.ParentOrganizationKey, o1.PercentageOfOwnership, o1.OrganizationName, o1.CurrencyKey 17 | FROM 18 | aw.DimOrganization o1 19 | LEFT JOIN 20 | aw.DimOrganization o2 ON o1.ParentOrganizationKey = o2.OrganizationKey 21 | LEFT JOIN 22 | aw.DimOrganization o3 ON o2.ParentOrganizationKey = o3.OrganizationKey 23 | LEFT JOIN 24 | aw.DimOrganization o4 ON o3.ParentOrganizationKey = o4.OrganizationKey 25 | ), 26 | DelPos AS 27 | ( 28 | SELECT CHARINDEX('|', OrganizationPath) AS Delimiter1 29 | ,CHARINDEX('|', OrganizationPath, CHARINDEX('|', OrganizationPath) + 1) AS Delimiter2 30 | ,CHARINDEX('|', OrganizationPath, CHARINDEX('|', OrganizationPath, CHARINDEX('|', OrganizationPath) + 1) + 1) AS Delimiter3 31 | ,OrganizationPath, OrganizationKey, ParentOrganizationKey, PercentageOfOwnership, OrganizationName, CurrencyKey 32 | FROM OrgPath 33 | ) 34 | SELECT OrganizationKey, ParentOrganizationKey, PercentageOfOwnership, OrganizationName AS [Organization Name], CurrencyKey 35 | ,CASE WHEN Delimiter1 = 0 THEN OrganizationPath 36 | ELSE SUBSTRING(OrganizationPath, 1, Delimiter1 - 1) 37 | END AS Company 38 | ,CASE WHEN Delimiter1 = 0 THEN '' 39 | WHEN Delimiter2 = 0 THEN SUBSTRING(OrganizationPath, Delimiter1 + 1, LEN(OrganizationPath) - Delimiter1 ) 40 | ELSE SUBSTRING(OrganizationPath, Delimiter1 + 1, Delimiter2 - Delimiter1 -1) 41 | END AS Region 42 | ,CASE WHEN Delimiter1 = 0 OR Delimiter2 = 0 THEN '' 43 | WHEN Delimiter3 = 0 THEN SUBSTRING(OrganizationPath, Delimiter2 + 1, LEN(OrganizationPath) - Delimiter2) 44 | ELSE SUBSTRING(OrganizationPath, Delimiter2 + 1, Delimiter3 - Delimiter2 - 1) 45 | END Country 46 | ,CASE WHEN Delimiter1 = 0 OR Delimiter2 = 0 OR Delimiter3 = 0 THEN '' 47 | ELSE SUBSTRING(OrganizationPath, Delimiter3 + 1, LEN(OrganizationPath) - Delimiter3) 48 | END Division 49 | FROM DelPos -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/dbo/Functions/ufn_BuildChatPayloadJson.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Retutn the JSon Payload for Both the v1 completions API or the V2 Responses API for AI LLM Calls as per: 3 | https://platform.openai.com/docs/api-reference/completions 4 | https://platform.openai.com/docs/api-reference/responses 5 | 6 | 7 | History: 06/11/2025, Bob, Completed 8 | 9 | */ 10 | CREATE FUNCTION [dbo].[ufn_BuildChatPayloadJson]( 11 | @model varchar(4000)=null 12 | ,@api_family varchar (4000) = 'chat' /* chat | responses */ 13 | ,@instructions varchar(max)=null 14 | ,@input varchar(max)=null 15 | ,@json_schema varchar(max)=null 16 | ,@temperature float=null 17 | ,@ExtraSettings varchar(max)=null 18 | ) 19 | RETURNS varchar(max) 20 | WITH SCHEMABINDING 21 | AS 22 | BEGIN 23 | DECLARE @Json varchar(max) 24 | 25 | ;WITH keys AS ( 26 | SELECT 'model' AS [key], @model AS [value], '1' AS [type] WHERE @model IS NOT NULL 27 | UNION ALL 28 | SELECT 'messages' AS [key], 29 | '[ ' 30 | + COALESCE('{"role": "system", "content": "' + @instructions + '"},', '') 31 | + '{"role": "user", "content": "' + @input + '"}' 32 | + ']' AS [value], 33 | '4' AS [type] 34 | WHERE @input IS NOT NULL AND @api_family='chat' 35 | UNION ALL 36 | SELECT 'response_format' AS [key], 37 | '{"type": "json_schema","json_schema":' + @json_schema + '}' AS [value], 38 | '4' AS [type] 39 | WHERE @json_schema IS NOT NULL and @api_family='chat' 40 | UNION ALL 41 | SELECT 'text' AS [key], '{"format": {"type": "json_schema","name": "text_schema","schema": ' + @json_schema+ ',"strict": true}}' 42 | , '4' AS [type] 43 | WHERE @json_schema IS NOT NULL AND @api_family<>'chat' 44 | UNION ALL 45 | SELECT 'instructions' AS [key], @instructions, '1' as [type] 46 | WHERE @instructions IS NOT NULL and @api_family<>'chat' 47 | UNION ALL 48 | SELECT 'input' AS [key], @input, '1' as [type] 49 | WHERE @input IS NOT NULL and @api_family<>'chat' 50 | UNION ALL 51 | SELECT 'temperature' AS [key], CONVERT(varchar, @temperature) AS [value], '2' AS [type] 52 | WHERE @temperature IS NOT NULL 53 | 54 | ) 55 | SELECT @Json= ( 56 | SELECT '{' 57 | + STRING_AGG( 58 | '"' + s.[key] + '":' 59 | + CASE 60 | WHEN s.[type] = '1' THEN '"' + s.[value] + '"' COLLATE SQL_Latin1_General_CP1_CI_AS 61 | WHEN s.[type] = '0' THEN 'null' 62 | ELSE s.[value] COLLATE SQL_Latin1_General_CP1_CI_AS 63 | END 64 | , ',' 65 | ) 66 | + '}' as b 67 | FROM ( 68 | SELECT k.[key], k.[value], k.[type] 69 | FROM keys k 70 | 71 | UNION ALL 72 | 73 | SELECT j.[key], j.[value], j.[type] 74 | FROM OPENJSON(@ExtraSettings) AS j 75 | LEFT JOIN keys k ON k.[key] = j.[key] 76 | WHERE k.[key] IS NULL 77 | ) s 78 | ) 79 | RETURN @Json 80 | END 81 | 82 | GO 83 | 84 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/StoredProcedures/usp_TableLoad_CheckSchemaDrift.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: Compare Source and Target Tables to see if Schema Drift. Return 1 or 0 3 | Used By: usp_TableLoad 4 | History: 20/02/2025 Created 5 | 15/03/2025 Bob, Tuning to reduce queries 6 | */ 7 | CREATE PROC [dwa].[usp_TableLoad_CheckSchemaDrift] @SourceObjectID int, @TargetObjectID int, @PrimaryKey sysname, @BusinessKeys varchar(4000) , @RelatedTables varchar(4000) , @RowChecksum bit,@NonKeyColumns varchar(4000), @IsSchemaDrift bit OUT AS 8 | BEGIN 9 | SET NOCOUNT ON 10 | DECLARE @CheckSumSource int 11 | , @CheckSumTarget int 12 | 13 | /*Check Tables with surrogate FKs*/ 14 | IF len(coalesce(@RelatedTables,'')) >0 15 | SELECT @CheckSumSource = checksum(string_agg(checksum(sc.name , sc.tname ,sc.max_length ,sc.precision,sc.scale,sc.is_nullable),',') WITHIN GROUP (Order by sc.name)) 16 | FROM ( 17 | SELECT c.name, st.name AS tname, c.max_length, c.precision, c.scale, c.is_nullable 18 | FROM sys.schemas s 19 | INNER JOIN sys.tables t ON s.schema_id = t.schema_id 20 | INNER JOIN sys.columns c ON t.object_id = c.object_id 21 | INNER JOIN sys.types st ON st.user_type_id = c.user_type_id 22 | WHERE s.name + '.' + t.name IN ( SELECT ltrim(value) FROM string_split(@RelatedTables,',')) AND c.column_id =1 23 | UNION ALL 24 | SELECT c.name , st.name as tname ,c.max_length ,c.precision,c.scale,c.is_nullable 25 | FROM sys.columns c 26 | INNER JOIN sys.types st ON c.user_type_id = st.user_type_id 27 | WHERE c.object_id = @SourceObjectID AND c.name in (select ltrim(value) from string_split(@NonKeyColumns ,',')) 28 | UNION ALL 29 | SELECT 'RowChecksum','int',4,10,0,0 30 | WHERE @RowChecksum=1 31 | 32 | )sc 33 | ELSE 34 | SELECT @CheckSumSource = checksum(string_agg(checksum(sc.name , sc.tname ,sc.max_length ,sc.precision,sc.scale,sc.is_nullable),',') WITHIN GROUP (Order by sc.name)) 35 | FROM ( 36 | SELECT c.name , st.name as tname ,c.max_length ,c.precision,c.scale,c.is_nullable 37 | FROM sys.columns c 38 | INNER JOIN sys.types st ON c.user_type_id = st.user_type_id 39 | WHERE c.object_id = @SourceObjectID AND c.name not in ( 'FileName','LineageKey','RowChecksum','RowVersionNo',coalesce(@PrimaryKey,'')) 40 | UNION ALL 41 | SELECT 'RowChecksum','int',4,10,0,0 42 | WHERE @RowChecksum=1 43 | ) sc 44 | 45 | SELECT @CheckSumTarget = checksum(string_agg(checksum(sc.name , st.name ,sc.max_length ,sc.precision,sc.scale,sc.is_nullable),',') WITHIN GROUP (Order by sc.name)) 46 | FROM sys.columns sc 47 | INNER JOIN sys.types st ON sc.user_type_id = st.user_type_id 48 | WHERE sc.object_id = @TargetObjectID AND sc.name not in ('FileName','LineageKey','RowVersionNo',coalesce(@PrimaryKey,'')) 49 | 50 | SET @IsSchemaDrift = CASE WHEN @CheckSumSource = @CheckSumTarget then 0 else 1 end 51 | 52 | END -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/StoredProcedures/usp_TableLoad_DeploySchemaDrift.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: DeploySchemaDrift for Table Load framework - Execute main usp_TableLoad proc 3 | Example: EXEC dwa.[usp_TableLoad_DeploySchemaDrift] 2 4 | History: 25/08/2022 Deepak, Created 5 | */ 6 | CREATE PROC [dwa].[usp_TableLoad_DeploySchemaDrift] @TableID [int] AS 7 | BEGIN 8 | SET NOCOUNT ON 9 | DECLARE @sql nvarchar(max) 10 | , @SourceObject varchar(128) 11 | , @TargetObject varchar(512) 12 | , @TempTable varchar(102) 13 | , @SelectColumns [nvarchar](max) 14 | , @AddColumns [nvarchar](max) 15 | , @RowChecksum [nvarchar](max) 16 | 17 | raiserror ('Error in usp_TableLoad_DeploySchemaDrift. Not supported yet. Turn off SchemaDrift',16,1) 18 | 19 | SELECT @SourceObject =t.SourceObject 20 | , @TargetObject = t.SchemaName + '.' + t.TableName 21 | FROM Meta.config.edwTables t 22 | WHERE t.TableID = @TableID 23 | 24 | SET @TempTable = @TargetObject+'_tmp' 25 | 26 | BEGIN 27 | IF OBJECT_ID(@TempTable) IS NOT NULL 28 | EXEC [dwa].[usp_TableDrop] @Table= @TempTable 29 | SET @SQL = 'CREATE TABLE ' +@TargetObject+'_tmp AS' + char(13) + 'SELECT * FROM '+@TargetObject +';' 30 | PRINT @SQL 31 | EXEC(@SQL) 32 | EXEC [dwa].[usp_TableDrop] @Table= @TargetObject; 33 | EXEC [dwa].[usp_TableCreate] @TableID = @TableID 34 | 35 | SET @AddColumns= ( 36 | SELECT string_agg( 37 | CASE WHEN c.is_nullable = 1 THEN 'NULL' 38 | WHEN c.name LIKE '%RowChecksum%' THEN '-1' 39 | WHEN st.name LIKE '%char%' AND c.is_nullable = 0 THEN '''''' 40 | WHEN st.name IN ('decimal', 'numeric', 'int') AND c.is_nullable = 0 THEN '0' 41 | END + ' AS ' + c.name + ' ', ',') 42 | FROM sys.columns c 43 | INNER JOIN sys.types st ON c.user_type_id = st.user_type_id 44 | WHERE c.object_id = object_id(@TargetObject) AND c.name NOT IN (SELECT name FROM sys.columns WHERE object_id = object_id(@TempTable)) 45 | ) 46 | 47 | SET @RowChecksum = ( 48 | SELECT string_agg(c.name, ',') 49 | FROM sys.columns c 50 | WHERE c.object_id = object_id(@TargetObject) 51 | AND c.name IN (SELECT name FROM sys.columns WHERE object_id = object_id(@TempTable)) 52 | AND c.name LIKE '%RowChecksum%') 53 | 54 | SET @SelectColumns = ( 55 | SELECT string_agg(c.name, ',') 56 | FROM sys.columns c 57 | WHERE c.object_id = object_id(@TargetObject) AND c.name IN (SELECT name FROM sys.columns WHERE object_id = object_id(@TempTable)) 58 | AND c.name NOT LIKE '%RowChecksum%') 59 | 60 | SET @SQL = 'INSERT INTO ' + @TargetObject + char(13) + 'SELECT ' + @SelectColumns 61 | IF @AddColumns IS NOT NULL SET @SQL = @SQL + ' , ' + @AddColumns 62 | IF @RowChecksum IS NOT NULL SET @SQL = @SQL + ' , ' + '-1 AS' + @RowChecksum 63 | SET @SQL = @SQL + char(13) + 'FROM ' + @TempTable 64 | 65 | PRINT @SQL 66 | EXEC (@SQL) 67 | IF OBJECT_ID(@TempTable) IS NOT NULL 68 | EXEC [dwa].[usp_TableDrop] @Table=@TempTable 69 | END 70 | END -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/definition/tables/Account.tmdl: -------------------------------------------------------------------------------- 1 | table Account 2 | lineageTag: 192ff249-38e0-47eb-bd72-7379a259e060 3 | 4 | column AccountKey 5 | dataType: string 6 | isHidden 7 | lineageTag: 10cba98e-73b3-4a84-b3cd-04b88bd9b1e1 8 | summarizeBy: none 9 | sourceColumn: AccountKey 10 | 11 | changedProperty = IsHidden 12 | 13 | annotation SummarizationSetBy = Automatic 14 | 15 | column ParentAccountKey 16 | dataType: string 17 | isHidden 18 | lineageTag: fca0efb9-4c7d-4645-9cfb-f98dedd3f9ac 19 | summarizeBy: none 20 | sourceColumn: ParentAccountKey 21 | 22 | changedProperty = IsHidden 23 | 24 | annotation SummarizationSetBy = Automatic 25 | 26 | column Operator 27 | dataType: string 28 | lineageTag: cd60858c-7c0c-403a-a675-48ab2f2ab8e3 29 | summarizeBy: none 30 | sourceColumn: Operator 31 | 32 | annotation SummarizationSetBy = Automatic 33 | 34 | column 'Account Code' 35 | dataType: int64 36 | formatString: 0 37 | lineageTag: 44edbb13-0b31-4c83-9563-531a9d779eb1 38 | summarizeBy: none 39 | sourceColumn: Account Code 40 | 41 | annotation SummarizationSetBy = User 42 | 43 | column Account 44 | dataType: string 45 | lineageTag: 863cb65e-a6d5-41a6-a663-c71e0e05985c 46 | summarizeBy: none 47 | sourceColumn: Account 48 | 49 | annotation SummarizationSetBy = Automatic 50 | 51 | column 'Account Type' 52 | dataType: string 53 | lineageTag: 8a86a89d-23b3-48c4-9f3b-6e2d6bb088f8 54 | summarizeBy: none 55 | sourceColumn: Account Type 56 | 57 | annotation SummarizationSetBy = Automatic 58 | 59 | column 'Value Type' 60 | dataType: string 61 | lineageTag: 16bcfad6-ba99-45ab-b73c-2a3e1444a10d 62 | summarizeBy: none 63 | sourceColumn: Value Type 64 | 65 | annotation SummarizationSetBy = Automatic 66 | 67 | column Report 68 | dataType: string 69 | lineageTag: 3b9899c4-c8c7-4859-b1fc-bf398d9d2b79 70 | summarizeBy: none 71 | sourceColumn: Report 72 | 73 | annotation SummarizationSetBy = Automatic 74 | 75 | column 'Account L2' 76 | dataType: string 77 | lineageTag: ccea48df-121f-41e4-a1cd-9f9659d20d9c 78 | summarizeBy: none 79 | sourceColumn: Account L2 80 | 81 | annotation SummarizationSetBy = Automatic 82 | 83 | column 'Account L3' 84 | dataType: string 85 | lineageTag: 9332c90d-19b1-4de2-84ce-87a7d632b6f7 86 | summarizeBy: none 87 | sourceColumn: Account L3 88 | 89 | annotation SummarizationSetBy = Automatic 90 | 91 | partition Account-f1b39765-9b25-4192-a248-07893818c1f2 = m 92 | mode: import 93 | queryGroup: 'Finance GL' 94 | source = 95 | let 96 | dbo_DimAccount= Sql.Database("fkm4vwf6l6zebg4lqrhbtdcmsq-yctecwmyrr4u3mdfun4j5hqjma.datawarehouse.fabric.microsoft.com", "DW", [Query="SELECT [AccountKey],[ParentAccountKey],[Account Code],[Account],[Account Type],[Operator],[Value Type],[Report],[Account L2],[Account L3] FROM [reports].[DimAccount]"]) 97 | in 98 | dbo_DimAccount 99 | 100 | annotation PBI_ResultType = Table 101 | 102 | annotation LinkedQueryName = Account 103 | 104 | annotation PBI_NavigationStepName = Navigation 105 | 106 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/audit/StoredProcedures/usp_PipelineLogCopy.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: Extended Logging for Copy Activity 3 | Used by: Extract-SQL-RiskFlow and any Copy Activity 4 | Example: 5 | exec [audit].[usp_PipelineLogCopy] '330C571C-4AB4-4C84-B184-B461770FE882','{ 6 | "dataRead": 594, 7 | "dataWritten": 1509, 8 | "filesWritten": 1, 9 | "sourcePeakConnections": 1, 10 | "sinkPeakConnections": 1, 11 | "rowsRead": 22, 12 | "rowsCopied": 22, 13 | "copyDuration": 15, 14 | "throughput": 0.074, 15 | "errors": [], 16 | "usedParallelCopies": 1, 17 | "executionDetails": [ 18 | { 19 | "source": { 20 | "type": "SqlServer" 21 | }, 22 | "sink": { 23 | "type": "Lakehouse" 24 | }, 25 | "status": "Succeeded", 26 | "start": "5/2/2024, 10:18:28 AM", 27 | "duration": 15, 28 | "usedParallelCopies": 1, 29 | "profile": { 30 | "queue": { 31 | "status": "Completed", 32 | "duration": 0 33 | }, 34 | "transfer": { 35 | "status": "Completed", 36 | "duration": 8, 37 | "details": { 38 | "readingFromSource": { 39 | "type": "SqlServer", 40 | "workingDuration": 0, 41 | "timeToFirstByte": 0 42 | }, 43 | "writingToSink": { 44 | "type": "Lakehouse", 45 | "workingDuration": 0 46 | } 47 | } 48 | } 49 | }, 50 | "detailedDurations": { 51 | "queuingDuration": 0, 52 | "timeToFirstByte": 0, 53 | "transferDuration": 8 54 | } 55 | } 56 | ], 57 | "dataConsistencyVerification": { 58 | "VerificationResult": "NotVerified" 59 | } 60 | }' 61 | 62 | History: 63 | 22/08/2021 Bob, Created for Fabric DWA 64 | */ 65 | CREATE PROC [audit].[usp_PipelineLogCopy] @LineageKey uniqueidentifier,@Json varchar(8000) 66 | AS 67 | BEGIN 68 | SET NOCOUNT ON 69 | DECLARE @Status varchar(100) 70 | DECLARE @errors varchar(8000) 71 | DECLARE @RowCount bigint 72 | DECLARE @copyDuration int 73 | DECLARE @DataWritten bigint 74 | DECLARE @DataRead bigint 75 | DECLARE @ErrorCode varchar(4000) 76 | DECLARE @ErrorMessage varchar(4000) 77 | 78 | SELECT @errors=value 79 | FROM OPENJSON (@Json ) t 80 | WHERE [key]='errors' 81 | 82 | IF @errors ='[]' 83 | SET @Status='Completed' 84 | ELSE 85 | BEGIN 86 | SET @Status ='Failed' 87 | SELECT @ErrorCode = Code, @ErrorMessage =[Message] 88 | FROM OPENJSON (@errors) 89 | WITH ( 90 | Code varchar(4000) , 91 | Message varchar(4000) 92 | ); 93 | END 94 | 95 | select @RowCount = rowsRead, @CopyDuration=copyDuration, @DataRead =dataRead, @DataWritten=dataWritten 96 | FROM OPENJSON (@Json) 97 | WITH ( 98 | dataRead bigint , 99 | dataWritten bigint , 100 | rowsRead bigint , 101 | copyDuration int 102 | ); 103 | 104 | UPDATE [audit].[PipelineLog] 105 | SET [Status]=@Status, ErrorCode=@ErrorCode, ErrorMessage=@ErrorMessage, CopyDuration=@copyDuration, RowsRead=@RowCount, DataRead=@DataRead, DataWritten=@DataWritten, ExtendedLog=@Json 106 | WHERE LineageKey=@LineageKey 107 | 108 | END 109 | 110 | GO 111 | 112 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/audit/StoredProcedures/usp_PipelineStart.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | /* 7 | Description: Start a Pipeline AND return all Meta Data (complex) 8 | Used By: ADF Pipeline-Worker 9 | 10 | -- Example 11 | exec [audit].[usp_PipelineStart] 37, '7EEFA88E-D9F9-4204-AC47-97F2FC3362CE', '1087FDF2-90FB-437B-A3CB-D2F94E8C4D82', '1087FDF2-90FB-437B-A3CB-D2F94E8C4D82' 12 | SELECT * FROM audit.PipelineLog ORDER BY StartDateTime DESC 13 | SELECT * FROM audit.LineageLog ORDER BY LineageKey DESC 14 | exec [audit].[usp_PipelineStart] 37, 'cfb66af5-4815-4664-a494-f08fdcf9e0ab' 15 | History: 16 | 24/11/2024 Bob, Migrated to Fabric Databaase 17 | 18 | */ 19 | CREATE PROC [audit].[usp_PipelineStart] 20 | @PipelineID [int] 21 | ,@LineageKey uniqueidentifier = null 22 | ,@RunID uniqueidentifier = null 23 | ,@ParentRunID uniqueidentifier = null 24 | ,@WorkspaceID uniqueidentifier = null 25 | ,@Pipeline uniqueidentifier = null 26 | ,@PipelineName varchar(512) = null 27 | ,@PackageGroup varchar(512) =null 28 | AS 29 | BEGIN 30 | 31 | DECLARE @SourceConnectionSettings varchar(8000) 32 | ,@TargetConnectionSettings varchar(8000) 33 | ,@SourceSettings varchar(8000) 34 | ,@TargetSettings varchar(8000) 35 | ,@ActivitySettings varchar(8000) 36 | ,@PreExecuteSQL varchar(8000) 37 | ,@PostExecuteSQL varchar(8000) 38 | ,@Template varchar(8000) 39 | ,@StartDateTime datetime2 = getdate() 40 | ,@Stage varchar(50) 41 | ,@PipelineSequence int 42 | ,@MonitoringUrl varchar(512) 43 | ,@TableID int = null 44 | 45 | 46 | SET @LineageKey=COALESCE(@LineageKey, NEWID()) 47 | SELECT @SourceConnectionSettings=SourceConnectionSettings 48 | , @TargetConnectionSettings=TargetConnectionSettings 49 | , @SourceSettings=SourceSettings 50 | , @TargetSettings=TargetSettings 51 | , @ActivitySettings=ActivitySettings 52 | , @PreExecuteSQL=PreExecuteSQL 53 | , @PostExecuteSQL=PostExecuteSQL 54 | , @Template=Template 55 | , @Stage=Stage 56 | ,@MonitoringUrl = lower('https://app.powerbi.com/workloads/data-pipeline/monitoring/workspaces/' + CONVERT(varchar(36), @WorkspaceID) + '/pipelines/' + @PipelineName + '/' + CONVERT(varchar(36), @RunID) ) 57 | ,@TableID = TableID 58 | FROM [config].[PipelineMeta] 59 | WHERE PipelineID=@PipelineID 60 | 61 | INSERT INTO [audit].[PipelineLog] ( 62 | [LineageKey] 63 | , [RunID] 64 | , [ParentRunID] 65 | , [PipelineID] 66 | , WorkspaceID 67 | , [PackageGroup] 68 | , [Stage] 69 | , [StartDate] 70 | , [StartDateTime] 71 | , [Status] 72 | , PipelineSequence 73 | , [Template] 74 | , Pipeline 75 | , PipelineName 76 | , MonitoringUrl 77 | ) 78 | 79 | VALUES (@LineageKey 80 | ,@RunID 81 | ,@ParentRunID 82 | ,@PipelineID 83 | ,@WorkspaceID 84 | ,@PackageGroup 85 | ,@Stage 86 | ,CONVERT(DATE, @StartDateTime) 87 | ,@StartDateTime 88 | ,'Started' 89 | ,@PipelineSequence 90 | ,@Template 91 | ,@Pipeline 92 | ,@PipelineName 93 | ,@MonitoringUrl) 94 | 95 | IF @PreExecuteSQL IS NOT NULL 96 | BEGIN 97 | PRINT ('/*Executing Pre Execute*/' + char(13) + @PreExecuteSQL) 98 | EXEC( @PreExecuteSQL ) 99 | END 100 | 101 | END 102 | 103 | GO 104 | 105 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/StoredProcedures/usp_PipelineSequence.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | /* 10 | Description: Retrieve Array of PipelineOrder for Which Pipelines to run in which sequence 11 | Used By: ADF Pipeline-Controller 12 | 13 | -- Example 14 | exec [config].[usp_PipelineSequence] 'AW','ALL' 15 | exec [config].[usp_PipelineSequence] 'test-copy-blob' 16 | exec [config].[usp_PipelineSequence] 'GAMDAY0','Ingest' 17 | 18 | History: 19 | 08/05/2024 Bob, Migrated to Fabric 20 | 27/08/2024 Aidan, Implemented package group hierarchy 21 | 29/08/2024 Bob, Trigger build of meta data cache if needed 22 | 01/11/2024 Kristan, Added PackageGroup coalesce for PipelineGroups 23 | 11/11/2024 Aiddan, Changed join to Pipelinegroups to a Left join 24 | 25 | */ 26 | CREATE PROC [config].[usp_PipelineSequence] 27 | @PackageGroup [varchar](4000) ='ALL' 28 | ,@Stage [varchar](50) =null 29 | AS 30 | BEGIN 31 | SET NOCOUNT ON; 32 | DECLARE @ChecksumToken INT 33 | DECLARE @OldChecksumToken INT 34 | 35 | SELECT @OldChecksumToken=ChecksumToken FROM [config].[PipelineMetaCache] 36 | SELECT @ChecksumToken = CHECKSUM_AGG(checksum(*)) 37 | FROM config.Pipelines p 38 | LEFT JOIN config.PipelineGroups pg ON pg.PipelineGroupID =p.PipelineGroupID 39 | LEFT JOIN config.Configurations c on pg.TargetConfigurationID =c.ConfigurationID 40 | LEFT JOIN config.Configurations c2 on pg.SourceConfigurationID =c2.ConfigurationID 41 | 42 | IF @OldChecksumToken <> @ChecksumToken or @OldChecksumToken is null 43 | BEGIN 44 | PRINT 'Rebuilding Cache. exec [config].[usp_PipelineBuildMetaData]' 45 | exec [config].[usp_PipelineBuildMetaData] 46 | 47 | IF EXISTS (SELECT * FROM [config].[PipelineMetaCache] ) 48 | UPDATE [config].[PipelineMetaCache] SET ChecksumToken=@ChecksumToken 49 | ELSE 50 | INSERT INTO [config].[PipelineMetaCache] VALUES (@ChecksumToken) 51 | END 52 | 53 | SELECT @PackageGroup =coalesce(@PackageGroup,'ALL') 54 | ,@Stage =coalesce(NULLIF(@Stage,''),'ALL') 55 | 56 | ;WITH pg as 57 | ( 58 | SELECT CONVERT(VARCHAR(50), TRIM(value)) AS PackageGroup FROM string_split (@PackageGroup,',') WHERE value <> 'ALL' 59 | UNION ALL 60 | SELECT PackageGroup FROM config.PackageGroups pg WHERE @PackageGroup='ALL' 61 | UNION 62 | SELECT pgl.ChildPackageGroup AS PackageGroup FROM string_split(@PackageGroup,',') pg2 63 | INNER JOIN config.PackageGroupLinks pgl ON pgl.PackageGroup = TRIM(pg2.value) 64 | UNION 65 | SELECT pgl2.ChildPackageGroup AS PackageGroup FROM string_split(@PackageGroup,',') pg2 66 | INNER JOIN config.PackageGroupLinks pgl ON pgl.PackageGroup = TRIM(pg2.value) 67 | INNER JOIN config.PackageGroupLinks pgl2 ON pgl2.PackageGroup = pgl.ChildPackageGroup 68 | ), 69 | s as 70 | ( 71 | SELECT value as Stage FROM string_split (@Stage,',') WHERE value <> 'ALL' 72 | ) 73 | SELECT p.[PipelineSequence],max(coalesce(p.Stage, pg.Stage)) as Stage 74 | , convert(bit,max(coalesce(p.ContinueOnError, pg.ContinueOnError, 0 ))) as ContinueOnError 75 | FROM [config].[Pipelines] p 76 | LEFT JOIN config.PipelineGroups pg on pg.PipelineGroupID =p.PipelineGroupID 77 | WHERE coalesce(p.[PackageGroup],pg.[PackageGroup]) IN (SELECT PackageGroup FROM pg) 78 | AND (coalesce(p.Stage,pg.Stage ) IN (SELECT Stage FROM s) OR @Stage ='ALL') 79 | GROUP BY p.PipelineSequence 80 | ORDER BY p.PipelineSequence 81 | 82 | END 83 | 84 | GO 85 | 86 | -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Ops/Extract-Fabric-Logs.Notebook/notebook-content.py: -------------------------------------------------------------------------------- 1 | # Fabric notebook source 2 | 3 | # METADATA ******************** 4 | 5 | # META { 6 | # META "kernel_info": { 7 | # META "name": "synapse_pyspark" 8 | # META }, 9 | # META "dependencies": { 10 | # META "lakehouse": { 11 | # META "default_lakehouse": "d58f4f2d-59d7-406d-ae4c-898354a6a75f", 12 | # META "default_lakehouse_name": "LH", 13 | # META "default_lakehouse_workspace_id": "5941a6c0-8c98-4d79-b065-a3789e9e0960" 14 | # META } 15 | # META } 16 | # META } 17 | 18 | # MARKDOWN ******************** 19 | 20 | # #### Extract-Fabric-Logs 21 | # 22 | # This notebook collects and stores refresh log data from datasets into delta lake table that can be used for monitoring and auditing purpose. 23 | # It retrieves latest refresh request logs for each dataset in a workspace. 24 | 25 | # PARAMETERS CELL ******************** 26 | 27 | # No Parameters as settings obtained dynamically from default spark lakehouse 28 | 29 | 30 | # METADATA ******************** 31 | 32 | # META { 33 | # META "language": "python", 34 | # META "language_group": "synapse_pyspark" 35 | # META } 36 | 37 | # CELL ******************** 38 | 39 | # Fabric Refresh Logs 40 | import pandas as pd 41 | from delta.tables import * 42 | import sempy.fabric as fabric 43 | import re 44 | from pyspark.sql.types import StructType, StructField, LongType, StringType,DateType, TimestampType,MapType 45 | 46 | tenant_id=spark.conf.get("trident.tenant.id") 47 | workspace_id=spark.conf.get("trident.workspace.id") 48 | lakehouse_id=spark.conf.get("trident.lakehouse.id") 49 | lakehouse_name=spark.conf.get("trident.lakehouse.name") 50 | column_pattern = '[ ,;{}()\n\t/=]' #Pattern to remove invalid columns for lakehouse 51 | 52 | # List Datasets/SemanticModels 53 | df_datasets =fabric.list_datasets() 54 | 55 | table_name="fabric_refresh_logs" 56 | for row in df_datasets.itertuples(index=True, name='datasets'): 57 | dataset = row[1] 58 | df=fabric.list_refresh_requests(dataset=dataset, workspace=workspace_id, top_n=100) 59 | df=df.rename(columns=dict(zip(df.columns, [re.sub(column_pattern, '_', col.strip(column_pattern).lower()) for col in df.columns]))) 60 | df.insert(0, 'dataset', dataset) 61 | df.insert(1, 'workspace', workspace_id) 62 | df['refresh_attempts'] = df['refresh_attempts'].astype(str) 63 | df.drop(columns=['extended_status']) 64 | 65 | schema = StructType([ 66 | StructField("dataset", StringType(), True), 67 | StructField("workspace", StringType(), True), 68 | StructField("id", LongType(), True), 69 | StructField("request_id", StringType(), True), 70 | StructField("start_time", TimestampType(), True), 71 | StructField("end_time", TimestampType(), True), 72 | StructField("refresh_type", StringType(), True), 73 | StructField("service_exception_json", StringType(), True), 74 | StructField("status", StringType(), True), 75 | StructField("refresh_attempts", StringType(), True) 76 | ]) 77 | 78 | spark_df =spark.createDataFrame(df,schema =schema ) 79 | if spark.catalog.tableExists(table_name): 80 | target_table = DeltaTable.forName(spark, f"{table_name}") 81 | target_table.alias("target").merge(spark_df.alias("source"), "source.id=target.id").whenNotMatchedInsertAll().execute() 82 | else: 83 | spark_df.write.mode('overwrite').saveAsTable(table_name) 84 | 85 | 86 | # METADATA ******************** 87 | 88 | # META { 89 | # META "language": "python", 90 | # META "language_group": "synapse_pyspark" 91 | # META } 92 | -------------------------------------------------------------------------------- /Workspaces/DWA/Finance-GL.SemanticModel/definition/tables/Organization.tmdl: -------------------------------------------------------------------------------- 1 | table Organization 2 | lineageTag: eaf5762c-b329-436a-b7c9-556fcc598f81 3 | 4 | column OrganizationKey 5 | dataType: string 6 | isHidden 7 | lineageTag: df6f19b0-997c-4f1f-a692-970de951f469 8 | summarizeBy: none 9 | sourceColumn: OrganizationKey 10 | 11 | changedProperty = IsHidden 12 | 13 | annotation SummarizationSetBy = Automatic 14 | 15 | column ParentOrganizationKey 16 | dataType: string 17 | isHidden 18 | lineageTag: edc862bf-9661-4cf0-afff-a726fe2b54a7 19 | summarizeBy: none 20 | sourceColumn: ParentOrganizationKey 21 | 22 | changedProperty = IsHidden 23 | 24 | annotation SummarizationSetBy = Automatic 25 | 26 | column PercentageOfOwnership 27 | dataType: string 28 | isHidden 29 | lineageTag: db526a92-5be8-45db-9c21-2c8f86a4fc03 30 | summarizeBy: none 31 | sourceColumn: PercentageOfOwnership 32 | 33 | changedProperty = IsHidden 34 | 35 | annotation SummarizationSetBy = Automatic 36 | 37 | column CurrencyKey 38 | dataType: string 39 | isHidden 40 | lineageTag: f73a8fcb-ccc7-4c08-9454-92ab6121f0a6 41 | summarizeBy: none 42 | sourceColumn: CurrencyKey 43 | 44 | changedProperty = IsHidden 45 | 46 | annotation SummarizationSetBy = Automatic 47 | 48 | column 'Organization Name' 49 | dataType: string 50 | lineageTag: 4ce2fca3-bb27-4539-8bee-3be8632b5821 51 | summarizeBy: none 52 | sourceColumn: Organization Name 53 | 54 | annotation SummarizationSetBy = Automatic 55 | 56 | column Company 57 | dataType: string 58 | lineageTag: 79972d97-ce3a-434f-a8b8-edc4c82307c4 59 | summarizeBy: none 60 | sourceColumn: Company 61 | 62 | annotation SummarizationSetBy = Automatic 63 | 64 | column Region 65 | dataType: string 66 | lineageTag: 1e317c68-7f67-4aff-8dcf-3554f9285ad0 67 | summarizeBy: none 68 | sourceColumn: Region 69 | 70 | annotation SummarizationSetBy = Automatic 71 | 72 | column Country 73 | dataType: string 74 | lineageTag: 76a5326f-0da2-4edd-b11b-b7613beb8ba0 75 | summarizeBy: none 76 | sourceColumn: Country 77 | 78 | annotation SummarizationSetBy = Automatic 79 | 80 | column Division 81 | dataType: string 82 | lineageTag: 8b3c34f1-c606-4cab-b0a0-6c1b21d532e0 83 | summarizeBy: none 84 | sourceColumn: Division 85 | 86 | annotation SummarizationSetBy = Automatic 87 | 88 | hierarchy Organization 89 | lineageTag: 6fa3b8fc-ccf2-405a-9043-03e344af4d74 90 | 91 | level Company 92 | lineageTag: 7c2e05e5-0485-401d-ad41-d9dca43978a5 93 | column: Company 94 | 95 | level Region 96 | lineageTag: 4ddaaf3b-913d-44e7-99db-27bbef816b84 97 | column: Region 98 | 99 | level Country 100 | lineageTag: 4b406f54-308a-4855-b051-bc2ea04effb5 101 | column: Country 102 | 103 | level Division 104 | lineageTag: dbe4c12e-e227-480d-babf-1144a5376581 105 | column: Division 106 | 107 | partition Organization-d9ef92dd-2885-427f-bb6c-29b8742818b8 = m 108 | mode: import 109 | queryGroup: 'Finance GL' 110 | source = 111 | let 112 | dbo_DimOrganization= Sql.Database("fkm4vwf6l6zebg4lqrhbtdcmsq-yctecwmyrr4u3mdfun4j5hqjma.datawarehouse.fabric.microsoft.com", "DW", [Query="SELECT OrganizationKey, ParentOrganizationKey, PercentageOfOwnership, [Organization Name], CurrencyKey, Company, Region, Country, Division FROM reports.DimOrganization"]) 113 | in 114 | dbo_DimOrganization 115 | 116 | annotation PBI_ResultType = Table 117 | 118 | annotation PBI_NavigationStepName = Navigation 119 | 120 | -------------------------------------------------------------------------------- /Workspaces/DWA/Meta.SQLDatabase/config/StoredProcedures/usp_PipelineQueue.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: Return a Queue of Pipelines to Run based on PipelineSequence 3 | Used By: ADF Pipeline-Worker 4 | 5 | Example 6 | exec config.[usp_PipelineQueue] 'AW' 7 | 8 | History: 9 | 30/04/2024 Bob, Migrated to Fabric 10 | 27/08/2024 Aidan, Implemented package group hierarchy 11 | 29/08/2024 Bob, Added TemplateType for Dynamic Pipeline-Worker 12 | 11/09/2024 Aidan, Ignores PackageGroup if PipelineID is specified 13 | 31/09/2024 Kristan, added to config schema 14 | 06/11/2024 Aidan, Added TableID 15 | 29/06/2025 Bob, Added TemplateWorkspaceID and SessionTag 16 | */ 17 | CREATE PROC [config].[usp_PipelineQueue] 18 | @PackageGroup [varchar](50) ='ALL' 19 | ,@PipelineSequence [smallint] =-1 20 | ,@PipelineID [int] =-1 21 | AS 22 | BEGIN 23 | SET NOCOUNT ON; 24 | DECLARE @ChecksumToken INT 25 | DECLARE @OldChecksumToken INT 26 | 27 | IF @PipelineID <> -1 AND @PipelineID IS NOT NULL 28 | BEGIN 29 | SELECT @PackageGroup='ALL',@PipelineSequence=-1 30 | END 31 | 32 | SELECT @PackageGroup=COALESCE(@PackageGroup, 'ALL'), @PipelineSequence=COALESCE(@PipelineSequence,-1) 33 | 34 | 35 | SELECT @OldChecksumToken=ChecksumToken FROM [config].[PipelineMetaCache] 36 | SELECT @ChecksumToken = CHECKSUM_AGG(checksum(*)) 37 | FROM config.Pipelines p 38 | INNER JOIN config.PipelineGroups pg ON pg.PipelineGroupID =p.PipelineGroupID 39 | LEFT JOIN config.Configurations c on pg.TargetConfigurationID =c.ConfigurationID 40 | LEFT JOIN config.Configurations c2 on pg.SourceConfigurationID =c2.ConfigurationID 41 | 42 | IF @OldChecksumToken <> @ChecksumToken or @OldChecksumToken is null 43 | BEGIN 44 | PRINT 'Rebuilding Cache. exec [config].[usp_PipelineBuildMetaData]' 45 | exec [config].[usp_PipelineBuildMetaData] 46 | 47 | IF EXISTS (SELECT * FROM [config].[PipelineMetaCache] ) 48 | UPDATE [config].[PipelineMetaCache] SET ChecksumToken=@ChecksumToken 49 | ELSE 50 | INSERT INTO [config].[PipelineMetaCache] VALUES (@ChecksumToken) 51 | END 52 | 53 | 54 | ;WITH pg AS 55 | ( 56 | SELECT CONVERT(VARCHAR(50), TRIM(value)) AS PackageGroup FROM string_split (@PackageGroup,',') WHERE value <> 'ALL' 57 | UNION ALL 58 | SELECT PackageGroup FROM config.PackageGroups pg WHERE @PackageGroup='ALL' 59 | UNION 60 | SELECT pgl.ChildPackageGroup AS PackageGroup FROM string_split(@PackageGroup,',') pg2 61 | INNER JOIN config.PackageGroupLinks pgl ON pgl.PackageGroup = TRIM(pg2.value) 62 | UNION 63 | SELECT pgl2.ChildPackageGroup AS PackageGroup FROM string_split(@PackageGroup,',') pg2 64 | INNER JOIN config.PackageGroupLinks pgl ON pgl.PackageGroup = TRIM(pg2.value) 65 | INNER JOIN config.PackageGroupLinks pgl2 ON pgl2.PackageGroup = pgl.ChildPackageGroup 66 | ) 67 | SELECT NEWID() as LineageKey, [PipelineID], [PipelineSequence],[SourceConnectionSettings],[TargetConnectionSettings] 68 | , [SourceSettings] 69 | , [TargetSettings],[ActivitySettings],[PreExecuteSQL],[PostExecuteSQL],[Stage] 70 | , [Template], coalesce(TemplateType, Template) as TemplateType, TemplateID, p.TableID as TableID 71 | , TemplateWorkspaceID 72 | , p.SessionTag 73 | FROM [config].[PipelineMeta] p 74 | WHERE p.Enabled=1 75 | AND (p.PackageGroup=@PackageGroup OR p.[PackageGroup] IN (SELECT PackageGroup FROM pg) or @PackageGroup ='ALL') 76 | AND (p.PipelineID=@PipelineID OR @PipelineID=-1) 77 | AND (p.PipelineSequence=@PipelineSequence OR @PipelineSequence =-1) 78 | ORDER BY p.PipelineSequence, p.PipelineID 79 | 80 | END 81 | 82 | GO 83 | 84 | -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Refresh/Refresh-Fabric.Notebook/notebook-content.py: -------------------------------------------------------------------------------- 1 | # Fabric notebook source 2 | 3 | # METADATA ******************** 4 | 5 | # META { 6 | # META "kernel_info": { 7 | # META "name": "synapse_pyspark" 8 | # META }, 9 | # META "dependencies": { 10 | # META "lakehouse": { 11 | # META "default_lakehouse": "d58f4f2d-59d7-406d-ae4c-898354a6a75f", 12 | # META "default_lakehouse_name": "LH", 13 | # META "default_lakehouse_workspace_id": "5941a6c0-8c98-4d79-b065-a3789e9e0960" 14 | # META } 15 | # META } 16 | # META } 17 | 18 | # PARAMETERS CELL ******************** 19 | 20 | SourceConnectionSettings='{}' 21 | #https://learn.microsoft.com/en-us/python/api/semantic-link-sempy/sempy.fabric?view=semantic-link-python#sempy-fabric-refresh-dataset 22 | SourceSettings='{"dataset":"Finance-GL","refresh_type":"full","max_parallelism":"10","retry_count":"0"}' 23 | TargetConnectionSettings='{}' 24 | TargetSettings='{}' 25 | ActivitySettings=None 26 | 27 | # METADATA ******************** 28 | 29 | # META { 30 | # META "language": "python", 31 | # META "language_group": "synapse_pyspark" 32 | # META } 33 | 34 | # CELL ******************** 35 | 36 | import json 37 | import sempy.fabric as fabric 38 | import pandas as pd 39 | from datetime import datetime 40 | import time 41 | 42 | SourceConnectionSettings = SourceConnectionSettings or '{}' 43 | SourceSettings = SourceSettings or '{}' 44 | 45 | source_connection_settings = json.loads(SourceConnectionSettings) 46 | refresh_settings = json.loads(SourceSettings) 47 | 48 | workspace = source_connection_settings.get("workspace") 49 | 50 | if not workspace: 51 | workspace_id = spark.conf.get("trident.workspace.id") 52 | workspace = fabric.FabricRestClient().get(f"/v1/workspaces/{workspace_id}").json()["displayName"] 53 | 54 | refresh_settings["workspace"] = workspace 55 | 56 | dataset = refresh_settings["dataset"] 57 | 58 | 59 | # METADATA ******************** 60 | 61 | # META { 62 | # META "language": "python", 63 | # META "language_group": "synapse_pyspark" 64 | # META } 65 | 66 | # CELL ******************** 67 | 68 | refresh_request_id=fabric.refresh_dataset(**refresh_settings) 69 | print (refresh_request_id) 70 | 71 | # METADATA ******************** 72 | 73 | # META { 74 | # META "language": "python", 75 | # META "language_group": "synapse_pyspark" 76 | # META } 77 | 78 | # CELL ******************** 79 | 80 | t1=datetime.now() 81 | class RefreshError(Exception): 82 | pass 83 | def check_refresh_requests(): 84 | refresh_details = fabric.get_refresh_execution_details(dataset=dataset, workspace=workspace, refresh_request_id=refresh_request_id) 85 | 86 | print(".", end='', flush=True) 87 | if refresh_details == "Cancelled": 88 | raise RefreshError(f"Refresh of {workspace}\{dataset} was cancelled." ) 89 | if refresh_details.status == "Failed": 90 | first_error = next(item["Message"] for _, item in refresh_details.messages.iterrows() if item["Type"] == "Error") 91 | raise RefreshError(f"Refresh of {workspace}\{dataset} failed with error: {first_error}" ) 92 | return refresh_details.status == "Completed" 93 | 94 | while not check_refresh_requests(): 95 | time.sleep(5) 96 | 97 | duration_secs=(datetime.now()-t1).total_seconds() 98 | print (f"\n{workspace}\{dataset} Refreshed {int(duration_secs // 60)} min and {int(duration_secs % 60)} sec.") 99 | 100 | 101 | # METADATA ******************** 102 | 103 | # META { 104 | # META "language": "python", 105 | # META "language_group": "synapse_pyspark" 106 | # META } 107 | -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Ops/Extract-Artefacts.Notebook/notebook-content.py: -------------------------------------------------------------------------------- 1 | # Fabric notebook source 2 | 3 | # METADATA ******************** 4 | 5 | # META { 6 | # META "kernel_info": { 7 | # META "name": "synapse_pyspark" 8 | # META }, 9 | # META "dependencies": { 10 | # META "lakehouse": { 11 | # META "default_lakehouse": "d58f4f2d-59d7-406d-ae4c-898354a6a75f", 12 | # META "default_lakehouse_name": "LH", 13 | # META "default_lakehouse_workspace_id": "5941a6c0-8c98-4d79-b065-a3789e9e0960", 14 | # META "known_lakehouses": [ 15 | # META { 16 | # META "id": "d58f4f2d-59d7-406d-ae4c-898354a6a75f" 17 | # META } 18 | # META ] 19 | # META } 20 | # META } 21 | # META } 22 | 23 | # MARKDOWN ******************** 24 | 25 | # ## Extract-Artefacts 26 | # Collects and stores metadata information from Data Warehouse into Lakehouse for data governance, backup and data cataloguing purpose.
Metadata information could be notebook information, datasets, data warehouse tables.

27 | # This is required to populate dict_artefacts for data dictionary and DWA Framework 28 | # 29 | 30 | 31 | # CELL ******************** 32 | 33 | %run SQL-Connection-Shared-Functions 34 | 35 | # METADATA ******************** 36 | 37 | # META { 38 | # META "language": "python", 39 | # META "language_group": "synapse_pyspark" 40 | # META } 41 | 42 | # CELL ******************** 43 | 44 | import sempy.fabric as fabric 45 | import re 46 | import pandas as pd 47 | 48 | workspace_id=spark.conf.get("trident.workspace.id") 49 | lakehouse_id=spark.conf.get("trident.lakehouse.id") 50 | 51 | items = fabric.FabricRestClient().get(f"/v1/workspaces/{workspace_id}/SQLDatabases").json()["value"] 52 | sql_database=next((endpoint for endpoint in items if endpoint["displayName"] == "Meta")) 53 | sql_end_point = sql_database["properties"]["serverFqdn"] 54 | sql_database_name = sql_database["properties"]["databaseName"] 55 | connection_string = f"Driver={{ODBC Driver 18 for SQL Server}};Server={sql_end_point};database={sql_database_name}" 56 | 57 | 58 | 59 | # METADATA ******************** 60 | 61 | # META { 62 | # META "language": "python", 63 | # META "language_group": "synapse_pyspark" 64 | # META } 65 | 66 | # CELL ******************** 67 | 68 | engine = create_engine(connection_string) 69 | connection = engine.raw_connection() 70 | 71 | df=pd.DataFrame() 72 | with engine.connect() as alchemy_connection: 73 | workspaces = pd.read_sql_query ("exec [config].[usp_GetWorkspaces]", alchemy_connection) 74 | for ws_id in workspaces['WorkspaceID']: 75 | print (ws_id) 76 | items = fabric.list_items(workspace=ws_id) 77 | if df.empty: 78 | df=items 79 | else: 80 | df=pd.concat([df,items], ignore_index=True) 81 | pattern = '[ ,;{}()\n\t/=]' 82 | df=df.rename(columns=dict(zip(df.columns, [re.sub(pattern, '_', col.strip(pattern).lower()) for col in df.columns]))) 83 | 84 | 85 | # METADATA ******************** 86 | 87 | # META { 88 | # META "language": "python", 89 | # META "language_group": "synapse_pyspark" 90 | # META } 91 | 92 | # CELL ******************** 93 | 94 | #Write Back to Meta SqlDatabase (used for DWA Engine) 95 | from sqlalchemy import text 96 | with engine.begin() as alchemy_connection: 97 | df.to_sql('dict_artefacts', con=engine, if_exists='replace', index=False,method='multi', chunksize=1000) 98 | alchemy_connection.execute(text("EXEC [config].[usp_PipelineBuildMetaData]")) 99 | 100 | # METADATA ******************** 101 | 102 | # META { 103 | # META "language": "python", 104 | # META "language_group": "synapse_pyspark" 105 | # META } 106 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/StoredProcedures/usp_TableCreate.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Description: Create a TABLE with CREATE TABLE SYNTAX. We often need to do this instead of CTAS due to IDENTITY 3 | Example: 4 | [dwa].[usp_TableCreate] 1 /* Dim */ 5 | [dwa].[usp_TableCreate] 7 /* Fact */ 6 | History: 7 | 13/01/2021 Bob, Created 8 | */ 9 | CREATE PROC [dwa].[usp_TableCreate] @TableID [int],@TargetObject [varchar](512) =NULL AS 10 | BEGIN 11 | SET NOCOUNT ON 12 | BEGIN TRY 13 | DECLARE @ExecutionKey INT 14 | , @TargetSchema sysname 15 | , @TargetTable sysname 16 | , @TableType varchar(10) 17 | , @ColumnStore bit 18 | , @SourceType varchar(10) 19 | , @Identity bit 20 | , @sql nvarchar(4000) 21 | , @TargetColumns nvarchar(max) 22 | , @BusinessKeys nvarchar(4000) 23 | , @PrimaryKey sysname 24 | , @SourceColumns nvarchar(4000) 25 | , @SourceObject sysname 26 | , @Columns nvarchar(4000) 27 | , @RelatedBusinessKeys nvarchar(4000) 28 | 29 | IF @TableID IS NULL 30 | SELECT @TableID = t.TableID FROM Meta.config.edwTables t WHERE ((t.SchemaName + '.' + t.TableName =@TargetObject) OR t.TableName =@TargetObject) 31 | 32 | SELECT @TargetSchema =t.SchemaName, @TargetTable =t.TableName, @TableType =t.TableType , @SourceObject =t.SourceObject, @PrimaryKey=t.PrimaryKey, @BusinessKeys =t.BusinessKeys 33 | FROM Meta.config.edwTables t WHERE (t.SchemaName + '.' + t.TableName =@TargetObject) OR t.TableName =@TargetObject OR t.TableID = @TableID 34 | 35 | IF @TargetTable is not null SET @TargetObject=quotename(coalesce(@TargetSchema,'dbo')) + '.' + quotename(@TargetTable) 36 | IF @TargetTable is null 37 | BEGIN 38 | SET @TargetObject=coalesce(@TargetObject, convert(varchar(10),@TableID)) 39 | RAISERROR ('No meta data found in edwTables for %s',16,1,@TargetObject) 40 | END 41 | 42 | IF OBJECT_ID(@TargetObject) is NULL 43 | BEGIN 44 | SELECT @PrimaryKey = coalesce(@PrimaryKey, case when charindex (',',@BusinessKeys) =0 then @BusinessKeys end ,replace(@TargetTable, 'Dim','') + 'Key') 45 | SELECT @SourceColumns =string_agg(c.name, ',') FROM sys.columns c WHERE c.object_id = object_id(@SourceObject) 46 | 47 | SET @SQL = 'CREATE TABLE ' + @TargetObject + '(' + char(13) 48 | exec [dwa].[usp_TableCreateColumns] @TableID, @Columns output 49 | SET @sql=@sql + @Columns + char(13) + ');' 50 | PRINT @sql 51 | exec (@sql) 52 | SET @sql = NULL 53 | 54 | /*ALTER command to add Constraint*/ 55 | IF @TableType='Dim' 56 | SET @sql = 'ALTER TABLE ' +@TargetObject+' ADD CONSTRAINT [PK_' + @TargetTable + '] PRIMARY KEY NONCLUSTERED (' + @PrimaryKey + ' ASC) NOT ENFORCED;' 57 | 58 | PRINT @sql 59 | exec (@sql) 60 | /* 61 | IF @TableStructure like 'CLUSTERED INDEX%' 62 | BEGIN 63 | DECLARE @cxKeys sysname 64 | SET @cxKeys = substring(@TableStructure, charindex('(',@TableStructure)+1, charindex(')',@TableStructure)-charindex('(',@TableStructure)-1 ) 65 | SET @sql ='CREATE CLUSTERED INDEX CX_' + @TargetTable + ' ON ' + @TargetObject + ' (' + @cxKeys + ');' 66 | PRINT @sql 67 | exec (@sql) 68 | END*/ 69 | END 70 | /* NOT SUPPORTED 71 | /* Create PK and Indexes if Missing*/ 72 | IF @TableType='Dim' 73 | BEGIN 74 | IF NOT EXISTS (SELECT * From sys.indexes i where object_id=object_id(@TargetObject) and i.name ='UX_' + @TargetTable) 75 | BEGIN 76 | IF charindex(@BusinessKeys,',') > 0 77 | BEGIN 78 | SET @sql ='CREATE INDEX [UX_' + @TargetTable + '] ON ' + @TargetObject + '(' 79 | SET @sql=@sql + @PrimaryKey 80 | SET @sql=@sql + ' )' 81 | PRINT @sql 82 | exec (@sql) 83 | END 84 | END 85 | 86 | END 87 | */ 88 | 89 | END TRY 90 | BEGIN CATCH 91 | THROW 92 | END CATCH 93 | END -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dbo/StoredProcedures/sp_whoisactive.sql: -------------------------------------------------------------------------------- 1 | /********************************************************************************************* 2 | Fabric sp_WhoIsActive by Bob Duffy. bob@prodata.ie 3 | 4 | This is based on the SqlDbEngine sp_WhoIsActive 5 | (C) 2007-2018, Adam Machanic 6 | http://whoisactive.com 7 | 8 | *********************************************************************************************/ 9 | CREATE PROCEDURE [dbo].[sp_whoisactive] 10 | AS 11 | BEGIN 12 | SET NOCOUNT ON 13 | DECLARE @sql varchar(4000) 14 | DECLARE @queryHashJson varchar(8000) 15 | DECLARE @queryTimeJson varchar(8000) 16 | 17 | --Cache last execution times into JSON as not supported to Join System DMVs and user tables like QueryInsights 18 | --Todo: QueryInsights has seperate tables per database, but we only check active one. 19 | SELECT @queryHashJson='[' + string_agg ('{"query_hash":"' + replace(upper(sys.fn_varbintohexstr( query_hash)),'0X','0x')+ '"}',',') + ']' 20 | FROM sys.dm_exec_requests r 21 | INNER JOIN sys.dm_exec_sessions s on s.session_id =r.session_id 22 | WHERE r.session_id <> @@SPID 23 | AND s.program_name <> 'QueryInsights' 24 | 25 | ;WITH rh as ( 26 | SELECT distinct query_hash, FIRST_VALUE(total_elapsed_time_ms) OVER (PARTITION BY query_hash ORDER BY start_time ) as time_ms 27 | FROM queryinsights.exec_requests_history h 28 | WHERE query_hash IN 29 | (SELECT query_hash FROM OPENJSON(@queryhashJson) WITH (query_hash varchar(4000)) x) 30 | AND status='Succeeded' 31 | ) 32 | SELECT @queryTimeJson = '[' + string_agg ('{"query_hash":"' + query_hash + '","time_ms":"' + convert(varchar(50),time_ms) + '"}',',') + ']' 33 | FROM rh 34 | 35 | /* Query DMVs and Join to QueryInsights cached times in Json */ 36 | SELECT RIGHT('0' + CAST(r.total_elapsed_time / (1000 * 60 * 60 * 24) AS VARCHAR(10)),2) + ' ' + -- Days 37 | RIGHT('0' + CAST((r.total_elapsed_time / (1000 * 60 * 60)) % 24 AS VARCHAR(2)), 2) + ':' + -- Hours 38 | RIGHT('0' + CAST((r.total_elapsed_time / (1000 * 60)) % 60 AS VARCHAR(2)), 2) + ':' + -- Minutes 39 | RIGHT('0' + CAST((r.total_elapsed_time / 1000) % 60 AS VARCHAR(2)), 2) + '.' + -- Seconds 40 | RIGHT('00' + CAST(r.total_elapsed_time % 1000 AS VARCHAR(3)), 3) AS [dd hh:mm:ss.mss] 41 | , r.session_id 42 | , SUBSTRING(st.text, (r.statement_start_offset / 2)+1, ((CASE statement_end_offset WHEN -1 THEN DATALENGTH(st.text)ELSE r.statement_end_offset END-r.statement_start_offset)/ 2)+1) AS sql_text 43 | , s.login_name 44 | , r.wait_type + ':' + CONVERT(varchar, r.wait_time) as wait_info 45 | , r.cpu_time as cpu 46 | , r.blocking_session_id 47 | , r.logical_reads as reads 48 | , r.writes 49 | , r.reads as physical_reads 50 | , r.status 51 | , r.open_transaction_count as open_tran_count 52 | , CASE WHEN r.percent_complete > 0 OR qt.time_ms is null THEN r.percent_complete else convert(decimal (9,2),r.total_elapsed_time /qt.time_ms * 100) END as percent_complete 53 | , CASE WHEN r.percent_complete > 0 OR coalesce(qt.time_ms,0) < 100 THEN null else dateadd(ss,convert(int,qt.time_ms/1000), r.start_time) END as eta_time 54 | , s.host_name 55 | , db_name(s.database_id) as database_name 56 | , s.program_name 57 | , r.start_time 58 | , s.login_time 59 | , r.request_id 60 | , r.query_plan_hash 61 | , r.query_hash 62 | , GETDATE() as collection_time 63 | FROM sys.dm_exec_requests r 64 | CROSS APPLY sys.dm_exec_sql_text(r.sql_handle) st 65 | INNER JOIN sys.dm_exec_sessions s on s.session_id =r.session_id 66 | LEFT JOIN OPENJSON(@queryTimeJson) WITH ( 67 | query_hash varchar(50) , 68 | time_ms float 69 | ) qt ON convert(varbinary(max), qt.query_hash,1)=r.query_hash 70 | WHERE r.session_id <> @@SPID 71 | AND s.program_name <> 'QueryInsights' 72 | END -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Extract/Extract-JSON.Notebook/notebook-content.py: -------------------------------------------------------------------------------- 1 | # Fabric notebook source 2 | 3 | # METADATA ******************** 4 | 5 | # META { 6 | # META "kernel_info": { 7 | # META "name": "synapse_pyspark" 8 | # META }, 9 | # META "dependencies": { 10 | # META "lakehouse": { 11 | # META "default_lakehouse": "19785e4d-5572-4ced-bfab-f26e7c5de3ce", 12 | # META "default_lakehouse_name": "FabricLH", 13 | # META "default_lakehouse_workspace_id": "9b8a6500-5ccb-49a9-885b-b5b081efed75", 14 | # META "known_lakehouses": [ 15 | # META { 16 | # META "id": "19785e4d-5572-4ced-bfab-f26e7c5de3ce" 17 | # META }, 18 | # META { 19 | # META "id": "f2f9c5fa-ca0c-41b2-b0e1-3028165b4f6c" 20 | # META } 21 | # META ] 22 | # META } 23 | # META } 24 | # META } 25 | 26 | # MARKDOWN ******************** 27 | 28 | # #### Extract-JSON 29 | # 30 | # This notebook ingests a JSON file from a configured Data Lake path into a Delta table within the Fabric Lakehouse. It flattens the JSON structure using a specified nodes in ActivitySettings, adds lineage and file metadata, and writes the data to the target table using the defined write mode (e.g., overwrite). 31 | 32 | # PARAMETERS CELL ******************** 33 | 34 | SourceSettings ='{"Directory" : "unittest/JSON", "File" : "simple.json"}' 35 | TargetSettings ='{"TableName" : "simpleJSON", "SchemaName" :"dbo","mode":"overwrite" }' 36 | ActivitySettings='{"record_path": "book"}' 37 | SourceConnectionSettings = None 38 | TargetConnectionSettings = None 39 | LineageKey = 1 40 | 41 | # METADATA ******************** 42 | 43 | # META { 44 | # META "language": "python", 45 | # META "language_group": "synapse_pyspark" 46 | # META } 47 | 48 | # CELL ******************** 49 | 50 | import pandas as pd 51 | import json 52 | import os 53 | 54 | SourceSettings=SourceSettings or "{}" 55 | TargetSettings=TargetSettings or "{}" 56 | ActivitySettings=ActivitySettings or '{}' 57 | 58 | if ActivitySettings: 59 | activity_settings = json.loads(ActivitySettings) 60 | 61 | source_settings = json.loads(SourceSettings) 62 | target_settings = json.loads(TargetSettings) 63 | 64 | source_directory = source_settings["Directory"] 65 | source_file = source_settings["File"] 66 | 67 | target_schema = target_settings.get("SchemaName", "dbo") 68 | target_table = target_settings.get("TableName", source_file.split(".")[0]) 69 | 70 | if target_schema != "dbo": 71 | target_table = f"{target_schema}_{target_table}" 72 | 73 | 74 | FILES_PREFIX = "Files" 75 | LAKEHOUSE_PREFIX = "/lakehouse/default" 76 | if not source_directory.startswith(FILES_PREFIX): 77 | source_directory = os.path.join(FILES_PREFIX, source_directory) 78 | if not source_directory.startswith(LAKEHOUSE_PREFIX): 79 | source_directory = os.path.join(LAKEHOUSE_PREFIX, source_directory) 80 | 81 | file_path = os.path.join(source_directory, source_file) 82 | 83 | 84 | mode = target_settings.get("mode","overwrite") 85 | 86 | 87 | # METADATA ******************** 88 | 89 | # META { 90 | # META "language": "python", 91 | # META "language_group": "synapse_pyspark" 92 | # META } 93 | 94 | # CELL ******************** 95 | 96 | with open(file_path, 'r') as f: 97 | json_data = json.load(f) 98 | 99 | df = pd.json_normalize(json_data, **activity_settings) 100 | df["LineageKey"] = LineageKey 101 | df["File"] = source_file 102 | 103 | row_count = df.shape[0] 104 | 105 | if mode == "overwrite": 106 | spark.sql(f"DROP TABLE IF EXISTS {target_table}") 107 | spark.createDataFrame(df).write.mode(mode).options(**target_settings).format("delta").saveAsTable(target_table) 108 | 109 | print(f"Wrote {row_count} rows from {file_path} to FabricLH.dbo.{target_table}.") 110 | 111 | # METADATA ******************** 112 | 113 | # META { 114 | # META "language": "python", 115 | # META "language_group": "synapse_pyspark" 116 | # META } 117 | -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Shared/MD-Sync.Notebook/notebook-content.py: -------------------------------------------------------------------------------- 1 | # Fabric notebook source 2 | 3 | # METADATA ******************** 4 | 5 | # META { 6 | # META "kernel_info": { 7 | # META "name": "synapse_pyspark" 8 | # META }, 9 | # META "dependencies": { 10 | # META "lakehouse": { 11 | # META "default_lakehouse": "f2f9c5fa-ca0c-41b2-b0e1-3028165b4f6c", 12 | # META "default_lakehouse_name": "FabricLH", 13 | # META "default_lakehouse_workspace_id": "9b8a6500-5ccb-49a9-885b-b5b081efed75" 14 | # META } 15 | # META } 16 | # META } 17 | 18 | # CELL ******************** 19 | 20 | import json 21 | import time 22 | from pyspark.sql import functions as fn 23 | from datetime import datetime 24 | import sempy.fabric as fabric 25 | from sempy.fabric.exceptions import FabricHTTPException 26 | workspace_id = spark.conf.get("trident.workspace.id") 27 | lakehouse_id = spark.conf.get("trident.lakehouse.id") 28 | if not lakehouse_id: 29 | raise Exception("No lakehouse is attached to this notebook.") 30 | client = fabric.FabricRestClient() 31 | 32 | sql_endpoint_id = client.get(f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}").json()['properties']['sqlEndpointProperties']['id'] 33 | 34 | md_sync_start_time = datetime.now() 35 | 36 | try: 37 | response = client.post(f"/v1.0/myorg/lhdatamarts/{sql_endpoint_id}",json= {"commands":[{"$type":"MetadataRefreshCommand"}]}) 38 | data = json.loads(response.text or '{}') 39 | batchId = data["batchId"] 40 | progressState = data["progressState"] 41 | except FabricHTTPException as e: 42 | response = e.response 43 | data = response.json() 44 | if response.status_code == 400 and data.get("error", {"code":None}).get("code", None) == 'LockConflict': # safer execution - reduce risk of errors 45 | details = data["error"]["pbi.error"]["details"] 46 | batchId = next( 47 | (item["detail"]["value"] for item in data["error"]["pbi.error"]["details"] if item["code"] == "LockingBatchId"), 48 | None) 49 | 50 | print(f"MD Sync already running - using already running batch ID ({batchId})") 51 | progressState = 'inProgress' 52 | else: 53 | raise e 54 | 55 | statusuri = f"/v1.0/myorg/lhdatamarts/{sql_endpoint_id}/batches/{batchId}" 56 | 57 | statusresponsedata = "" 58 | 59 | initial_delay = 1.0 60 | max_delay = 32.0 61 | 62 | while progressState == 'inProgress': 63 | print(f".", end='') 64 | statusresponsedata = client.get(statusuri).json() 65 | 66 | progressState = statusresponsedata["progressState"] 67 | 68 | 69 | if progressState == 'inProgress': 70 | time.sleep(initial_delay) 71 | initial_delay = min(initial_delay * 2.0, max_delay) 72 | 73 | md_sync_finish_time = datetime.now() 74 | 75 | print(f"\nMD Sync call finished in {(md_sync_finish_time - md_sync_start_time).total_seconds()}s") 76 | 77 | class MDSyncFailed(Exception): 78 | pass 79 | if progressState == 'success': 80 | table_details = [ 81 | { 82 | 'tableName': table['tableName'], 83 | 'warningMessages': table.get('warningMessages', []), 84 | 'lastSuccessfulUpdate': table.get('lastSuccessfulUpdate', 'N/A'), 85 | 'tableSyncState': table['tableSyncState'], 86 | 'sqlSyncState': table['sqlSyncState'] 87 | } 88 | for table in statusresponsedata['operationInformation'][0]['progressDetail']['tablesSyncStatus'] if table['tableSyncState'] != "NotRun" 89 | ] 90 | if not table_details: 91 | print("No tables synced!") 92 | mssparkutils.notebook.exit(0) 93 | 94 | print("Tables synced:") 95 | for detail in table_details: 96 | print(f"Table: {detail['tableName']}\n\t - tableSyncState: {detail['tableSyncState']}\n\t - Warnings: {detail['warningMessages']}") 97 | else: 98 | raise MDSyncFailed(json.dumps(statusresponsedata)) 99 | 100 | # METADATA ******************** 101 | 102 | # META { 103 | # META "language": "python", 104 | # META "language_group": "synapse_pyspark" 105 | # META } 106 | -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/Library/Shared/MD-Sync Python.Notebook/notebook-content.py: -------------------------------------------------------------------------------- 1 | # Fabric notebook source 2 | 3 | # METADATA ******************** 4 | 5 | # META { 6 | # META "kernel_info": { 7 | # META "name": "jupyter", 8 | # META "jupyter_kernel_name": "python3.11" 9 | # META }, 10 | # META "dependencies": { 11 | # META "lakehouse": { 12 | # META "default_lakehouse": "f2f9c5fa-ca0c-41b2-b0e1-3028165b4f6c", 13 | # META "default_lakehouse_name": "FabricLH", 14 | # META "default_lakehouse_workspace_id": "9b8a6500-5ccb-49a9-885b-b5b081efed75" 15 | # META } 16 | # META } 17 | # META } 18 | 19 | # PARAMETERS CELL ******************** 20 | 21 | 22 | # METADATA ******************** 23 | 24 | # META { 25 | # META "language": "python", 26 | # META "language_group": "jupyter_python" 27 | # META } 28 | 29 | # CELL ******************** 30 | 31 | import json 32 | import time 33 | from pyspark.sql import functions as fn 34 | from datetime import datetime 35 | import sempy.fabric as fabric 36 | from sempy.fabric.exceptions import FabricHTTPException 37 | workspace_id = spark.conf.get("trident.workspace.id") 38 | lakehouse_id = spark.conf.get("trident.lakehouse.id") 39 | if not lakehouse_id: 40 | raise Exception("No lakehouse is attached to this notebook.") 41 | client = fabric.FabricRestClient() 42 | 43 | sql_endpoint_id = client.get(f"/v1/workspaces/{workspace_id}/lakehouses/{lakehouse_id}").json()['properties']['sqlEndpointProperties']['id'] 44 | 45 | md_sync_start_time = datetime.now() 46 | 47 | try: 48 | response = client.post(f"/v1.0/myorg/lhdatamarts/{sql_endpoint_id}",json= {"commands":[{"$type":"MetadataRefreshCommand"}]}) 49 | data = json.loads(response.text or '{}') 50 | batchId = data["batchId"] 51 | progressState = data["progressState"] 52 | except FabricHTTPException as e: 53 | response = e.response 54 | data = response.json() 55 | if response.status_code == 400 and data.get("error", {"code":None}).get("code", None) == 'LockConflict': # safer execution - reduce risk of errors 56 | details = data["error"]["pbi.error"]["details"] 57 | batchId = next( 58 | (item["detail"]["value"] for item in data["error"]["pbi.error"]["details"] if item["code"] == "LockingBatchId"), 59 | None) 60 | 61 | print(f"MD Sync already running - using already running batch ID ({batchId})") 62 | progressState = 'inProgress' 63 | else: 64 | raise e 65 | 66 | statusuri = f"/v1.0/myorg/lhdatamarts/{sql_endpoint_id}/batches/{batchId}" 67 | 68 | statusresponsedata = "" 69 | 70 | initial_delay = 1.0 71 | max_delay = 32.0 72 | 73 | while progressState == 'inProgress': 74 | print(f".", end='') 75 | statusresponsedata = client.get(statusuri).json() 76 | 77 | progressState = statusresponsedata["progressState"] 78 | 79 | 80 | if progressState == 'inProgress': 81 | time.sleep(initial_delay) 82 | initial_delay = min(initial_delay * 2.0, max_delay) 83 | 84 | md_sync_finish_time = datetime.now() 85 | 86 | print(f"\nMD Sync call finished in {(md_sync_finish_time - md_sync_start_time).total_seconds()}s") 87 | 88 | class MDSyncFailed(Exception): 89 | pass 90 | if progressState == 'success': 91 | table_details = [ 92 | { 93 | 'tableName': table['tableName'], 94 | 'warningMessages': table.get('warningMessages', []), 95 | 'lastSuccessfulUpdate': table.get('lastSuccessfulUpdate', 'N/A'), 96 | 'tableSyncState': table['tableSyncState'], 97 | 'sqlSyncState': table['sqlSyncState'] 98 | } 99 | for table in statusresponsedata['operationInformation'][0]['progressDetail']['tablesSyncStatus'] if table['tableSyncState'] != "NotRun" 100 | ] 101 | if not table_details: 102 | print("No tables synced!") 103 | mssparkutils.notebook.exit(0) 104 | 105 | print("Tables synced:") 106 | for detail in table_details: 107 | print(f"Table: {detail['tableName']}\n\t - tableSyncState: {detail['tableSyncState']}\n\t - Warnings: {detail['warningMessages']}") 108 | else: 109 | raise MDSyncFailed(json.dumps(statusresponsedata)) 110 | 111 | # METADATA ******************** 112 | 113 | # META { 114 | # META "language": "python", 115 | # META "language_group": "jupyter_python" 116 | # META } 117 | -------------------------------------------------------------------------------- /Workspaces/DWA/DW.Warehouse/dwa/StoredProcedures/usp_TableCreateColumns.sql: -------------------------------------------------------------------------------- 1 | /* Description: Return the Column Meta Data DDL for a CREATE TABLE, CTAS, Polybase or other method in SQLDB or SQLDW 2 | Example: 3 | DECLARE @Columns nvarchar(max) 4 | exec [dwa].[usp_TableCreateColumns] 2, @Columns output /* Dim */ 5 | PRINT @Columns 6 | 7 | DECLARE @Columns nvarchar(max) 8 | exec [dwa].[usp_TableCreateColumns] 7,@Columns output /* Fact */ 9 | PRINT @Columns 10 | 11 | DECLARE @Columns nvarchar(max) 12 | exec [dwa].[usp_TableCreateColumns] 4,@Columns output /* Dim */ 13 | PRINT @Columns 14 | History: 15 | 02/08/2023 Deepak, Created 16 | */ 17 | CREATE PROC [dwa].[usp_TableCreateColumns] @TableID [int],@Columns [varchar](4000) OUT AS 18 | BEGIN 19 | 20 | --SET XACT_ABORT ON 21 | SET NOCOUNT ON; 22 | 23 | BEGIN TRY 24 | DECLARE @SourceObject sysname 25 | , @RelatedBK nvarchar(4000) 26 | , @BusinessKeys nvarchar(4000) 27 | , @PrimaryKey sysname 28 | , @TableType sysname 29 | , @ColumnsPK nvarchar(4000) 30 | , @DeDupeFlag bit /* Flag to check if DeDupeRows is ON for TableID */ 31 | , @HideColumns nvarchar(4000) /* Array of Columns to Hide */ 32 | , @PKs VARCHAR(400) 33 | , @BaseDims VARCHAR(400) 34 | 35 | SELECT @SourceObject =SourceObject, @BusinessKeys=BusinessKeys, @TableType=TableType, @PrimaryKey=PrimaryKey,@DeDupeFlag = DedupeFlag 36 | FROM Meta.config.edwTables 37 | WHERE TableID=@TableID 38 | 39 | SELECT @RelatedBK = string_agg(r.BusinessKeys, ',') 40 | FROM Meta.config.edwTableJoins j 41 | INNER JOIN Meta.config.edwTables r on j.RelatedTableID =r.TableID 42 | WHERE j.TableID=@TableID 43 | 44 | IF @DeDupeFlag=1 45 | SET @HideColumns =coalesce(@HideColumns + ',','') + 'RowVersionNo'; 46 | 47 | /* Primary Keys (if any) from joined table if Fact or Star Schema) */ 48 | SELECT @PKs = string_agg(r.PrimaryKey, ','), @BaseDims = string_agg(r.SchemaName + '.' + r.TableName, ',') 49 | FROM Meta.config.edwTableJoins j 50 | INNER JOIN Meta.config.edwTables r ON j.RelatedTableID = r.TableID 51 | WHERE j.TableID = @TableID 52 | 53 | SET @ColumnsPK = (SELECT string_agg( c.name + ' ' + d.name + case when c.collation_name is not null then '(' + convert (varchar(50),COALESCE(c.max_length/(length/prec),c.max_length) ) + ')' else '' end + CASE WHEN c.is_nullable=0 then ' NOT NULL' ELSE ' NULL' END 54 | , char(13) + char(9) + ',' ) WITHIN GROUP ( ORDER BY column_id) 55 | FROM sys.schemas s 56 | INNER JOIN sys.tables t ON s.schema_id = t.schema_id 57 | INNER JOIN sys.columns c ON t.object_id = c.object_id 58 | INNER JOIN sys.types d ON d.user_type_id = c.user_type_id 59 | LEFT JOIN sys.systypes stt ON stt.name = t.name AND stt.name IN ('varchar', 'nvarchar', 'nchar') 60 | WHERE s.name <> 'sys' 61 | AND s.name + '.' + t.name IN ( SELECT ltrim(value) FROM string_split(@BaseDims,',')) 62 | AND c.name IN (SELECT ltrim(value) FROM string_split(@PKs,',')) 63 | ) 64 | 65 | IF @ColumnsPK is NOT NULL 66 | SET @Columns = coalesce(@Columns + char(13) + char(9)+ ',' , char(9) ) + @ColumnsPK 67 | 68 | SET @columns = coalesce(@columns + CHAR(13) + CHAR(9) + ',', CHAR(9)) + ( 69 | SELECT TOP 255 string_agg(c.name + ' ' + t.name 70 | + CASE WHEN c.collation_name IS NOT NULL THEN '(' + CONVERT(VARCHAR(50), coalesce(c.max_length / (length / prec), c.max_length)) + ')' 71 | WHEN t.name = 'decimal' THEN '(' + CONVERT(VARCHAR(10), c.precision) + ',' + CONVERT(VARCHAR(10), c.scale) + ')' 72 | ELSE '' 73 | END 74 | + CASE WHEN bk.columnname IS NOT NULL OR c.is_nullable = 0 THEN ' NOT NULL' 75 | ELSE ' NULL' 76 | END, CHAR(13) + CHAR(9) + ',') WITHIN GROUP (ORDER BY column_id) 77 | FROM sys.columns c 78 | LEFT JOIN (SELECT ltrim(value) AS columnname FROM string_split(@businesskeys, ',')) bk ON c.name = bk.columnname 79 | INNER JOIN sys.types t ON t.user_type_id = c.user_type_id 80 | LEFT JOIN sys.systypes stt ON stt.name = t.name AND stt.name IN ('varchar', 'nvarchar', 'nchar') 81 | WHERE c.object_id = object_id(@sourceobject) 82 | AND c.name NOT IN (SELECT ltrim(value) FROM string_split(@relatedbk , ',')) 83 | AND c.name NOT IN (SELECT ltrim(value) FROM string_split(@hidecolumns, ',')) 84 | ) 85 | END TRY 86 | BEGIN CATCH 87 | THROW 88 | END CATCH 89 | END -------------------------------------------------------------------------------- /Workspaces/DWA/DWA/ArchiveFiles.Notebook/notebook-content.py: -------------------------------------------------------------------------------- 1 | # Fabric notebook source 2 | 3 | # METADATA ******************** 4 | 5 | # META { 6 | # META "kernel_info": { 7 | # META "name": "synapse_pyspark" 8 | # META }, 9 | # META "dependencies": { 10 | # META "lakehouse": { 11 | # META "default_lakehouse": "19785e4d-5572-4ced-bfab-f26e7c5de3ce", 12 | # META "default_lakehouse_name": "FabricLH", 13 | # META "default_lakehouse_workspace_id": "9b8a6500-5ccb-49a9-885b-b5b081efed75", 14 | # META "known_lakehouses": [ 15 | # META { 16 | # META "id": "19785e4d-5572-4ced-bfab-f26e7c5de3ce" 17 | # META } 18 | # META ] 19 | # META } 20 | # META } 21 | # META } 22 | 23 | # MARKDOWN ******************** 24 | 25 | # #### Archivefiles 26 | # 27 | # This notebook connects to a SQL Server database, retrieves file details via a stored procedure, and moves files from a source directory to an archive directory within a lakehouse structure. It handles file operations, error checking, and updates the database to mark the files as archived. 28 | 29 | # PARAMETERS CELL ******************** 30 | 31 | Server = 'fkm4vwf6l6zebg4lqrhbtdcmsq-absyvg6llsuutcc3wwyid37nou.datawarehouse.pbidedicated.windows.net' 32 | Database = 'FabricDW' 33 | PackageGroup='AW' 34 | 35 | # METADATA ******************** 36 | 37 | # META { 38 | # META "language": "python", 39 | # META "language_group": "synapse_pyspark" 40 | # META } 41 | 42 | # CELL ******************** 43 | 44 | import pyodbc 45 | import struct 46 | import os 47 | import shutil 48 | 49 | LAKEHOUSE_PREFIX = "/lakehouse/default/" 50 | # Obtain an access token 51 | token = mssparkutils.credentials.getToken("https://analysis.windows.net/powerbi/api") 52 | token_bytes = token.encode("UTF-16-LE") 53 | token_struct = struct.pack(f'