├── .gitattributes ├── .gitignore ├── COPYING.txt ├── README.md ├── UpgradeReport.sarif ├── Virtual_EDW ├── App.config ├── CustomTabPage.cs ├── ExtensionMethods.cs ├── FodyWeavers.xml ├── FodyWeavers.xsd ├── Form_About.Designer.cs ├── Form_About.cs ├── Form_About.resx ├── Form_Alert.Designer.cs ├── Form_Alert.cs ├── Form_Alert.resx ├── Form_Base.Designer.cs ├── Form_Base.cs ├── Form_Base.resx ├── Form_Main.Designer.cs ├── Form_Main.cs ├── Form_Main.resx ├── GlobalSuppressions.cs ├── InterfaceObjectModel.cs ├── Libraries │ ├── Microsoft.Data.SqlClient.SNI.x64.dll │ ├── Microsoft.Data.SqlClient.SNI.x86.dll │ ├── Microsoft.Data.SqlClient.dll │ ├── Snowflake.Data.dll │ └── TEAM_Library.dll ├── LoadPatternGrid.cs ├── Metadata │ ├── DIM_CUSTOMER.json │ ├── DIM_CUSTOMER_DAILY_SNAPSHOT.json │ ├── DIM_CUSTOMER_Virtual.json │ ├── HUB_CUSTOMER.json │ ├── HUB_INCENTIVE_OFFER.json │ ├── HUB_MEMBERSHIP_PLAN.json │ ├── HUB_SEGMENT.json │ ├── LNK_CUSTOMER_COSTING.json │ ├── LNK_CUSTOMER_OFFER.json │ ├── LNK_MEMBERSHIP.json │ ├── LNK_RENEWAL_MEMBERSHIP.json │ ├── LSAT_CUSTOMER_COSTING.json │ ├── LSAT_CUSTOMER_OFFER.json │ ├── LSAT_MEMBERSHIP.json │ ├── PSA_PROFILER_CUSTOMER_CONTACT.json │ ├── PSA_PROFILER_CUSTOMER_OFFER.json │ ├── PSA_PROFILER_CUSTOMER_PERSONAL.json │ ├── PSA_PROFILER_CUST_MEMBERSHIP.json │ ├── PSA_PROFILER_ESTIMATED_WORTH.json │ ├── PSA_PROFILER_OFFER.json │ ├── PSA_PROFILER_PERSONALISED_COSTING.json │ ├── PSA_PROFILER_PLAN.json │ ├── PSA_USERMANAGED_SEGMENT.json │ ├── SAT_CUSTOMER.json │ ├── SAT_CUSTOMER_CONTACT_DETAILS.json │ ├── SAT_CUSTOMER_DERIVED.json │ ├── SAT_INCENTIVE_OFFER.json │ ├── SAT_MEMBERSHIP_PLAN_DETAIL.json │ ├── SAT_MEMBERSHIP_PLAN_VALUATION.json │ ├── SAT_SEGMENT.json │ ├── STG_PROFILER_CUSTOMER_CONTACT.json │ ├── STG_PROFILER_CUSTOMER_OFFER.json │ ├── STG_PROFILER_CUSTOMER_PERSONAL.json │ ├── STG_PROFILER_CUST_MEMBERSHIP.json │ ├── STG_PROFILER_ESTIMATED_WORTH.json │ ├── STG_PROFILER_OFFER.json │ ├── STG_PROFILER_PERSONALISED_COSTING.json │ └── STG_PROFILER_PLAN.json ├── Program.cs ├── Properties │ ├── AssemblyInfo.cs │ ├── Resources.Designer.cs │ ├── Resources.resx │ └── app.manifest ├── Resources │ ├── CogIcon.ico │ ├── CubeIcon.ico │ ├── DocumentationIcon.ico │ ├── ETLIcon.ico │ ├── ExitApplication.ico │ ├── HelpIconSmall.ico │ ├── LinkIcon.ico │ ├── OpenDirectoryIcon.ico │ ├── OpenFileIcon.ico │ ├── Ravos-logo-small.ico │ ├── Ravos-logo-small_transparant.ico │ ├── SaveFile.ico │ ├── Time.png │ ├── database_icon.png │ ├── ghost-icon.png │ ├── log-file.png │ └── transparent-green-checkmark-hi.png ├── Schema │ └── interfaceDataWarehouseAutomationMetadata.json ├── TemplateHandling.cs ├── Templates │ ├── loadPatternHubInsertIntofromSTG.Handlebars │ ├── loadPatternLinkInsertIntofromSTG.Handlebars │ ├── loadPatternLinkSatelliteInsertInto.Handlebars │ ├── loadPatternLinkSatelliteReinitialisation.Handlebars │ ├── loadPatternPersistentStagingAreaInsertInto.Handlebars │ ├── loadPatternSatelliteInsertInto.Handlebars │ ├── loadPatternSatelliteReinitialisation.Handlebars │ ├── loadPattern_UT_HUB_SK_has_SAT_record.Handlebars │ ├── loadPattern_UT_HUB_SKgenerationCheck.Handlebars │ ├── loadPattern_UT_HUB_uniqueKeyCheck.Handlebars │ ├── loadPattern_UT_LNK_HUB_SK_existsCheck.Handlebars │ ├── loadPattern_UT_LNK_SKgenerationCheck.Handlebars │ ├── loadPattern_UT_LNK_keyTrimCheck.Handlebars │ ├── loadPattern_UT_LNK_nullKeyCheck.Handlebars │ ├── loadPattern_UT_LNK_uniqueKeyCheck.Handlebars │ ├── loadPattern_UT_LSAT_SK_has_LNK_record.Handlebars │ ├── loadPattern_UT_LSAT_currentRecordCheck.Handlebars │ ├── loadPattern_UT_LSAT_duplicateRecordCheck.Handlebars │ ├── loadPattern_UT_PSA_STG_to_PSA_Check.Handlebars │ ├── loadPattern_UT_PSA_currentRecordCheck.Handlebars │ ├── loadPattern_UT_PSA_duplicateRecordCheck.Handlebars │ ├── loadPattern_UT_SAT_SK_has_HUB_record.Handlebars │ ├── loadPattern_UT_SAT_currentRecordCheck.Handlebars │ ├── loadPattern_UT_SAT_duplicateRecordCheck.Handlebars │ ├── loadPattern_UT_SAT_nullKeyCheck.Handlebars │ ├── templateCollection.json │ ├── templateDataLakeSilver01.Handlebars │ ├── templateDataLakeSilver01CDC.Handlebars │ ├── templateDataLakeSilver02.Handlebars │ ├── templateDataLakeSilver02CDC.Handlebars │ ├── templateDataLakeSilver03.Handlebars │ ├── templateDataLakeSilverToStagingSelectCDC.Handlebars │ ├── templateDimSelect.Handlebars │ ├── templateHubDeltaSelect.Handlebars │ ├── templateHubDirectRegistration.Handlebars │ ├── templateHubGenerateTables.Handlebars │ ├── templateHubInsertInto.Handlebars │ ├── templateHubInsertIntoDirectWrapper.Handlebars │ ├── templateHubInsertIntoProcedure.Handlebars │ ├── templateHubView.Handlebars │ ├── templateHubViewWithDeliveryKey.Handlebars │ ├── templateHubView_DBT_Snowflake.Handlebars │ ├── templateLanding.Handlebars │ ├── templateLandingTestReconciliation.Handlebars │ ├── templateLandingWithDirect.Handlebars │ ├── templateLandingtoSilverFromView.Handlebars │ ├── templateLinkDirectRegistration.Handlebars │ ├── templateLinkGenerateTables.Handlebars │ ├── templateLinkInsertInto.Handlebars │ ├── templateLinkInsertIntoProcedure.Handlebars │ ├── templateLinkSatelliteDirectRegistration.Handlebars │ ├── templateLinkSatelliteDrivingKeyInsertInto.Handlebars │ ├── templateLinkSatelliteDrivingKeyView.Handlebars │ ├── templateLinkSatelliteInsertInto.Handlebars │ ├── templateLinkSatelliteInsertIntoProcedure.Handlebars │ ├── templateLinkSatelliteView.Handlebars │ ├── templateLinkSatelliteViewZeroRecord.Handlebars │ ├── templateLinkView.Handlebars │ ├── templateLinkViewNoHash.Handlebars │ ├── templatePersistentStagingGenerateTables.Handlebars │ ├── templatePersistentStagingInsertInto.Handlebars │ ├── templatePersistentStagingInsertIntoProcedure.Handlebars │ ├── templatePersistentStagingStoredProcedureDelta.Handlebars │ ├── templatePersistentStagingStoredProcedureFOJ.Handlebars │ ├── templatePersistentStagingView.Handlebars │ ├── templatePersistentStagingViewSimplified.Handlebars │ ├── templatePersistentStagingViewWithDedup.Handlebars │ ├── templatePresentationDailySnapshot.Handlebars │ ├── templatePresentationGenerateTables.Handlebars │ ├── templateSatelliteDirectRegistration.Handlebars │ ├── templateSatelliteGenerateTables.Handlebars │ ├── templateSatelliteInsertInto.Handlebars │ ├── templateSatelliteInsertIntoProcedure.Handlebars │ ├── templateSatelliteReferentialIntegrity.Handlebars │ ├── templateSatelliteView.Handlebars │ ├── templateSatelliteViewWithDeliveryKey.Handlebars │ ├── templateSatelliteViewZeroRecord.Handlebars │ ├── templateStagingDirectRegistration.Handlebars │ ├── templateStagingGenerateTables.Handlebars │ ├── templateStagingInsertInto.Handlebars │ ├── templateStagingStoredProcedureDelta.Handlebars │ ├── templateStagingStoredProcedureLanding.Handlebars │ ├── templateStagingTestData.Handlebars │ ├── templateStagingView.Handlebars │ ├── template_test_HUB_KeyNullCheck.Handlebars │ ├── template_test_HUB_KeyTrimCheck.Handlebars │ ├── template_test_Hub_HasSatRecord.Handlebars │ ├── template_test_Hub_KeyGeneratedCorrectly.Handlebars │ ├── template_test_Hub_UniqueKeyCheck.Handlebars │ ├── template_test_LinkSatellite_DuplicateRecordCheck.Handlebars │ ├── template_test_LinkSatellite_ReferentialIntegrity.Handlebars │ ├── template_test_Link_KeyGeneratedCorrectly.Handlebars │ ├── template_test_Link_KeyNullCheck.Handlebars │ ├── template_test_Link_KeyTrimCheck.Handlebars │ ├── template_test_Link_ReferentialIntegrity.Handlebars │ ├── template_test_Link_UniqueKeyCheck.Handlebars │ ├── template_test_PersistentStaging_DuplicateRecordCheck.Handlebars │ ├── template_test_PersistentStaging_MappingCheck.Handlebars │ ├── template_test_SAT_StagingMappingCheck.Handlebars │ ├── template_test_Satellite_CurrentRecordUniqueCheck.Handlebars │ ├── template_test_Satellite_DuplicateRecordCheck.Handlebars │ ├── template_test_Satellite_KeyNullCheck.Handlebars │ ├── template_test_Satellite_ReferentialIntegrity.Handlebars │ └── template_test_Satellite_StagingMappingCheck.Handlebars ├── TextHandling.cs ├── VdwUtility.cs ├── Virtual_Data_Warehouse.csproj ├── Virtual_Data_Warehouse.csproj.DotSettings └── Virtual_Data_Warehouse.deps.json ├── Virtual_EDW_Installer └── Virtual_EDW_Installer.vdproj ├── Virtual_Enterprise_Data_Warehouse.sln └── upgrade-assistant.clef /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Virtual Data Warehouse 2 | 3 | The Virtual Data Warehouse (VDW) and the associated metadata management software TEAM are free tools to quickly prototype your Data Warehouse model (output), validate your metadata and generally get insight in the your Data Warehouse patterns. 4 | 5 | VDW can be used to test out various new patterns and ideas, as well as regression testing for pattern changes in various technologies and platforms.. 6 | 7 | The key concepts are related to the idea that, if you can generate all your ETL and you can ‘replay’ your entire Data Warehouse (when you refactor for instance – using Persistent Staging Areas), you can basically virtualise your entire Data Warehouse (I coined the term ‘NoETL’ for this :-)). 8 | 9 | Your Data Warehouse model essentially becomes a ‘schema-on-read’ on your raw data. 10 | 11 | More information at present is here: http://roelantvos.com/blog/articles-and-white-papers/virtualisation-software/. 12 | 13 | # A library of patterns 14 | 15 | VDW comes with a library of patterns based on Handlebars as templating engine. With this it is possible to generate a sample Data Warehouse using Data Vault methodology. 16 | -------------------------------------------------------------------------------- /Virtual_EDW/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /Virtual_EDW/ExtensionMethods.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.ComponentModel; 4 | using System.Data; 5 | using Microsoft.Win32; 6 | 7 | namespace Virtual_Data_Warehouse 8 | { 9 | /// 10 | /// Extension methods 11 | /// 12 | public static class ExtensionMethod 13 | { 14 | public static DataTable ToDataTable(this IList data) 15 | { 16 | PropertyDescriptorCollection props = 17 | TypeDescriptor.GetProperties(typeof(T)); 18 | DataTable table = new DataTable(); 19 | for (int i = 0; i < props.Count; i++) 20 | { 21 | PropertyDescriptor prop = props[i]; 22 | table.Columns.Add(prop.Name, prop.PropertyType); 23 | } 24 | object[] values = new object[props.Count]; 25 | foreach (T item in data) 26 | { 27 | for (int i = 0; i < values.Length; i++) 28 | { 29 | values[i] = props[i].GetValue(item); 30 | } 31 | table.Rows.Add(values); 32 | } 33 | return table; 34 | } 35 | 36 | private static string CleanifyBrowserPath(string p) 37 | { 38 | string[] url = p.Split('"'); 39 | string clean = url[1]; 40 | return clean; 41 | } 42 | 43 | public static string GetDefaultBrowserPath() 44 | { 45 | string urlAssociation = @"Software\Microsoft\Windows\Shell\Associations\UrlAssociations\http"; 46 | string browserPathKey = @"$BROWSER$\shell\open\command"; 47 | 48 | Microsoft.Win32.RegistryKey userChoiceKey = null; 49 | string browserPath = ""; 50 | 51 | try 52 | { 53 | //Read default browser path from userChoiceLKey 54 | userChoiceKey = Microsoft.Win32.Registry.CurrentUser.OpenSubKey(urlAssociation + @"\UserChoice", false); 55 | 56 | //If user choice was not found, try machine default 57 | if (userChoiceKey == null) 58 | { 59 | //Read default browser path from Win XP registry key 60 | var browserKey = Registry.ClassesRoot.OpenSubKey(@"HTTP\shell\open\command", false); 61 | 62 | //If browser path wasn’t found, try Win Vista (and newer) registry key 63 | if (browserKey == null) 64 | { 65 | browserKey = 66 | Microsoft.Win32.Registry.CurrentUser.OpenSubKey( 67 | urlAssociation, false); 68 | } 69 | var path = CleanifyBrowserPath(browserKey.GetValue(null) as string); 70 | browserKey.Close(); 71 | return path; 72 | } 73 | else 74 | { 75 | // user defined browser choice was found 76 | string progId = (userChoiceKey.GetValue("ProgId").ToString()); 77 | userChoiceKey.Close(); 78 | 79 | // now look up the path of the executable 80 | string concreteBrowserKey = browserPathKey.Replace("$BROWSER$", progId); 81 | var kp = Registry.ClassesRoot.OpenSubKey(concreteBrowserKey, false); 82 | browserPath = CleanifyBrowserPath(kp.GetValue(null) as string); 83 | kp.Close(); 84 | return browserPath; 85 | } 86 | } 87 | catch (Exception) 88 | { 89 | return ""; 90 | } 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Virtual_EDW/FodyWeavers.xml: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /Virtual_EDW/Form_About.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | using System.Windows.Forms; 3 | 4 | namespace Virtual_Data_Warehouse 5 | { 6 | public partial class FormAbout : FormBase 7 | { 8 | //private readonly FormMain _myParent; 9 | 10 | public FormAbout() 11 | { 12 | AutoScaleMode = AutoScaleMode.Dpi; 13 | InitializeComponent(); 14 | FormBorderStyle = FormBorderStyle.FixedSingle; 15 | } 16 | 17 | private void linkLabel1_LinkClicked_1(object sender, LinkLabelLinkClickedEventArgs e) 18 | { 19 | // Specify that the link was visited. 20 | linkLabelTeam.LinkVisited = true; 21 | // Navigate to a URL. 22 | Process.Start("https://github.com/RoelantVos/virtual-data-warehouse"); 23 | } 24 | 25 | private void linkLabelTeam_LinkClicked(object sender, LinkLabelLinkClickedEventArgs e) 26 | { 27 | // Specify that the link was visited. 28 | linkLabel1.LinkVisited = true; 29 | // Navigate to a URL. 30 | Process.Start("http://www.roelantvos.com"); 31 | } 32 | 33 | private void buttonClose_Click(object sender, System.EventArgs e) 34 | { 35 | Close(); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /Virtual_EDW/Form_Base.Designer.cs: -------------------------------------------------------------------------------- 1 | namespace Virtual_Data_Warehouse 2 | { 3 | partial class FormBase 4 | { 5 | /// 6 | /// Required designer variable. 7 | /// 8 | private System.ComponentModel.IContainer components = null; 9 | 10 | /// 11 | /// Clean up any resources being used. 12 | /// 13 | /// true if managed resources should be disposed; otherwise, false. 14 | protected override void Dispose(bool disposing) 15 | { 16 | if (disposing && (components != null)) 17 | { 18 | components.Dispose(); 19 | } 20 | base.Dispose(disposing); 21 | } 22 | 23 | #region Windows Form Designer generated code 24 | 25 | /// 26 | /// Required method for Designer support - do not modify 27 | /// the contents of this method with the code editor. 28 | /// 29 | private void InitializeComponent() 30 | { 31 | SuspendLayout(); 32 | // 33 | // FormBase 34 | // 35 | AutoScaleDimensions = new System.Drawing.SizeF(96F, 96F); 36 | AutoScaleMode = System.Windows.Forms.AutoScaleMode.Dpi; 37 | ClientSize = new System.Drawing.Size(284, 261); 38 | Font = new System.Drawing.Font("Segoe UI", 8.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point); 39 | Name = "FormBase"; 40 | Text = "FormBase"; 41 | ResumeLayout(false); 42 | } 43 | 44 | #endregion 45 | } 46 | } -------------------------------------------------------------------------------- /Virtual_EDW/GlobalSuppressions.cs: -------------------------------------------------------------------------------- 1 | // This file is used by Code Analysis to maintain SuppressMessage 2 | // attributes that are applied to this project. 3 | // Project-level suppressions either have no target or are given 4 | // a specific target and scoped to a namespace, type, member, etc. 5 | 6 | using System.Diagnostics.CodeAnalysis; 7 | 8 | [assembly: SuppressMessage("Interoperability", "CA1416:Validate platform compatibility", Justification = "", Scope = "member", Target = "~M:Virtual_Data_Warehouse.FormMain.#ctor")] 9 | [assembly: SuppressMessage("Interoperability", "CA1416:Validate platform compatibility", Justification = "", Scope = "member", Target = "~M:Virtual_Data_Warehouse.FormMain.SaveTemplateCollectionFileToolStripMenuItem_Click(System.Object,System.EventArgs)")] 10 | [assembly: SuppressMessage("Interoperability", "CA1416:Validate platform compatibility", Justification = "", Scope = "member", Target = "~M:Virtual_Data_Warehouse.FormMain.PictureBoxUpdateTemplatePath_Click(System.Object,System.EventArgs)")] 11 | [assembly: SuppressMessage("Interoperability", "CA1416:Validate platform compatibility", Justification = "", Scope = "member", Target = "~M:Virtual_Data_Warehouse.FormMain.openTemplateCollectionFileToolStripMenuItem_Click(System.Object,System.EventArgs)")] 12 | [assembly: SuppressMessage("Interoperability", "CA1416:Validate platform compatibility", Justification = "", Scope = "member", Target = "~M:Virtual_Data_Warehouse.CustomTabPage.GenerateFromTemplate")] 13 | [assembly: SuppressMessage("Interoperability", "CA1416:Validate platform compatibility", Justification = "", Scope = "member", Target = "~M:Virtual_Data_Warehouse.CustomTabPage.Generate(System.String,Virtual_Data_Warehouse.VdwDataObjectMappingList,HandlebarsDotNet.HandlebarsTemplate{System.Object,System.Object},System.String)")] 14 | -------------------------------------------------------------------------------- /Virtual_EDW/InterfaceObjectModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using DataWarehouseAutomation; 3 | 4 | namespace Virtual_Data_Warehouse 5 | { 6 | /// 7 | /// The parent object containing the list of source-to-target mappings. This is the highest level and contains the list of mappings (as individual objects 8 | /// but also the parameters inherited from TEAM and VDW. 9 | /// 10 | class VdwDataObjectMappingList : DataObjectMappingList 11 | { 12 | // TEAM and VDW specific details 13 | public MetadataConfiguration metadataConfiguration { get; set; } 14 | 15 | public string metadataFileName { get; set; } 16 | 17 | public GenerationSpecificMetadata generationSpecificMetadata { get; set; } 18 | } 19 | 20 | 21 | /// 22 | /// Specific metadata related for generation purposes, but which is relevant to use in templates. 23 | /// 24 | public class GenerationSpecificMetadata 25 | { 26 | public DataObject selectedDataObject { get; set; } 27 | public DateTime generationDateTime { get; set; } 28 | } 29 | 30 | /// 31 | /// The parameters that have been inherited from TEAM or are set in VDW, passed as properties of the metadata - and can be used in the templates. 32 | /// 33 | class MetadataConfiguration 34 | { 35 | public string vdwSchemaName { get; set; } = FormBase.VdwConfigurationSettings.VdwSchema; 36 | 37 | // Attributes 38 | public string changeDataCaptureAttribute { get; set; } 39 | public string recordSourceAttribute { get; set; } 40 | public string loadDateTimeAttribute { get; set; } 41 | public string expiryDateTimeAttribute { get; set; } 42 | public string eventDateTimeAttribute { get; set; } 43 | public string recordChecksumAttribute { get; set; } 44 | public string etlProcessAttribute { get; set; } 45 | public string sourceRowIdAttribute { get; set; } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /Virtual_EDW/Libraries/Microsoft.Data.SqlClient.SNI.x64.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Libraries/Microsoft.Data.SqlClient.SNI.x64.dll -------------------------------------------------------------------------------- /Virtual_EDW/Libraries/Microsoft.Data.SqlClient.SNI.x86.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Libraries/Microsoft.Data.SqlClient.SNI.x86.dll -------------------------------------------------------------------------------- /Virtual_EDW/Libraries/Microsoft.Data.SqlClient.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Libraries/Microsoft.Data.SqlClient.dll -------------------------------------------------------------------------------- /Virtual_EDW/Libraries/Snowflake.Data.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Libraries/Snowflake.Data.dll -------------------------------------------------------------------------------- /Virtual_EDW/Libraries/TEAM_Library.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Libraries/TEAM_Library.dll -------------------------------------------------------------------------------- /Virtual_EDW/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Windows.Forms; 4 | using Virtual_Data_Warehouse; 5 | 6 | namespace Virtual_Data_Warehouse 7 | { 8 | static class Program 9 | { 10 | /// 11 | /// The main entry point for the application. 12 | /// 13 | [STAThread] 14 | static void Main() 15 | { 16 | Application.EnableVisualStyles(); 17 | Application.SetHighDpiMode(HighDpiMode.PerMonitor); 18 | Application.SetCompatibleTextRenderingDefault(false); 19 | Application.Run(new FormMain()); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Virtual_EDW/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("Virtual Data Warehouse")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("Virtual Data Warehouse")] 13 | [assembly: AssemblyCopyright("Open Source")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("0d140741-01fc-431d-8f81-e52168dd870a")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.6.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /Virtual_EDW/Properties/app.manifest: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 54 | -------------------------------------------------------------------------------- /Virtual_EDW/Resources/CogIcon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/CogIcon.ico -------------------------------------------------------------------------------- /Virtual_EDW/Resources/CubeIcon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/CubeIcon.ico -------------------------------------------------------------------------------- /Virtual_EDW/Resources/DocumentationIcon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/DocumentationIcon.ico -------------------------------------------------------------------------------- /Virtual_EDW/Resources/ETLIcon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/ETLIcon.ico -------------------------------------------------------------------------------- /Virtual_EDW/Resources/ExitApplication.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/ExitApplication.ico -------------------------------------------------------------------------------- /Virtual_EDW/Resources/HelpIconSmall.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/HelpIconSmall.ico -------------------------------------------------------------------------------- /Virtual_EDW/Resources/LinkIcon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/LinkIcon.ico -------------------------------------------------------------------------------- /Virtual_EDW/Resources/OpenDirectoryIcon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/OpenDirectoryIcon.ico -------------------------------------------------------------------------------- /Virtual_EDW/Resources/OpenFileIcon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/OpenFileIcon.ico -------------------------------------------------------------------------------- /Virtual_EDW/Resources/Ravos-logo-small.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/Ravos-logo-small.ico -------------------------------------------------------------------------------- /Virtual_EDW/Resources/Ravos-logo-small_transparant.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/Ravos-logo-small_transparant.ico -------------------------------------------------------------------------------- /Virtual_EDW/Resources/SaveFile.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/SaveFile.ico -------------------------------------------------------------------------------- /Virtual_EDW/Resources/Time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/Time.png -------------------------------------------------------------------------------- /Virtual_EDW/Resources/database_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/database_icon.png -------------------------------------------------------------------------------- /Virtual_EDW/Resources/ghost-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/ghost-icon.png -------------------------------------------------------------------------------- /Virtual_EDW/Resources/log-file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/log-file.png -------------------------------------------------------------------------------- /Virtual_EDW/Resources/transparent-green-checkmark-hi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/data-solution-automation-engine/virtual-data-warehouse/61bed91af2c234ea70c589d34c401d07633475d6/Virtual_EDW/Resources/transparent-green-checkmark-hi.png -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPatternHubInsertIntofromSTG.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Hub Insert Into statement for {{targetDataObject.name}} 4 | -- 5 | IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'INT.SP_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}') AND type in (N'P')) 6 | DROP PROCEDURE [INT].[SP_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}]; 7 | GO 8 | 9 | CREATE PROCEDURE [INT].[SP_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}] @OMD_INSERT_MODULE_INSTANCE_ID int 10 | AS 11 | 12 | -- 13 | -- Hub Insert Into statement for [{{targetDataObject.name}}] 14 | -- 15 | 16 | INSERT INTO [INT].[{{targetDataObject.name}}] 17 | ( 18 | [{{targetDataObject.name}}_SK], 19 | OMD_INSERT_DATETIME, 20 | OMD_INSERT_MODULE_INSTANCE_ID, 21 | OMD_RECORD_SOURCE_ID, 22 | {{#each businessKeyDefinitions}}{{#if @first}}{{#each businessKeyComponentMappings}} 23 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}}{{/if}}{{/each}} 24 | ) 25 | SELECT 26 | CONVERT(CHAR,HASHBYTES('SHA1',UPPER({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(ISNULL({{stringcompare sourceDataItems.0.isHardCodedValue 'True'}} [{{/stringcompare }}{{sourceDataItems.0.name}}{{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}] {{/stringcompare }},'NA')){{#unless @last}}+'|'+{{/unless}}{{/each}} {{/each}} )),2) as [{{targetDataObject.name}}_SK], 27 | min(OMD_INSERT_DATETIME), 28 | @OMD_INSERT_MODULE_INSTANCE_ID, 29 | min(OMD_RECORD_SOURCE_ID), 30 | {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 31 | ISNULL({{stringcompare sourceDataItems.0.isHardCodedValue 'True'}} [{{/stringcompare }}{{sourceDataItems.0.name}}{{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}] {{/stringcompare }},'NA') AS [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}}{{/each}} 32 | 33 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] stg 34 | WHERE CONVERT(CHAR,HASHBYTES('SHA1',UPPER({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(ISNULL({{stringcompare sourceDataItems.0.isHardCodedValue 'True'}} [{{/stringcompare }}{{sourceDataItems.0.name}}{{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}] {{/stringcompare }},'NA')){{#unless @last}}+'|'+{{/unless}}{{/each}} {{/each}} )),2) NOT IN 35 | ( 36 | SELECT [{{targetDataObject.name}}_SK] 37 | FROM [INT].[{{targetDataObject.name}}] INT 38 | ) 39 | GROUP BY {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}}{{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}{{#unless @first}},{{/unless}} [{{sourceDataItems.0.name}}]{{/stringcompare}}{{/each}}{{/each}} ; 40 | 41 | SELECT @@ROWCOUNT AS ROWS_INSERTED 42 | GO 43 | {{/each}} 44 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPatternLinkInsertIntofromSTG.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Link Insert Into statement for {{targetDataObject.name}} 4 | -- 5 | 6 | IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'INT.SP_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}') AND type in (N'P')) 7 | DROP PROCEDURE [INT].[SP_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}]; 8 | GO 9 | 10 | CREATE PROCEDURE [INT].[SP_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}] @OMD_INSERT_MODULE_INSTANCE_ID int 11 | AS 12 | 13 | -- 14 | -- LNK Insert Into statement for [{{targetDataObject.name}}] 15 | -- Includes the GROUP BY clause 16 | 17 | INSERT INTO [INT].[{{targetDataObject.name}}] 18 | ( {{#each businessKeyDefinitions}} {{#if @first}} [{{surrogateKey}}], {{/if}}{{/each}} 19 | {{#each businessKeyDefinitions}}{{#unless @first}} 20 | [{{surrogateKey}}],{{/unless}} {{/each}} 21 | OMD_INSERT_DATETIME, 22 | OMD_RECORD_SOURCE_ID, 23 | OMD_INSERT_MODULE_INSTANCE_ID 24 | ) 25 | 26 | SELECT 27 | CONVERT(CHAR,HASHBYTES('SHA1',UPPER( {{#each businessKeyDefinitions}} {{#unless @first}}{{!-- Create the Surrogate Key using the combined Business Keys --}} 28 | {{#each dataItemMappings}} RTRIM(ISNULL([{{targetDataItem.name}}], 'NA')) + '|'+{{/each}} 29 | {{#each businessKeyComponentMappings}} RTRIM(ISNULL([{{targetDataItem.name}}{{@../index }}], 'NA')) {{#unless @last}} + '|' +{{/unless}}{{/each}}{{#unless @last}} +'|'+{{/unless}}{{/unless}}{{/each}} 30 | {{#each dataItemMappings}} + RTRIM(ISNULL([{{targetDataItem.name}}], 'NA')) + '|'+{{/each}}{{#each businessKeyDefinitions}} {{#if @first}})),2) AS [{{surrogateKey}}],{{/if}}{{/each}} 31 | {{#each businessKeyDefinitions}}{{#unless @first}} {{!-- List out the separate (Hub) business keys, by skipping the first generic relationship key. --}} 32 | {{#unless businessKeyClassification}} 33 | CONVERT(CHAR,HASHBYTES('SHA1',UPPER( {{#each businessKeyComponentMappings}} RTRIM(ISNULL([{{targetDataItem.name}}{{@../index }}],'NA')) {{#unless @last}} +'|'+{{/unless}}{{/each}})),2) AS [{{surrogateKey}}], 34 | {{/unless}}{{#if businessKeyClassification}}{{#each businessKeyComponentMappings}}[{{targetDataItem.name}}{{@../index }}]{{/each}} AS [{{surrogateKey}}], 35 | {{/if}} {{/unless}} {{/each}}{{#each dataItemMappings}} 36 | [{{targetDataItem.name}}],{{/each}} 37 | OMD_INSERT_DATETIME, 38 | OMD_RECORD_SOURCE_ID, 39 | @OMD_INSERT_MODULE_INSTANCE_ID 40 | FROM 41 | ( 42 | SELECT {{#each businessKeyDefinitions}} {{#unless @first}} {{#each businessKeyComponentMappings}} 43 | CAST({{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}[{{/stringcompare}}{{sourceDataItems.0.name}}{{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}]{{/stringcompare}} AS NVARCHAR(100)) AS [{{targetDataItem.name}}{{@../index }}],{{/each}}{{/unless}}{{/each}}{{#each dataItemMappings}} 44 | [{{sourceDataItems.0.name}}] AS [{{targetDataItem.name}}],{{/each}} 45 | min(OMD_INSERT_DATETIME) AS OMD_INSERT_DATETIME, 46 | min(OMD_RECORD_SOURCE_ID) AS OMD_RECORD_SOURCE_ID 47 | FROM STG.[{{sourceDataObjects.0.name}}] 48 | GROUP BY {{#each businessKeyDefinitions}}{{#if @first}}{{#each businessKeyComponentMappings}}{{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}{{#unless @first}},{{/unless}} [{{sourceDataItems.0.name}}]{{/stringcompare}}{{/each}}{{/if}}{{/each}} 49 | ) LINK_selection 50 | WHERE CONVERT(CHAR,HASHBYTES('SHA1',UPPER( {{#each businessKeyDefinitions}} {{#unless @first}}{{!-- Create the Surrogate Key using the combined Business Keys --}} 51 | {{#each dataItemMappings}} RTRIM(ISNULL([{{targetDataItem.name}}], 'NA')) + '|'+{{/each}} 52 | {{#each businessKeyComponentMappings}} RTRIM(ISNULL([{{targetDataItem.name}}{{@../index }}], 'NA')) {{#unless @last}} + '|' +{{/unless}}{{/each}}{{#unless @last}} +'|'+{{/unless}}{{/unless}}{{/each}} 53 | {{#each dataItemMappings}} + RTRIM(ISNULL([{{targetDataItem.name}}], 'NA')) + '|'+{{/each}}{{#each businessKeyDefinitions}} {{#if @first}})),2) {{/if}}{{/each}} NOT IN 54 | ( 55 | SELECT [{{targetDataObject.name}}_SK] 56 | FROM [INT].[{{targetDataObject.name}}] 57 | ); 58 | SELECT @@ROWCOUNT AS ROWS_INSERTED 59 | GO 60 | {{/each}} 61 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPatternLinkSatelliteReinitialisation.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | {{#if enabled}} 3 | -- 4 | -- Satellite Reinitialisation logic for [{{#each targetDataObject.dataObjectConnection.extensions}}{{#stringcompare key schema}}{{value}}{{/stringcompare}}{{/each}}].[{{targetDataObject.name}}] 5 | -- 6 | 7 | -- Truncate the source for the Satellite, the Staging Area table. 8 | TRUNCATE TABLE [{{#each sourceDataObjects.0.dataObjectConnection.extensions}}{{#stringcompare key schema}}{{value}}{{/stringcompare}}{{/each}}].[{{sourceDataObjects.0.name}}] 9 | 10 | -- Copy the data from the Staging Area to the PSA. 11 | INSERT INTO STG.{{sourceDataObjects.0.name}} 12 | ( 13 | [OMD_INSERT_MODULE_INSTANCE_ID] 14 | ,[OMD_FILE_LOCATION] 15 | ,[OMD_EVENT_DATETIME] 16 | ,[OMD_INSERT_DATETIME] 17 | ,[OMD_HASH_DIFF] 18 | ,[OMD_CURRENT_RECORD_INDICATOR] 19 | ,[OMD_DELETED_RECORD_INDICATOR] 20 | ,[OMD_EXPIRY_DATETIME] 21 | ,[OMD_RECORD_SOURCE_ID] 22 | ,[OMD_SOURCE_ROW_ID] 23 | {{#each dataItemMappings}} 24 | ,[{{sourceDataItems.0.name}}] 25 | {{/each}} 26 | ) 27 | SELECT 28 | [OMD_INSERT_MODULE_INSTANCE_ID] 29 | ,[OMD_FILE_LOCATION] 30 | ,[OMD_EVENT_DATETIME] 31 | ,[OMD_EFFECTIVE_DATETIME] 32 | ,[OMD_HASH_DIFF] 33 | ,[OMD_CURRENT_RECORD_INDICATOR] 34 | ,[OMD_DELETED_RECORD_INDICATOR] 35 | ,[OMD_EXPIRY_DATETIME] 36 | ,[OMD_RECORD_SOURCE_ID] 37 | ,[OMD_SOURCE_ROW_ID] 38 | {{#each dataItemMappings}} 39 | ,[{{sourceDataItems.0.name}}] 40 | {{/each}} 41 | FROM PSA.{{StringReplace sourceDataObjects.0.name "STG" "PSA"}} 42 | 43 | -- Truncate the Satellite table. 44 | TRUNCATE TABLE [{{#each targetDataObject.dataObjectConnection.extensions}}{{#stringcompare key schema}}{{value}}{{/stringcompare}}{{/each}}].[{{targetDataObject.name}}]; 45 | 46 | -- Rerun the Satellite loading procedure. 47 | EXEC [INT].[SP_{{targetDataObject.name}}] 0; 48 | 49 | {{/if}}{{/each}} 50 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPatternPersistentStagingAreaInsertInto.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Persistent Staging Area Insert Into statement for [{{targetDataObject.name}}] 4 | -- 5 | 6 | IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'PSA.SP_{{targetDataObject.name}}') AND type in (N'P')) 7 | DROP PROCEDURE [PSA].[SP_{{targetDataObject.name}}] 8 | GO 9 | 10 | CREATE PROCEDURE [PSA].[SP_{{targetDataObject.name}}] @OMD_INSERT_MODULE_INSTANCE_ID int 11 | AS 12 | INSERT INTO [PSA].[{{targetDataObject.name}}] 13 | ( 14 | [{{businessKeyDefinitions.0.surrogateKey}}], 15 | {{#each dataItemMappings}} 16 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}} 17 | {{/each}} 18 | ,OMD_INSERT_MODULE_INSTANCE_ID 19 | ,OMD_FILE_LOCATION 20 | ,OMD_EVENT_DATETIME 21 | ,OMD_EFFECTIVE_DATETIME 22 | ,OMD_HASH_DIFF 23 | ,OMD_CURRENT_RECORD_INDICATOR 24 | ,OMD_DELETED_RECORD_INDICATOR 25 | ,OMD_EXPIRY_DATETIME 26 | ,OMD_RECORD_SOURCE_ID 27 | ,OMD_SOURCE_ROW_ID 28 | ) 29 | SELECT 30 | CONVERT(CHAR,HASHBYTES('SHA1',UPPER({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(ISNULL([{{sourceDataItems.0.name}}],'NA')){{#unless @last}}+'|'+{{/unless}}{{/each}} {{/each}} )),2), 31 | {{#each dataItemMappings}} 32 | [{{sourceDataItems.0.name}}] as [{{targetDataItem.name}}]{{#unless @last}},{{/unless}} 33 | {{/each}} 34 | ,@OMD_INSERT_MODULE_INSTANCE_ID 35 | ,OMD_FILE_LOCATION 36 | ,OMD_EVENT_DATETIME 37 | ,OMD_INSERT_DATETIME 38 | ,OMD_HASH_DIFF 39 | ,OMD_CURRENT_RECORD_INDICATOR 40 | ,OMD_DELETED_RECORD_INDICATOR 41 | ,OMD_EXPIRY_DATETIME 42 | ,OMD_RECORD_SOURCE_ID 43 | ,OMD_SOURCE_ROW_ID 44 | 45 | FROM [STG].[{{sourceDataObjects.0.name}}] ; 46 | SELECT @@ROWCOUNT AS ROWS_INSERTED 47 | GO 48 | {{/each}} 49 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPatternSatelliteReinitialisation.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | {{#if enabled}} 3 | -- 4 | -- Satellite Reinitialisation logic for [{{#each targetDataObject.dataObjectConnection.extensions}}{{#stringcompare key schema}}{{value}}{{/stringcompare}}{{/each}}].[{{targetDataObject.name}}] 5 | -- 6 | 7 | -- Truncate the source for the Satellite, the Staging Area table. 8 | TRUNCATE TABLE [{{#each sourceDataObjects.0.dataObjectConnection.extensions}}{{#stringcompare key schema}}{{value}}{{/stringcompare}}{{/each}}].[{{sourceDataObjects.0.name}}] 9 | 10 | -- Copy the data from the Staging Area to the PSA. 11 | INSERT INTO STG.{{sourceDataObjects.0.name}} 12 | ( 13 | [OMD_INSERT_MODULE_INSTANCE_ID] 14 | ,[OMD_FILE_LOCATION] 15 | ,[OMD_EVENT_DATETIME] 16 | ,[OMD_INSERT_DATETIME] 17 | ,[OMD_HASH_DIFF] 18 | ,[OMD_CURRENT_RECORD_INDICATOR] 19 | ,[OMD_DELETED_RECORD_INDICATOR] 20 | ,[OMD_EXPIRY_DATETIME] 21 | ,[OMD_RECORD_SOURCE_ID] 22 | ,[OMD_SOURCE_ROW_ID] 23 | {{#each dataItemMappings}} 24 | ,[{{sourceDataItems.0.name}}] 25 | {{/each}} 26 | ) 27 | SELECT 28 | [OMD_INSERT_MODULE_INSTANCE_ID] 29 | ,[OMD_FILE_LOCATION] 30 | ,[OMD_EVENT_DATETIME] 31 | ,[OMD_EFFECTIVE_DATETIME] 32 | ,[OMD_HASH_DIFF] 33 | ,[OMD_CURRENT_RECORD_INDICATOR] 34 | ,[OMD_DELETED_RECORD_INDICATOR] 35 | ,[OMD_EXPIRY_DATETIME] 36 | ,[OMD_RECORD_SOURCE_ID] 37 | ,[OMD_SOURCE_ROW_ID] 38 | {{#each dataItemMappings}} 39 | ,[{{sourceDataItems.0.name}}] 40 | {{/each}} 41 | FROM PSA.{{StringReplace sourceDataObjects.0.name "STG" "PSA"}} 42 | 43 | -- Truncate the Satellite table. 44 | TRUNCATE TABLE [{{#each targetDataObject.dataObjectConnection.extensions}}{{#stringcompare key schema}}{{value}}{{/stringcompare}}{{/each}}].[{{targetDataObject.name}}]; 45 | 46 | -- Rerun the Satellite loading procedure. 47 | EXEC [SP_{{targetDataObject.name}}] 0; 48 | {{/if}}{{/each}} 49 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_HUB_SK_has_SAT_record.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 11; 3 | DECLARE @HUB nvarchar(1000); 4 | {{#each dataObjectMappings}} 5 | 6 | -- 7 | -- HUB has a SAT record UT check for {{targetDataObject.name}} and {{businessKeyDefinitions.0.surrogateKey}} 8 | -- 9 | 10 | SET @HUB = REPLACE('{{businessKeyDefinitions.0.surrogateKey}}','_SK',''); 11 | 12 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 13 | SELECT DISTINCT 14 | @UT_ID,@HUB,'{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 15 | 16 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 17 | WHEN 'STG' 18 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 19 | ELSE 'DV' 20 | END, 21 | 22 | 'WITH recordcount 23 | AS (SELECT Count(*) AS record_count 24 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].['+@HUB+']), 25 | testcount 26 | AS ( 27 | SELECT Count(*) AS test_count 28 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].['+@HUB+'] 29 | WHERE [{{businessKeyDefinitions.0.surrogateKey}}] 30 | NOT IN 31 | ( 32 | SELECT [{{businessKeyDefinitions.0.surrogateKey}}] 33 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 34 | ) 35 | ) 36 | SELECT record_count, 37 | test_count, 38 | CASE 39 | WHEN test_count = 0 THEN ''Pass'' 40 | ELSE ''Fail'' 41 | END AS Status, 42 | Getdate() AS Test_Execution_DateTime, 43 | CURRENT_USER AS Test_Executor 44 | FROM recordcount, 45 | testcount; ' 46 | , 47 | 48 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}] AS HUB_SK, [OMD_INSERT_MODULE_INSTANCE_ID] 49 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].['+@HUB+'] 50 | WHERE [{{businessKeyDefinitions.0.surrogateKey}}] 51 | NOT IN 52 | ( 53 | SELECT [{{businessKeyDefinitions.0.surrogateKey}}] 54 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 55 | )' 56 | 57 | , 'Y' 58 | , 'This test is optional as HUB may contain keys which do not exist in particular SAT (if more than one SAT available for HUB), however all combined SATs must have all keys from the HUB' 59 | 60 | FROM OMD.UT a 61 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 62 | and TARGET_TABLE = @HUB 63 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 64 | WHERE exist.UT_DETAIL_ID is null 65 | 66 | 67 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 68 | where [name] = 'UT_DETAIL_ID' 69 | 70 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 71 | SELECT DISTINCT 72 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 73 | m.MODULE_ID as MODULE_ID 74 | FROM OMD.UT ut 75 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = @HUB 76 | and ud.SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 77 | and ud.UT_ID = @UT_ID 78 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_' + @HUB +'_{{sourceDataObjects.0.name}}' 79 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 80 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 81 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 82 | 83 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_HUB_uniqueKeyCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 10; 3 | 4 | {{#each dataObjectMappings}} 5 | {{#if @first}} 6 | -- 7 | -- HUB UT unique key check for {{targetDataObject.name}} 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','ALL',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 'DV', 13 | 14 | 'WITH recordcount 15 | AS (SELECT Count(*) AS Record_Count 16 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 17 | testcount 18 | AS (SELECT Count(DISTINCT [{{businessKeyDefinitions.0.surrogateKey}}]) AS Test_Count 19 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]) 20 | SELECT record_count, 21 | test_count, 22 | CASE 23 | WHEN record_count = test_count THEN ''Pass'' 24 | ELSE ''Fail'' 25 | END AS Status, 26 | Getdate() AS Test_Execution_DateTime, 27 | CURRENT_USER AS Test_Executor 28 | FROM recordcount, 29 | testcount; ' 30 | , 31 | 32 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}] as HUB_SK, [OMD_INSERT_MODULE_INSTANCE_ID] 33 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 34 | GROUP BY [{{businessKeyDefinitions.0.surrogateKey}}], [OMD_INSERT_MODULE_INSTANCE_ID] 35 | HAVING Count(*) > 1 ' 36 | 37 | , 'N' 38 | , NULL 39 | 40 | FROM OMD.UT a 41 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 42 | and TARGET_TABLE = '{{targetDataObject.name}}' 43 | and SOURCE_TABLE = 'ALL' 44 | and REALM = 'DV' 45 | WHERE exist.UT_DETAIL_ID is null 46 | 47 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 48 | where [name] = 'UT_DETAIL_ID' 49 | 50 | {{/if}} 51 | 52 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 53 | SELECT DISTINCT 54 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 55 | m.MODULE_ID as MODULE_ID 56 | FROM OMD.UT ut 57 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 58 | and ud.UT_ID = @UT_ID 59 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 60 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 61 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 62 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 63 | 64 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_LNK_HUB_SK_existsCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 12; 3 | DECLARE @HUB nvarchar(1000); 4 | 5 | {{#each dataObjectMappings}} 6 | 7 | {{#each businessKeyDefinitions}} 8 | {{#unless @first}} 9 | -- 10 | -- LNK UT SK Generation check for {{../targetDataObject.name}} - {{surrogateKey}} pair 11 | -- 12 | 13 | SET @HUB = REPLACE('{{surrogateKey}}','_SK',''); 14 | 15 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 16 | SELECT DISTINCT 17 | @UT_ID,'{{../targetDataObject.name}}',@HUB,NULL, NULL, '{{../targetDataObject.dataObjectConnection.extensions.1.value}}','{{../targetDataObject.dataObjectConnection.extensions.1.value}}', 18 | 19 | CASE '{{../sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 20 | WHEN 'STG' 21 | THEN substring('{{../sourceDataObjects.0.name}}',5,charindex('_','{{../sourceDataObjects.0.name}}',5)-5) 22 | ELSE 'DV' 23 | END, 24 | 25 | 'WITH recordcount 26 | AS (SELECT Count(*) AS record_count 27 | FROM [{{../targetDataObject.dataObjectConnection.extensions.1.value}}].[{{../targetDataObject.name}}]), 28 | testcount 29 | AS ( 30 | SELECT Count(1) AS test_count 31 | FROM [{{../targetDataObject.dataObjectConnection.extensions.1.value}}].[{{../targetDataObject.name}}] [{{../targetDataObject.dataObjectConnection.extensions.1.value}}] 32 | WHERE {{surrogateKey}} 33 | NOT IN 34 | ( 35 | SELECT [{{surrogateKey}}] 36 | FROM [{{../targetDataObject.dataObjectConnection.extensions.1.value}}].[' + @HUB + '] 37 | ) 38 | ) 39 | SELECT record_count, 40 | test_count, 41 | CASE 42 | WHEN test_count = 0 THEN ''Pass'' 43 | ELSE ''Fail'' 44 | END AS Status, 45 | Getdate() AS Test_Execution_DateTime, 46 | CURRENT_USER AS Test_Executor 47 | FROM recordcount, 48 | testcount; ' 49 | , 50 | 51 | 'SELECT [{{surrogateKey}}], [OMD_INSERT_MODULE_INSTANCE_ID] 52 | FROM [{{../targetDataObject.dataObjectConnection.extensions.1.value}}].[{{../targetDataObject.name}}] [{{../targetDataObject.dataObjectConnection.extensions.1.value}}] 53 | WHERE {{surrogateKey}} 54 | NOT IN 55 | ( 56 | SELECT [{{surrogateKey}}] 57 | FROM [{{../targetDataObject.dataObjectConnection.extensions.1.value}}].[' + @HUB + '] 58 | )' 59 | 60 | , 'N' 61 | , NULL 62 | 63 | FROM OMD.UT a 64 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 65 | and TARGET_TABLE = '{{../targetDataObject.name}}' 66 | and SOURCE_TABLE = @HUB 67 | WHERE exist.UT_DETAIL_ID is null 68 | 69 | {{/unless}} 70 | {{/each}} 71 | 72 | 73 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 74 | SELECT DISTINCT 75 | ud.UT_DETAIL_ID, 76 | m.MODULE_ID as MODULE_ID 77 | FROM OMD.UT ut 78 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 79 | and ud.UT_ID = @UT_ID 80 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 81 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 82 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 83 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 84 | 85 | {{/each}} 86 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_LNK_nullKeyCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 21; 3 | 4 | {{#each dataObjectMappings}} 5 | {{#if @first}} 6 | -- 7 | -- LNK UT unique key check for {{targetDataObject.name}} 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','ALL',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 'DV', 13 | 14 | 'WITH recordcount 15 | AS (SELECT Count(*) AS record_count 16 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 17 | testcount 18 | AS ( 19 | SELECT Count(1) AS test_count 20 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 21 | WHERE CONVERT(CHAR,HASHBYTES(''SHA1'',UPPER( TRIM(ISNULL(NULL,''NA'')) )),2) = [{{businessKeyDefinitions.0.surrogateKey}}] 22 | ) 23 | 24 | SELECT record_count, 25 | test_count, 26 | CASE 27 | WHEN test_count = 0 THEN ''Pass'' 28 | ELSE ''Fail'' 29 | END AS Status, 30 | Getdate() AS Test_Execution_DateTime, 31 | CURRENT_USER AS Test_Executor 32 | FROM recordcount, 33 | testcount; ' 34 | , 35 | 36 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}], [{{businessKeyDefinitions.0.businessKeyComponentMappings.0.targetDataItem.name}}], [OMD_INSERT_MODULE_INSTANCE_ID] 37 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 38 | WHERE CONVERT(CHAR,HASHBYTES(''SHA1'',UPPER( TRIM(ISNULL(NULL,''NA'')) )),2) = [{{businessKeyDefinitions.0.surrogateKey}}]' 39 | 40 | , 'N' 41 | , NULL 42 | 43 | FROM OMD.UT a 44 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 45 | and TARGET_TABLE = '{{targetDataObject.name}}' 46 | and SOURCE_TABLE = 'ALL' 47 | and REALM = 'DV' 48 | WHERE exist.UT_DETAIL_ID is null 49 | 50 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 51 | where [name] = 'UT_DETAIL_ID' 52 | 53 | {{/if}} 54 | 55 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 56 | SELECT DISTINCT 57 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 58 | m.MODULE_ID as MODULE_ID 59 | FROM OMD.UT ut 60 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 61 | and ud.UT_ID = @UT_ID 62 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 63 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 64 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 65 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 66 | 67 | {{/each}} 68 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_LNK_uniqueKeyCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 13; 3 | 4 | {{#each dataObjectMappings}} 5 | {{#if @first}} 6 | -- 7 | -- LNK UT unique key check for {{targetDataObject.name}} 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','ALL',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 'DV', 13 | 14 | 'WITH recordcount 15 | AS (SELECT Count(*) AS Record_Count 16 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 17 | testcount 18 | AS (SELECT Count(DISTINCT [{{businessKeyDefinitions.0.surrogateKey}}]) AS Test_Count 19 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]) 20 | SELECT record_count, 21 | test_count, 22 | CASE 23 | WHEN record_count = test_count THEN ''Pass'' 24 | ELSE ''Fail'' 25 | END AS Status, 26 | Getdate() AS Test_Execution_DateTime, 27 | CURRENT_USER AS Test_Executor 28 | FROM recordcount, 29 | testcount; ' 30 | , 31 | 32 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}] as LNK_SK, [OMD_INSERT_MODULE_INSTANCE_ID] 33 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 34 | GROUP BY [{{businessKeyDefinitions.0.surrogateKey}}], [OMD_INSERT_MODULE_INSTANCE_ID] 35 | HAVING Count(*) > 1 ' 36 | 37 | , 'N' 38 | , NULL 39 | 40 | FROM OMD.UT a 41 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 42 | and TARGET_TABLE = '{{targetDataObject.name}}' 43 | and SOURCE_TABLE = 'ALL' 44 | and REALM = 'DV' 45 | WHERE exist.UT_DETAIL_ID is null 46 | 47 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 48 | where [name] = 'UT_DETAIL_ID' 49 | 50 | {{/if}} 51 | 52 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 53 | SELECT DISTINCT 54 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 55 | m.MODULE_ID as MODULE_ID 56 | FROM OMD.UT ut 57 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 58 | and ud.UT_ID = @UT_ID 59 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 60 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 61 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 62 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 63 | 64 | {{/each}} 65 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_LSAT_SK_has_LNK_record.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 27; 3 | {{#each dataObjectMappings}} 4 | -- 5 | -- HUB has a LSAT record UT check for {{targetDataObject.name}} and {{businessKeyDefinitions.0.surrogateKey}} 6 | -- 7 | 8 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 9 | SELECT DISTINCT 10 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 11 | 12 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 13 | WHEN 'STG' 14 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 15 | ELSE 'DV' 16 | END, 17 | 18 | 'WITH recordcount 19 | AS (SELECT Count(*) AS record_count 20 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 21 | testcount 22 | AS ( 23 | SELECT Count(*) AS test_count 24 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 25 | WHERE [{{businessKeyDefinitions.0.surrogateKey}}] 26 | NOT IN 27 | ( 28 | SELECT [{{businessKeyDefinitions.0.surrogateKey}}] 29 | FROM [{{relatedDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{relatedDataObjects.0.name}}] 30 | ) 31 | ) 32 | SELECT record_count, 33 | test_count, 34 | CASE 35 | WHEN test_count = 0 THEN ''Pass'' 36 | ELSE ''Fail'' 37 | END AS Status, 38 | Getdate() AS Test_Execution_DateTime, 39 | CURRENT_USER AS Test_Executor 40 | FROM recordcount, 41 | testcount; ' 42 | , 43 | 44 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}] AS LSAT_SK, [OMD_INSERT_MODULE_INSTANCE_ID] 45 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 46 | WHERE [{{businessKeyDefinitions.0.surrogateKey}}] 47 | NOT IN 48 | ( 49 | SELECT [{{businessKeyDefinitions.0.surrogateKey}}] 50 | FROM [{{relatedDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{relatedDataObjects.0.name}}] 51 | )' 52 | 53 | , 'N' 54 | , NULL 55 | 56 | FROM OMD.UT a 57 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 58 | and TARGET_TABLE = '{{targetDataObject.name}}' 59 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 60 | WHERE exist.UT_DETAIL_ID is null 61 | 62 | 63 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 64 | where [name] = 'UT_DETAIL_ID' 65 | 66 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 67 | SELECT DISTINCT 68 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 69 | m.MODULE_ID as MODULE_ID 70 | FROM OMD.UT ut 71 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 72 | and ud.SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 73 | and ud.UT_ID = @UT_ID 74 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 75 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 76 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 77 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 78 | 79 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_LSAT_currentRecordCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 14; 3 | 4 | {{#each dataObjectMappings}} 5 | -- 6 | -- LSAT UT current record check for {{targetDataObject.name}} 7 | -- 8 | 9 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 10 | SELECT DISTINCT 11 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 12 | 13 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 14 | WHEN 'STG' 15 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 16 | ELSE 'DV' 17 | END, 18 | 19 | 'WITH recordcount 20 | AS (SELECT Count(*) AS Record_Count 21 | FROM {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}.{{Name}}), 22 | 23 | testcount 24 | AS (SELECT Count(test_count) AS Test_Count 25 | FROM (SELECT Count(*) AS Test_Count 26 | FROM {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}.{{Name}} {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}} 27 | WHERE omd_current_record_indicator = ''Y'' 28 | GROUP BY {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}.{{businessKeyDefinitions.0.surrogateKey}} 29 | HAVING Count(*) > 1) AS a) 30 | 31 | SELECT record_count, 32 | test_count, 33 | CASE 34 | WHEN test_count = 0 THEN ''Pass'' 35 | ELSE ''Fail'' 36 | END AS STATUS, 37 | Getdate() AS Test_Execution_DateTime, 38 | CURRENT_USER AS Test_Executor 39 | FROM recordcount 40 | LEFT OUTER JOIN testcount 41 | ON 1 = 1; ' 42 | , 43 | 44 | 'SELECT {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}.{{businessKeyDefinitions.0.surrogateKey}} as LNK_SK, [OMD_INSERT_MODULE_INSTANCE_ID] 45 | FROM {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}.{{Name}} {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}} 46 | WHERE omd_current_record_indicator = ''Y'' 47 | GROUP BY {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}.{{businessKeyDefinitions.0.surrogateKey}}, [OMD_INSERT_MODULE_INSTANCE_ID] 48 | HAVING Count(*) > 1' 49 | 50 | , 'N' 51 | , NULL 52 | 53 | FROM OMD.UT a 54 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 55 | and TARGET_TABLE = '{{targetDataObject.name}}' 56 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 57 | and REALM = substring('{{targetDataObject.name}}',5,charindex('_','{{targetDataObject.name}}',5)-5) 58 | WHERE exist.UT_DETAIL_ID is null 59 | 60 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 61 | where [name] = 'UT_DETAIL_ID' 62 | 63 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 64 | SELECT DISTINCT 65 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 66 | m.MODULE_ID as MODULE_ID 67 | FROM OMD.UT ut 68 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' and ud.UT_ID = @UT_ID 69 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 70 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 71 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 72 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 73 | 74 | {{/each}} 75 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_LSAT_duplicateRecordCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 15; 3 | 4 | {{#each dataObjectMappings}} 5 | -- 6 | -- LSAT UT current record check for {{targetDataObject.name}} 7 | -- 8 | 9 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 10 | SELECT DISTINCT 11 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 12 | 13 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 14 | WHEN 'STG' 15 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 16 | ELSE 'DV' 17 | END, 18 | 19 | 'WITH recordcount 20 | AS (SELECT Count(*) AS Record_Count 21 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}]), 22 | 23 | testcount 24 | AS (SELECT Sum(test_count) AS Test_Count 25 | FROM (SELECT Count(DISTINCT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}]) AS test_count 26 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}} 27 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}] 28 | , OMD_HASH_DIFF 29 | HAVING Count(*) > 1) AS a) 30 | 31 | SELECT record_count, 32 | test_count, 33 | CASE 34 | WHEN test_count IS NULL THEN ''Pass'' 35 | ELSE ''Fail'' 36 | END AS STATUS, 37 | Getdate() AS Test_Execution_DateTime, 38 | CURRENT_USER AS Test_Executor 39 | FROM recordcount 40 | LEFT OUTER JOIN testcount 41 | ON 1 = 1; ' 42 | , 43 | 44 | 'SELECT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}] , [OMD_INSERT_MODULE_INSTANCE_ID] 45 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}} 46 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [OMD_INSERT_MODULE_INSTANCE_ID] 47 | , OMD_HASH_DIFF 48 | HAVING Count(*) > 1' 49 | 50 | , 'N' 51 | , NULL 52 | 53 | FROM OMD.UT a 54 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 55 | and TARGET_TABLE = '{{targetDataObject.name}}' 56 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 57 | and REALM = substring('{{targetDataObject.name}}',5,charindex('_','{{targetDataObject.name}}',5)-5) 58 | WHERE exist.UT_DETAIL_ID is null 59 | 60 | 61 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 62 | where [name] = 'UT_DETAIL_ID' 63 | 64 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 65 | SELECT DISTINCT 66 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 67 | m.MODULE_ID as MODULE_ID 68 | FROM OMD.UT ut 69 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' and ud.UT_ID = @UT_ID 70 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 71 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 72 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 73 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 74 | 75 | {{/each}} 76 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_PSA_STG_to_PSA_Check.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 26; 3 | 4 | {{#each dataObjectMappings}} 5 | 6 | -- 7 | -- PSA UT Staging to PSA check for {{targetDataObject.name}} - {{sourceDataObjects.0.name}} pair 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 13 | 14 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 15 | WHEN 'STG' 16 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 17 | ELSE 'DV' 18 | END, 19 | 20 | 'WITH recordcount 21 | AS (SELECT Count(*) AS record_count 22 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 23 | 24 | testcount 25 | AS (select count(1) as test_count from ( 26 | SELECT {{#each dataItemMappings}} 27 | [{{sourceDataItems.0.name}}] AS [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 28 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] 29 | EXCEPT 30 | SELECT {{#each dataItemMappings}} 31 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 32 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 33 | ) t 34 | ) 35 | 36 | SELECT record_count, 37 | test_count, 38 | CASE 39 | WHEN test_count = 0 THEN ''Pass'' 40 | ELSE ''Fail'' 41 | END AS STATUS, 42 | Getdate() AS Test_Execution_DateTime, 43 | CURRENT_USER AS Test_Executor 44 | FROM recordcount 45 | , testcount 46 | ' 47 | , 48 | 49 | 'select {{#each dataItemMappings}} 50 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 51 | from ( 52 | SELECT {{#each dataItemMappings}} 53 | [{{sourceDataItems.0.name}}] AS [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 54 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] 55 | EXCEPT 56 | SELECT {{#each dataItemMappings}} 57 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 58 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 59 | ) t' 60 | 61 | , 'N' 62 | , NULL 63 | 64 | FROM OMD.UT a 65 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 66 | and TARGET_TABLE = '{{targetDataObject.name}}' 67 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 68 | and REALM = substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 69 | WHERE exist.UT_DETAIL_ID is null 70 | 71 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 72 | where [name] = 'UT_DETAIL_ID' 73 | 74 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 75 | SELECT DISTINCT 76 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 77 | m.MODULE_ID as MODULE_ID 78 | FROM OMD.UT ut 79 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 80 | and ud.SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 81 | and ud.UT_ID = @UT_ID 82 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_150_{{targetDataObject.name}}' 83 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 84 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 85 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 86 | 87 | {{/each}} 88 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_PSA_currentRecordCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 4; 3 | 4 | {{#each dataObjectMappings}} 5 | -- 6 | -- PSA UT current record check for {{targetDataObject.name}} 7 | -- 8 | 9 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 10 | SELECT DISTINCT 11 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', substring('{{targetDataObject.name}}',5,charindex('_','{{targetDataObject.name}}',5)-5), 12 | 13 | 'WITH recordcount 14 | AS (SELECT Count(*) AS Record_Count 15 | FROM {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}.{{Name}}), 16 | 17 | testcount 18 | AS (SELECT Count(test_count) AS Test_Count 19 | FROM (SELECT Count(*) AS Test_Count 20 | FROM {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}.{{Name}} {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}} 21 | WHERE omd_current_record_indicator = ''Y'' 22 | GROUP BY {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}.{{businessKeyDefinitions.0.surrogateKey}} 23 | HAVING Count(*) > 1) AS a) 24 | 25 | SELECT record_count, 26 | test_count, 27 | CASE 28 | WHEN test_count = 0 THEN ''Pass'' 29 | ELSE ''Fail'' 30 | END AS STATUS, 31 | Getdate() AS Test_Execution_DateTime, 32 | CURRENT_USER AS Test_Executor 33 | FROM recordcount 34 | LEFT OUTER JOIN testcount 35 | ON 1 = 1; ' 36 | , 37 | 38 | 'SELECT {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}.{{businessKeyDefinitions.0.surrogateKey}}, [OMD_INSERT_MODULE_INSTANCE_ID] 39 | FROM {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}.{{Name}} {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}} 40 | WHERE omd_current_record_indicator = ''Y'' 41 | GROUP BY {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}.{{businessKeyDefinitions.0.surrogateKey}}, [OMD_INSERT_MODULE_INSTANCE_ID] 42 | HAVING Count(*) > 1' 43 | 44 | , 'N' 45 | , NULL 46 | 47 | FROM OMD.UT a 48 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 49 | and TARGET_TABLE = '{{targetDataObject.name}}' 50 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 51 | and REALM = substring('{{targetDataObject.name}}',5,charindex('_','{{targetDataObject.name}}',5)-5) 52 | WHERE exist.UT_DETAIL_ID is null 53 | 54 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 55 | where [name] = 'UT_DETAIL_ID' 56 | 57 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 58 | SELECT DISTINCT 59 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 60 | m.MODULE_ID as MODULE_ID 61 | FROM OMD.UT ut 62 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' and ud.UT_ID = @UT_ID 63 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_150_{{targetDataObject.name}}' 64 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 65 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 66 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 67 | 68 | {{/each}} 69 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_PSA_duplicateRecordCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 5; 3 | 4 | {{#each dataObjectMappings}} 5 | -- 6 | -- PSA UT current record check for {{targetDataObject.name}} 7 | -- 8 | 9 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 10 | SELECT DISTINCT 11 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', substring('{{targetDataObject.name}}',5,charindex('_','{{targetDataObject.name}}',5)-5), 12 | 13 | 'WITH recordcount 14 | AS (SELECT Count(*) AS Record_Count 15 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}]), 16 | 17 | testcount 18 | AS (SELECT Sum(test_count) AS Test_Count 19 | FROM (SELECT Count(DISTINCT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}]) AS test_count 20 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}} 21 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}] 22 | , OMD_HASH_DIFF 23 | HAVING Count(*) > 1) AS a) 24 | 25 | SELECT record_count, 26 | test_count, 27 | CASE 28 | WHEN test_count IS NULL THEN ''Pass'' 29 | ELSE ''Fail'' 30 | END AS STATUS, 31 | Getdate() AS Test_Execution_DateTime, 32 | CURRENT_USER AS Test_Executor 33 | FROM recordcount 34 | LEFT OUTER JOIN testcount 35 | ON 1 = 1; ' 36 | , 37 | 38 | 'SELECT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [OMD_INSERT_MODULE_INSTANCE_ID] 39 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}} 40 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [OMD_INSERT_MODULE_INSTANCE_ID] 41 | , OMD_HASH_DIFF 42 | HAVING Count(*) > 1' 43 | 44 | , 'N' 45 | , NULL 46 | 47 | FROM OMD.UT a 48 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 49 | and TARGET_TABLE = '{{targetDataObject.name}}' 50 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 51 | and REALM = substring('{{targetDataObject.name}}',5,charindex('_','{{targetDataObject.name}}',5)-5) 52 | WHERE exist.UT_DETAIL_ID is null 53 | 54 | 55 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 56 | where [name] = 'UT_DETAIL_ID' 57 | 58 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 59 | SELECT DISTINCT 60 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 61 | m.MODULE_ID as MODULE_ID 62 | FROM OMD.UT ut 63 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' and ud.UT_ID = @UT_ID 64 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_150_{{targetDataObject.name}}' 65 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 66 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 67 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 68 | 69 | {{/each}} 70 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_SAT_SK_has_HUB_record.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 9; 3 | DECLARE @HUB nvarchar(1000); 4 | {{#each dataObjectMappings}} 5 | 6 | -- 7 | -- HUB has a SAT record UT check for {{targetDataObject.name}} and {{businessKeyDefinitions.0.surrogateKey}} 8 | -- 9 | 10 | SET @HUB = REPLACE('{{businessKeyDefinitions.0.surrogateKey}}','_SK',''); 11 | 12 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 13 | SELECT DISTINCT 14 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 15 | 16 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 17 | WHEN 'STG' 18 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 19 | ELSE 'DV' 20 | END, 21 | 22 | 'WITH recordcount 23 | AS (SELECT Count(*) AS record_count 24 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 25 | testcount 26 | AS ( 27 | SELECT Count(*) AS test_count 28 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 29 | WHERE [{{businessKeyDefinitions.0.surrogateKey}}] 30 | NOT IN 31 | ( 32 | SELECT [{{businessKeyDefinitions.0.surrogateKey}}] 33 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].['+@HUB+'] 34 | ) 35 | ) 36 | SELECT record_count, 37 | test_count, 38 | CASE 39 | WHEN test_count = 0 THEN ''Pass'' 40 | ELSE ''Fail'' 41 | END AS Status, 42 | Getdate() AS Test_Execution_DateTime, 43 | CURRENT_USER AS Test_Executor 44 | FROM recordcount, 45 | testcount; ' 46 | , 47 | 48 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}] AS SAT_SK, [OMD_INSERT_MODULE_INSTANCE_ID] 49 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 50 | WHERE [{{businessKeyDefinitions.0.surrogateKey}}] 51 | NOT IN 52 | ( 53 | SELECT [{{businessKeyDefinitions.0.surrogateKey}}] 54 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].['+@HUB+'] 55 | )' 56 | 57 | , 'N' 58 | , NULL 59 | 60 | FROM OMD.UT a 61 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 62 | and TARGET_TABLE = '{{targetDataObject.name}}' 63 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 64 | WHERE exist.UT_DETAIL_ID is null 65 | 66 | 67 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 68 | where [name] = 'UT_DETAIL_ID' 69 | 70 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 71 | SELECT DISTINCT 72 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 73 | m.MODULE_ID as MODULE_ID 74 | FROM OMD.UT ut 75 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 76 | and ud.SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 77 | and ud.UT_ID = @UT_ID 78 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 79 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 80 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 81 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 82 | 83 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_SAT_currentRecordCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 7; 3 | 4 | {{#each dataObjectMappings}} 5 | -- 6 | -- SAT UT current record check for {{targetDataObject.name}} 7 | -- 8 | 9 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 10 | SELECT DISTINCT 11 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 12 | 13 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 14 | WHEN 'STG' 15 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 16 | ELSE 'DV' 17 | END, 18 | 19 | 'WITH recordcount 20 | AS (SELECT Count(*) AS Record_Count 21 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}]), 22 | 23 | testcount 24 | AS (SELECT Count(test_count) AS Test_Count 25 | FROM (SELECT Count(*) AS Test_Count 26 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}] 27 | WHERE omd_current_record_indicator = ''Y'' 28 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}] 29 | HAVING Count(*) > 1) AS a) 30 | 31 | SELECT record_count, 32 | test_count, 33 | CASE 34 | WHEN test_count = 0 THEN ''Pass'' 35 | ELSE ''Fail'' 36 | END AS STATUS, 37 | Getdate() AS Test_Execution_DateTime, 38 | CURRENT_USER AS Test_Executor 39 | FROM recordcount 40 | LEFT OUTER JOIN testcount 41 | ON 1 = 1; ' 42 | , 43 | 44 | 'SELECT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [OMD_INSERT_MODULE_INSTANCE_ID] 45 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}] 46 | WHERE omd_current_record_indicator = ''Y'' 47 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [OMD_INSERT_MODULE_INSTANCE_ID] 48 | HAVING Count(*) > 1' 49 | 50 | , 'N' 51 | , NULL 52 | 53 | FROM OMD.UT a 54 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 55 | and TARGET_TABLE = '{{targetDataObject.name}}' 56 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 57 | and REALM = substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 58 | WHERE exist.UT_DETAIL_ID is null 59 | 60 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 61 | where [name] = 'UT_DETAIL_ID' 62 | 63 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 64 | SELECT DISTINCT 65 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 66 | m.MODULE_ID as MODULE_ID 67 | FROM OMD.UT ut 68 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' and ud.UT_ID = @UT_ID 69 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 70 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 71 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 72 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 73 | 74 | {{/each}} 75 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_SAT_duplicateRecordCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 8; 3 | 4 | {{#each dataObjectMappings}} 5 | -- 6 | -- SAT UT current record check for {{targetDataObject.name}} 7 | -- 8 | 9 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 10 | SELECT DISTINCT 11 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 12 | 13 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 14 | WHEN 'STG' 15 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 16 | ELSE 'DV' 17 | END, 18 | 19 | 'WITH recordcount 20 | AS (SELECT Count(*) AS Record_Count 21 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}]), 22 | 23 | testcount 24 | AS (SELECT Sum(test_count) AS Test_Count 25 | FROM (SELECT Count(DISTINCT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}]) AS test_count 26 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}] 27 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}] 28 | , OMD_HASH_DIFF 29 | HAVING Count(*) > 1) AS a) 30 | 31 | SELECT record_count, 32 | test_count, 33 | CASE 34 | WHEN test_count IS NULL THEN ''Pass'' 35 | ELSE ''Fail'' 36 | END AS STATUS, 37 | Getdate() AS Test_Execution_DateTime, 38 | CURRENT_USER AS Test_Executor 39 | FROM recordcount 40 | LEFT OUTER JOIN testcount 41 | ON 1 = 1; ' 42 | , 43 | 44 | 'SELECT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [OMD_INSERT_MODULE_INSTANCE_ID] 45 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}] 46 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [OMD_INSERT_MODULE_INSTANCE_ID] 47 | , OMD_HASH_DIFF 48 | HAVING Count(*) > 1' 49 | 50 | , 'N' 51 | , NULL 52 | 53 | FROM OMD.UT a 54 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 55 | and TARGET_TABLE = '{{targetDataObject.name}}' 56 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 57 | and REALM = substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 58 | WHERE exist.UT_DETAIL_ID is null 59 | 60 | 61 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 62 | where [name] = 'UT_DETAIL_ID' 63 | 64 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 65 | SELECT DISTINCT 66 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 67 | m.MODULE_ID as MODULE_ID 68 | FROM OMD.UT ut 69 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' and ud.UT_ID = @UT_ID 70 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 71 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 72 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 73 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 74 | 75 | {{/each}} 76 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/loadPattern_UT_SAT_nullKeyCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 24; 3 | 4 | {{#each dataObjectMappings}} 5 | 6 | -- 7 | -- SAT UT unique key check for {{targetDataObject.name}} - {{sourceDataObjects.0.name}} pair 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 13 | 14 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 15 | WHEN 'STG' 16 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 17 | ELSE 'DV' 18 | END, 19 | 20 | 'WITH recordcount 21 | AS (SELECT Count(*) AS record_count 22 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 23 | testcount 24 | AS ( 25 | SELECT Count(1) AS test_count 26 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 27 | WHERE CONVERT(CHAR,HASHBYTES(''SHA1'',UPPER( TRIM(ISNULL(NULL,''NA'')) )),2) = [{{businessKeyDefinitions.0.surrogateKey}}] 28 | ) 29 | 30 | SELECT record_count, 31 | test_count, 32 | CASE 33 | WHEN test_count = 0 THEN ''Pass'' 34 | ELSE ''Fail'' 35 | END AS Status, 36 | Getdate() AS Test_Execution_DateTime, 37 | CURRENT_USER AS Test_Executor 38 | FROM recordcount, 39 | testcount; ' 40 | , 41 | 42 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}], [OMD_INSERT_MODULE_INSTANCE_ID] 43 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 44 | WHERE CONVERT(CHAR,HASHBYTES(''SHA1'',UPPER( TRIM(ISNULL(NULL,''NA'')) )),2) = [{{businessKeyDefinitions.0.surrogateKey}}]' 45 | 46 | , 'N' 47 | , NULL 48 | 49 | FROM OMD.UT a 50 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 51 | and TARGET_TABLE = '{{targetDataObject.name}}' 52 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 53 | and REALM = 'DV' 54 | WHERE exist.UT_DETAIL_ID is null 55 | 56 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 57 | where [name] = 'UT_DETAIL_ID' 58 | 59 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 60 | SELECT DISTINCT 61 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 62 | m.MODULE_ID as MODULE_ID 63 | FROM OMD.UT ut 64 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 65 | and ud.SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 66 | and ud.UT_ID = @UT_ID 67 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 68 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 69 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 70 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 71 | 72 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateDataLakeSilver01.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | update omd.PARAMETER set PARAMETER_VALUE_CODE='MERGE INTO {{sourceDataObjects.0.name}} AS target 3 | 4 | USING ( 5 | 6 | SELECT 7 | sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL(updates.{{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) as mergeKey 8 | , updates.* 9 | 10 | FROM DL_{{targetDataObject.name}} AS updates 11 | 12 | UNION ALL 13 | 14 | SELECT NULL as mergeKey, updates.* 15 | 16 | FROM DL_{{targetDataObject.name}} AS updates 17 | 18 | JOIN {{sourceDataObjects.0.name}} AS target 19 | ON sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL(updates.{{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) = sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL(target.{{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) 20 | 21 | WHERE target.OMD_CURRENT_RECORD_INDICATOR = ''Y'' AND updates.OMD_HASH_DIFF <> target.OMD_HASH_DIFF 22 | 23 | ) staged_updates 24 | 25 | ON 26 | sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL(target.{{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) = mergeKey 27 | 28 | 29 | WHEN NOT MATCHED THEN INSERT 30 | 31 | ( 32 | {{#each dataItemMappings}} 33 | {{sourceDataItems.0.name}}{{#unless @last}},{{/unless}} 34 | {{/each}} 35 | ,OMD_RECORD_SOURCE_ID,OMD_INSERT_MODULE_INSTANCE_ID,OMD_FILE_LOCATION,OMD_EVENT_DATETIME,OMD_INSERT_DATETIME,OMD_HASH_DIFF,OMD_UPDATE_MODULE_INSTANCE_ID,OMD_CURRENT_RECORD_INDICATOR,OMD_DELETED_RECORD_INDICATOR,OMD_EXPIRY_DATETIME) 36 | 37 | VALUES 38 | 39 | ( 40 | {{#each dataItemMappings}} 41 | staged_updates.{{sourceDataItems.0.name}}{{#unless @last}},{{/unless}} 42 | {{/each}} 43 | ,staged_updates.OMD_RECORD_SOURCE_ID,staged_updates.OMD_INSERT_MODULE_INSTANCE_ID,staged_updates.OMD_FILE_LOCATION,staged_updates.OMD_EVENT_DATETIME,staged_updates.OMD_INSERT_DATETIME,staged_updates.OMD_HASH_DIFF,''NULL'',''Y'',''N'',''9999-12-31'');' where parameter_key_code='DL_SILVER_INSERT_{{sourceDataObjects.0.name}}'; 44 | {{/each}} 45 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateDataLakeSilver01CDC.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | update omd.PARAMETER set PARAMETER_VALUE_CODE='MERGE INTO {{sourceDataObjects.0.name}} AS target 3 | 4 | USING ( 5 | 6 | SELECT 7 | sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL(updates.{{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) as mergeKey 8 | , updates.* 9 | 10 | FROM DL_{{targetDataObject.name}} AS updates 11 | WHERE updates.IsDeleted =''False'' 12 | 13 | UNION ALL 14 | 15 | SELECT NULL as mergeKey, updates.* 16 | 17 | FROM DL_{{targetDataObject.name}} AS updates 18 | 19 | JOIN {{sourceDataObjects.0.name}} AS target 20 | ON sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL(updates.{{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) = sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL(target.{{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) 21 | 22 | WHERE target.OMD_CURRENT_RECORD_INDICATOR = ''Y'' AND updates.OMD_HASH_DIFF <> target.OMD_HASH_DIFF AND target.IsDeleted = ''False'' 23 | ) staged_updates 24 | 25 | ON 26 | sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL(target.{{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) = mergeKey 27 | 28 | 29 | WHEN NOT MATCHED THEN INSERT 30 | 31 | ( 32 | {{#each dataItemMappings}} 33 | {{sourceDataItems.0.name}}{{#unless @last}},{{/unless}} 34 | {{/each}} 35 | ,OMD_RECORD_SOURCE_ID,OMD_INSERT_MODULE_INSTANCE_ID,OMD_FILE_LOCATION,OMD_EVENT_DATETIME,OMD_INSERT_DATETIME,OMD_HASH_DIFF,OMD_UPDATE_MODULE_INSTANCE_ID,OMD_CURRENT_RECORD_INDICATOR,OMD_DELETED_RECORD_INDICATOR,OMD_EXPIRY_DATETIME) 36 | 37 | VALUES 38 | 39 | ( 40 | {{#each dataItemMappings}} 41 | staged_updates.{{sourceDataItems.0.name}}{{#unless @last}},{{/unless}} 42 | {{/each}} 43 | ,staged_updates.OMD_RECORD_SOURCE_ID,staged_updates.OMD_INSERT_MODULE_INSTANCE_ID,staged_updates.OMD_FILE_LOCATION,staged_updates.OMD_EVENT_DATETIME,staged_updates.OMD_INSERT_DATETIME,staged_updates.OMD_HASH_DIFF,''NULL'',''Y'',''N'',''9999-12-31'');' where parameter_key_code='DL_SILVER_INSERT_{{sourceDataObjects.0.name}}'; 44 | {{/each}} 45 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateDataLakeSilver02.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | update omd.PARAMETER set PARAMETER_VALUE_CODE='with cte as ( 3 | 4 | select * from {{sourceDataObjects.0.name}} as source 5 | 6 | where source.OMD_CURRENT_RECORD_INDICATOR =''Y'' and source.OMD_DELETED_RECORD_INDICATOR=''N'' 7 | 8 | and sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL(source.{{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) not in 9 | 10 | (select sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL(target.{{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) from DL_{{targetDataObject.name}} AS target) 11 | 12 | ) 13 | 14 | INSERT INTO {{sourceDataObjects.0.name}} ( 15 | {{#each dataItemMappings}} 16 | {{sourceDataItems.0.name}}{{#unless @last}},{{/unless}} 17 | {{/each}} 18 | , 19 | OMD_RECORD_SOURCE_ID, 20 | OMD_FILE_LOCATION, 21 | OMD_EVENT_DATETIME, 22 | OMD_HASH_DIFF, 23 | OMD_CURRENT_RECORD_INDICATOR, 24 | OMD_DELETED_RECORD_INDICATOR, 25 | OMD_EXPIRY_DATETIME, 26 | OMD_UPDATE_MODULE_INSTANCE_ID, 27 | OMD_INSERT_DATETIME, 28 | OMD_INSERT_MODULE_INSTANCE_ID 29 | ) 30 | 31 | SELECT {{#each dataItemMappings}} 32 | delete_insert.{{sourceDataItems.0.name}}{{#unless @last}},{{/unless}} 33 | {{/each}} 34 | ,delete_insert.OMD_RECORD_SOURCE_ID, 35 | delete_insert.OMD_FILE_LOCATION, 36 | CURRENT_TIMESTAMP() AS OMD_EVENT_DATETIME, 37 | delete_insert.OMD_HASH_DIFF, 38 | ''Y'', 39 | ''Y'', 40 | ''9999-12-31'' 41 | , ''NULL'' 42 | ,' where parameter_key_code='DL_SILVER_DELETE_{{sourceDataObjects.0.name}}'; 43 | 44 | {{/each}} 45 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateDataLakeSilver02CDC.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | update omd.PARAMETER set PARAMETER_VALUE_CODE='with cte as ( 3 | 4 | select * from {{sourceDataObjects.0.name}} as source 5 | 6 | where source.IsDeleted =''True'' 7 | 8 | ) 9 | 10 | INSERT INTO {{sourceDataObjects.0.name}} ( 11 | {{#each dataItemMappings}} 12 | {{sourceDataItems.0.name}}{{#unless @last}},{{/unless}} 13 | {{/each}} 14 | , 15 | OMD_RECORD_SOURCE_ID, 16 | OMD_FILE_LOCATION, 17 | OMD_EVENT_DATETIME, 18 | OMD_HASH_DIFF, 19 | OMD_CURRENT_RECORD_INDICATOR, 20 | OMD_DELETED_RECORD_INDICATOR, 21 | OMD_EXPIRY_DATETIME, 22 | OMD_UPDATE_MODULE_INSTANCE_ID, 23 | OMD_INSERT_DATETIME, 24 | OMD_INSERT_MODULE_INSTANCE_ID 25 | ) 26 | 27 | SELECT {{#each dataItemMappings}} 28 | delete_insert.{{sourceDataItems.0.name}}{{#unless @last}},{{/unless}} 29 | {{/each}} 30 | ,delete_insert.OMD_RECORD_SOURCE_ID, 31 | delete_insert.OMD_FILE_LOCATION, 32 | CURRENT_TIMESTAMP() AS OMD_EVENT_DATETIME, 33 | delete_insert.OMD_HASH_DIFF, 34 | ''Y'', 35 | ''Y'', 36 | ''9999-12-31'' 37 | , ''NULL'' 38 | ,' where parameter_key_code='DL_SILVER_DELETE_{{sourceDataObjects.0.name}}'; 39 | 40 | {{/each}} 41 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateDataLakeSilver03.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | update omd.PARAMETER set PARAMETER_VALUE_CODE='MERGE INTO {{sourceDataObjects.0.name}} as source 3 | 4 | USING ( 5 | 6 | SELECT sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL({{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) as DL_KEY 7 | ,OMD_INSERT_DATETIME 8 | ,OMD_SOURCE_ROW_ID 9 | ,lag(OMD_INSERT_DATETIME, 1, ''9999-12-31'') OVER (PARTITION BY sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL({{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) ORDER BY OMD_INSERT_DATETIME DESC ,OMD_SOURCE_ROW_ID DESC) AS OMD_EXPIRY_DATETIME 10 | ,lag(''N'', 1, ''Y'') OVER (PARTITION BY sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL({{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) ORDER BY OMD_INSERT_DATETIME DESC ,OMD_SOURCE_ROW_ID DESC) AS OMD_CURRENT_RECORD_INDICATOR 11 | 12 | FROM {{sourceDataObjects.0.name}} 13 | WHERE 14 | sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL({{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) IN 15 | ( 16 | SELECT sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL({{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) 17 | FROM {{sourceDataObjects.0.name}} 18 | WHERE OMD_CURRENT_RECORD_INDICATOR = ''Y'' 19 | GROUP BY sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL({{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING))) 20 | HAVING COUNT(*) > 1 21 | ) 22 | 23 | ) staged_updates 24 | 25 | ON sha1(UPPER(CAST({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(IFNULL({{sourceDataItems.0.name}},''NA'')){{#unless @last}}||''|''||{{/unless}}{{/each}} {{/each}} AS STRING)))= DL_KEY 26 | AND source.OMD_INSERT_DATETIME = staged_updates.OMD_INSERT_DATETIME 27 | AND source.OMD_SOURCE_ROW_ID = staged_updates.OMD_SOURCE_ROW_ID 28 | AND staged_updates.OMD_EXPIRY_DATETIME != ''9999-12-31'' 29 | AND source.OMD_CURRENT_RECORD_INDICATOR = ''Y'' 30 | WHEN MATCHED 31 | THEN 32 | UPDATE 33 | SET source.OMD_CURRENT_RECORD_INDICATOR = ''N'' 34 | ,source.OMD_EXPIRY_DATETIME = staged_updates.OMD_EXPIRY_DATETIME 35 | ,source.OMD_UPDATE_MODULE_INSTANCE_ID =' where parameter_key_code='DL_SILVER_UPDATE_{{sourceDataObjects.0.name}}'; 36 | {{/each}} 37 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateDataLakeSilverToStagingSelectCDC.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | update omd.PARAMETER set PARAMETER_VALUE_CODE='SELECT {{#each dataItemMappings}} 3 | target.{{sourceDataItems.0.name}}{{#unless @last}},{{/unless}} 4 | {{/each}} 5 | ,target.OMD_RECORD_SOURCE_ID,target.OMD_INSERT_MODULE_INSTANCE_ID,target.OMD_FILE_LOCATION,target.OMD_EVENT_DATETIME,target.OMD_INSERT_DATETIME,target.OMD_HASH_DIFF,target.OMD_CURRENT_RECORD_INDICATOR,target.OMD_DELETED_RECORD_INDICATOR,target.OMD_EXPIRY_DATETIME,target.OMD_SOURCE_ROW_ID FROM VW_{{sourceDataObjects.0.name}} AS target 6 | where target.OMD_INSERT_MODULE_INSTANCE_ID > ' where parameter_key_code='SILVER_TO_STAGING_SELECT_{{targetDataObject.name}}'; 7 | {{/each}} 8 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateHubDeltaSelect.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings }} 2 | -- 3 | -- Working on mapping to {{targetDataObject.name}} from source table {{sourceDataObjects.0.name}} 4 | -- Generated at {{now}} 5 | -- 6 | SELECT 7 | HASHBYTES('MD5', {{#each businessKeyDefinitions}} {{#if @first}} 8 | {{#each businessKeyComponentMappings}} 9 | ISNULL(RTRIM(CONVERT(NVARCHAR(100), [{{targetDataItem.name}}])), 'N/A') + '#~!'{{#unless @last}} +{{/unless}} 10 | {{/each}} 11 | ) AS [{{surrogateKey}}],{{/if}}{{/each}}{{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 12 | [{{targetDataItem.name}}],{{/each}}{{/each}} 13 | [{{../conventions.loadDateTimeAttribute}}], 14 | [{{../conventions.etlProcessAttribute}}], 15 | [{{../conventions.recordSourceAttribute}}] 16 | FROM 17 | ( 18 | SELECT 19 | sub.*, 20 | ROW_NUMBER() OVER (PARTITION BY {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 21 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}}{{/each}} 22 | ORDER BY [{{../conventions.loadDateTimeAttribute}}] 23 | ) AS LDTS_ORDER 24 | FROM 25 | ( 26 | SELECT {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 27 | {{sourceDataItems.0.name}} AS [{{targetDataItem.name}}],{{/each}}{{/each}} 28 | stg.[{{../conventions.loadDateTimeAttribute}}], 29 | -1 AS [{{../conventions.etlProcessAttribute}}], 30 | stg.[{{../conventions.recordSourceAttribute}}] 31 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] stg 32 | WHERE NOT EXISTS 33 | ( 34 | SELECT 1 35 | FROM [{{targetDataObject.name}}] hub 36 | WHERE {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 37 | {{sourceDataItems.0.name}} = hub.{{targetDataItem.name}}{{#unless @last}} AND{{/unless}}{{/each}}{{/each}} 38 | ) 39 | ) sub 40 | ) supersub 41 | WHERE LDTS_ORDER=1 42 | {{/each}} 43 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateHubGenerateTables.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}}{{#if @first}} 2 | -- 3 | -- Table creation statement for {{targetDataObject.name}} 4 | -- 5 | 6 | IF OBJECT_ID('[{{targetDataObject.name}}]', 'U') IS NOT NULL 7 | DROP TABLE [{{targetDataObject.name}}]; 8 | 9 | CREATE TABLE [{{targetDataObject.name}}] 10 | ( 11 | [{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] binary(16) NOT NULL, 12 | [{{../conventions.etlProcessAttribute}}] [int] NOT NULL, 13 | [{{../conventions.loadDateTimeAttribute}}] [datetime2](7) NOT NULL, 14 | [{{../conventions.recordSourceAttribute}}] [nvarchar](100) NOT NULL, 15 | {{#each businessKeyDefinitions.0.businessKeyComponentMappings}} 16 | [{{targetDataItem.name}}] [nvarchar](100) NOT NULL{{#unless @last}},{{/unless}} 17 | {{/each}} 18 | ){{/if}} 19 | {{/each}} 20 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateHubInsertInto.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}}{{#if @first}} 2 | -- 3 | -- Hub Insert Into statement for {{targetDataObject.name}} 4 | -- 5 | -- This template inserts the data delta that has been detected by comparing the Hub View to the existing Hub data into the target Hub table. 6 | -- 7 | -- Generated from template '200 Data Vault Insert Into'. 8 | -- 9 | 10 | INSERT INTO [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] 11 | ( 12 | [{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}],{{#each businessKeyDefinitions}}{{#if @first}}{{#each businessKeyComponentMappings}} 13 | [{{targetDataItem.name}}],{{/each}}{{/if}}{{/each}} 14 | [{{../conventions.etlProcessAttribute}}], 15 | [{{../conventions.loadDateTimeAttribute}}] 16 | --,[{{../conventions.recordSourceAttribute}}] 17 | ) 18 | SELECT 19 | hub_view.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}],{{#each businessKeyDefinitions}}{{#if @first}}{{#each businessKeyComponentMappings}} 20 | hub_view.[{{targetDataItem.name}}],{{/each}}{{/if}}{{/each}} 21 | -1 AS [{{../conventions.etlProcessAttribute}}], 22 | hub_view.[{{../conventions.loadDateTimeAttribute}}] 23 | --,hub_view.[{{../conventions.recordSourceAttribute}}] 24 | FROM [{{../conventions.vdwSchemaName}}].[{{targetDataObject.name}}] hub_view 25 | LEFT OUTER JOIN [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] hub_table 26 | ON hub_view.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] = hub_table.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] 27 | WHERE hub_table.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] IS NULL 28 | {{/if}} 29 | 30 | {{/each}} 31 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateHubInsertIntoProcedure.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Hub Insert Into statement for {{targetDataObject.name}} 4 | -- 5 | IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'INT.SP_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}') AND type in (N'P')) 6 | DROP PROCEDURE [INT].[SP_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}]; 7 | GO 8 | 9 | CREATE PROCEDURE [INT].[SP_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}] @{{../conventions.etlProcessAttribute}} int 10 | AS 11 | 12 | -- 13 | -- Hub Insert Into statement for [{{targetDataObject.name}}] 14 | -- 15 | 16 | INSERT INTO [INT].[{{targetDataObject.name}}] 17 | ( 18 | [{{targetDataObject.name}}_SK], 19 | [{{../conventions.loadDateTimeAttribute}}], 20 | {{../conventions.etlProcessAttribute}}, 21 | {{../conventions.recordSourceAttribute}}, 22 | {{#each businessKeyDefinitions}}{{#if @first}}{{#each businessKeyComponentMappings}} 23 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}}{{/if}}{{/each}} 24 | ) 25 | SELECT 26 | CONVERT(CHAR,HASHBYTES('SHA1',UPPER({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(ISNULL({{stringcompare sourceDataItems.0.isHardCodedValue 'True'}} [{{/stringcompare }}{{sourceDataItems.0.name}}{{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}] {{/stringcompare }},'NA')){{#unless @last}}+'|'+{{/unless}}{{/each}} {{/each}} )),2) as [{{targetDataObject.name}}_SK], 27 | min([{{../conventions.loadDateTimeAttribute}}]), 28 | @{{../conventions.etlProcessAttribute}}, 29 | min({{../conventions.recordSourceAttribute}}), 30 | {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 31 | ISNULL({{stringcompare sourceDataItems.0.isHardCodedValue 'True'}} [{{/stringcompare }}{{sourceDataItems.0.name}}{{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}] {{/stringcompare }},'NA') AS [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}}{{/each}} 32 | 33 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] stg 34 | WHERE CONVERT(CHAR,HASHBYTES('SHA1',UPPER({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(ISNULL({{stringcompare sourceDataItems.0.isHardCodedValue 'True'}} [{{/stringcompare }}{{sourceDataItems.0.name}}{{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}] {{/stringcompare }},'NA')){{#unless @last}}+'|'+{{/unless}}{{/each}} {{/each}} )),2) NOT IN 35 | ( 36 | SELECT [{{targetDataObject.name}}_SK] 37 | FROM [INT].[{{targetDataObject.name}}] INT 38 | ) 39 | GROUP BY {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}}{{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}{{#unless @first}},{{/unless}} [{{sourceDataItems.0.name}}]{{/stringcompare}}{{/each}}{{/each}} ; 40 | 41 | SELECT @@ROWCOUNT AS ROWS_INSERTED 42 | GO 43 | {{/each}} 44 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateHubView.Handlebars: -------------------------------------------------------------------------------- 1 | CREATE OR ALTER VIEW [{{conventions.vdwSchemaName}}].[{{dataObjectMappings.0.targetDataObject.name}}] 2 | AS 3 | 4 | -- 5 | -- Hub View definition for {{dataObjectMappings.0.targetDataObject.name}}. 6 | -- 7 | -- This template represents a standard Data Vault style 'Hub' table as a view. 8 | -- The view shows the same data as would otherwise be the case if the table would be created and all data logistics processes run. 9 | -- 10 | -- Generated from template '200 Data Vault Hub View'. 11 | -- 12 | 13 | SELECT hub.* 14 | FROM 15 | ( 16 | {{#each dataObjectMappings}}{{#if @first}} 17 | SELECT 18 | HASHBYTES('MD5', {{#each businessKeyDefinitions}} {{#if @first}} {{!-- Create the Surrogate Key using the Business Key and components --}} 19 | {{#each businessKeyComponentMappings}} 20 | ISNULL(RTRIM(CONVERT(NVARCHAR(100), {{targetDataItem.name}})), 'N/A') + '#~!'{{#unless @last}} +{{/unless}} 21 | {{/each}} 22 | ) AS {{surrogateKey}},{{/if}}{{/each}} 23 | -1 AS {{../conventions.etlProcessAttribute}}, {{!-- List out the ETL Framework attributes --}} 24 | MIN({{../conventions.loadDateTimeAttribute}}) AS {{../conventions.loadDateTimeAttribute}}, 25 | --{{../conventions.recordSourceAttribute}},{{#each businessKeyDefinitions}} {{!-- The Hub business key --}} 26 | {{#each businessKeyComponentMappings}} 27 | {{targetDataItem.name}},{{/each}}{{/each}} 28 | ROW_NUMBER() OVER (PARTITION BY {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} {{!-- Create a row number to enable selection only the earliest row, ordered by date/time --}} 29 | {{targetDataItem.name}}{{#unless @last}},{{/unless}}{{/each}} 30 | {{/each}} 31 | ORDER BY 32 | MIN({{../conventions.loadDateTimeAttribute}}) 33 | ) AS ROW_NR 34 | FROM 35 | ( {{/if}}{{/each}} 36 | {{#each dataObjectMappings }} 37 | {{#if enabled}}{{#stringcompare enabled "true"}} 38 | SELECT {{#each ../businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 39 | CAST({{sourceDataItems.0.name}} AS NVARCHAR(100)) AS {{targetDataItem.name}},{{/each}}{{/each}} 40 | MIN({{../../conventions.loadDateTimeAttribute}}) AS {{../../conventions.loadDateTimeAttribute}} 41 | FROM [{{lookupExtension sourceDataObjects.0.extensions "datastore"}}].[{{lookupExtension sourceDataObjects.0.extensions "location"}}].[{{sourceDataObjects.0.name}}] 42 | WHERE {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 43 | {{sourceDataItems.0.name}} IS NOT NULL{{#unless @last}} AND{{/unless}}{{/each}}{{/each}}{{#if filterCriterion}}{{#stringcompare filterCriterion ""}}{{else}}AND {{filterCriterion}}{{/stringcompare}}{{/if}} 44 | GROUP BY 45 | {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}}{{#each sourceDataItems}}{{!-- Exclude hard-coded values from the group by --}}{{#if extensions}}{{#each extensions}}{{#stringcompare key "isHardCodedValue"}}{{/stringcompare}}{{/each}}{{else}} {{name}}{{#unless @last}},{{/unless}}{{/if}}{{/each}}{{/each}}{{/each}} 46 | -- 47 | {{#unless @last}}UNION{{/unless}} 48 | {{/stringcompare}}{{/if}} 49 | {{/each}} 50 | ) HUB_selection {{#each dataObjectMappings}}{{#if @first}} 51 | GROUP BY {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 52 | {{targetDataItem.name}}{{#unless @last}},{{/unless}}{{/each}}{{/each}} 53 | --{{../conventions.recordSourceAttribute}} {{/if}}{{/each}} 54 | ) hub 55 | WHERE ROW_NR = 1 56 | UNION 57 | SELECT 58 | 0x00000000000000000000000000000000, 59 | - 1, 60 | '1900-01-01', 61 | --'Data Warehouse',{{#each dataObjectMappings}}{{#if @first}}{{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 62 | 'Unknown',{{/each}}{{/each}}{{/if}}{{/each}} 63 | 1 AS ROW_NR 64 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateHubViewWithDeliveryKey.Handlebars: -------------------------------------------------------------------------------- 1 | -- 2 | -- Hub View definition for [{{dataObjectMappings.0.targetDataObject.name}}] 3 | -- 4 | -- This template represents a standard Data Vault style 'Hub' table as a view. 5 | -- The view shows the same data as would otherwise be the case if the table would be created and all data logistics processes run. 6 | -- 7 | 8 | IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[{{conventions.vdwSchemaName}}].[{{dataObjectMappings.0.targetDataObject.name}}]') AND type in (N'V')) 9 | DROP VIEW [{{conventions.vdwSchemaName}}].[{{dataObjectMappings.0.targetDataObject.name}}] 10 | GO 11 | 12 | CREATE VIEW [{{conventions.vdwSchemaName}}].[{{dataObjectMappings.0.targetDataObject.name}}] AS 13 | SELECT {{#each dataObjectMappings}}{{#if @first}} 14 | {{#each businessKeyDefinitions}} {{#if @first}}hub.[{{surrogateKey}}],{{/if}}{{/each}} 15 | hub.[{{../conventions.etlProcessAttribute}}], 16 | hub.[{{../conventions.loadDateTimeAttribute}}],{{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 17 | hub.[{{targetDataItem.name}}],{{/each}}{{/each}} 18 | ROW_NUMBER() OVER ( ORDER BY 19 | [{{../conventions.loadDateTimeAttribute}}],{{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 20 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}}{{/each}} 21 | ) AS DATA_DELIVERY_KEY 22 | {{/if}}{{/each}} 23 | FROM ( 24 | {{#each dataObjectMappings}}{{#if @first}} 25 | SELECT 26 | HASHBYTES('MD5', {{#each businessKeyDefinitions}} {{#if @first}} {{!-- Create the Surrogate Key using the Business Key and components --}} 27 | {{#each businessKeyComponentMappings}} 28 | ISNULL(RTRIM(CONVERT(NVARCHAR(100), {{targetDataItem.name}})), 'N/A') + '#~!'{{#unless @last}} +{{/unless}} 29 | {{/each}} 30 | ) AS {{surrogateKey}},{{/if}}{{/each}} 31 | -1 AS {{../conventions.etlProcessAttribute}}, 32 | MIN({{../conventions.loadDateTimeAttribute}}) AS {{../conventions.loadDateTimeAttribute}}, 33 | --{{../conventions.recordSourceAttribute}},{{#each businessKeyDefinitions}} {{!-- The Hub business key --}} 34 | {{#each businessKeyComponentMappings}} 35 | {{targetDataItem.name}},{{/each}}{{/each}} 36 | ROW_NUMBER() OVER (PARTITION BY {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} {{!-- Create a row number to enable selection only the earliest row, ordered by date/time --}} 37 | {{targetDataItem.name}}{{#unless @last}},{{/unless}}{{/each}} 38 | {{/each}} 39 | ORDER BY 40 | MIN({{../conventions.loadDateTimeAttribute}}) 41 | ) AS ROW_NR 42 | FROM 43 | ( {{/if}}{{/each}} 44 | {{#each dataObjectMappings }} 45 | SELECT {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 46 | CAST({{sourceDataItems.0.name}} AS NVARCHAR(100)) AS {{targetDataItem.name}},{{/each}}{{/each}} 47 | --{{../conventions.recordSourceAttribute}}, 48 | MIN({{../conventions.loadDateTimeAttribute}}) AS {{../conventions.loadDateTimeAttribute}} 49 | FROM [{{lookupExtension sourceDataObjects.0.extensions "datastore"}}].[{{lookupExtension sourceDataObjects.0.extensions "location"}}].[{{sourceDataObjects.0.name}}] 50 | WHERE {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 51 | {{sourceDataItems.0.name}} IS NOT NULL{{#unless @last}} AND{{/unless}}{{/each}}{{/each}} 52 | {{#if filterCriterion}}AND {{filterCriterion}}{{/if}} 53 | GROUP BY 54 | {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}}{{#each sourceDataItems}}{{#stringcompare isHardCodedValue "True"}}{{else}}{{name}}{{#unless @last}},{{/unless}}{{/stringcompare}}{{/each}}{{/each}}{{/each}} 55 | -- {{../conventions.recordSourceAttribute}} 56 | {{#unless @last}}UNION{{/unless}} 57 | {{/each}} 58 | ) HUB_selection {{#each dataObjectMappings}}{{#if @first}} 59 | GROUP BY {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 60 | {{targetDataItem.name}}{{#unless @last}},{{/unless}}{{/each}}{{/each}} 61 | --{{../conventions.recordSourceAttribute}} {{/if}}{{/each}} 62 | ) hub 63 | WHERE ROW_NR = 1 64 | UNION 65 | SELECT 66 | 0x00000000000000000000000000000000, 67 | - 1, 68 | '1900-01-01', 69 | --'Data Warehouse',{{#each dataObjectMappings}}{{#if @first}}{{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 70 | 'Unknown',{{/each}}{{/each}}{{/if}}{{/each}} 71 | 0 AS DATA_DELIVERY_KEY 72 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateHubView_DBT_Snowflake.Handlebars: -------------------------------------------------------------------------------- 1 | \{{ config(materialized='view') }} 2 | -- 3 | -- Hub View definition for {{dataObjectMappings.0.targetDataObject}} 4 | -- 5 | 6 | SELECT hub.* 7 | FROM ( 8 | {{#each dataObjectMappings}}{{#if @first}} 9 | SELECT 10 | MD5({{#each businessKeyDefinitions}} {{#if @first}} {{!-- Create the Surrogate Key using the Business Key and components --}} 11 | {{#each businessKeyComponentMappings}} 12 | COALESCE(RTRIM(CAST({{targetDataItem.name}} AS NVARCHAR(100))), 'N/A') || '#~!'{{#unless @last}} +{{/unless}} 13 | {{/each}} 14 | ) AS {{surrogateKey}},{{/if}}{{/each}} 15 | -1 AS {{../conventions.etlProcessAttribute}}, {{!-- List out the ETL Framework attributes --}} 16 | MIN({{../conventions.loadDateTimeAttribute}}) AS {{../conventions.loadDateTimeAttribute}}, 17 | {{../conventions.recordSourceAttribute}},{{#each businessKeyDefinitions}} {{!-- The Hub business key --}} 18 | {{#each businessKeyComponentMappings}} 19 | {{targetDataItem.name}},{{/each}}{{/each}} 20 | ROW_NUMBER() OVER (PARTITION BY {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} {{!-- Create a row number to enable selection only the earliest row, ordered by date/time --}} 21 | {{targetDataItem.name}}{{#unless @last}},{{/unless}}{{/each}} 22 | {{/each}} 23 | ORDER BY 24 | MIN({{../conventions.loadDateTimeAttribute}}) 25 | ) AS ROW_NR 26 | FROM 27 | ( {{/if}}{{/each}} 28 | {{#each dataObjectMappings }} 29 | SELECT {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 30 | CAST({{sourceDataItems.0.name}} AS NVARCHAR(100)) AS {{targetDataItem.name}},{{/each}}{{/each}} 31 | {{../conventions.recordSourceAttribute}}, 32 | MIN({{../conventions.loadDateTimeAttribute}}) AS {{../conventions.loadDateTimeAttribute}} 33 | FROM VDW.{{sourceDataObjects.0.name}} 34 | WHERE {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 35 | {{sourceDataItems.0.name}} IS NOT NULL{{#unless @last}} AND{{/unless}}{{/each}}{{/each}} 36 | {{#if filterCriterion}}AND {{filterCriterion}}{{/if}} 37 | GROUP BY 38 | {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}}{{#each sourceDataItems}}{{#stringcompare isHardCodedValue "True"}}{{else}}{{name}},{{/stringcompare}}{{/each}}{{/each}}{{/each}} 39 | {{../conventions.recordSourceAttribute}} 40 | {{#unless @last}}UNION{{/unless}} 41 | {{/each}} 42 | ) HUB_selection {{#each dataObjectMappings}}{{#if @first}} 43 | GROUP BY {{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 44 | {{targetDataItem.name}},{{/each}}{{/each}} 45 | {{../conventions.recordSourceAttribute}} {{/if}}{{/each}} 46 | ) hub 47 | WHERE ROW_NR = 1 48 | UNION 49 | SELECT 50 | MD5('ZeroRecord'), 51 | - 1, 52 | '1900-01-01', 53 | 'Data Warehouse',{{#each dataObjectMappings}}{{#if @first}}{{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} 54 | 'Unknown',{{/each}}{{/each}}{{/if}}{{/each}} 55 | 1 AS ROW_NR 56 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateLanding.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Landing process for {{targetDataObject.name}}. 4 | -- 5 | 6 | SELECT 7 | {{#each dataItemMappings}} 8 | [{{sourceDataItems.0.name}}], 9 | {{/each}} 10 | SYSDATETIME() AS [{{../conventions.eventDateTimeAttribute}}] 11 | ,'I' AS [{{../conventions.changeDataCaptureAttribute}}] 12 | ,0 AS [{{../conventions.etlProcessAttribute}}] 13 | ,'0x00' AS [{{../conventions.recordChecksumAttribute}}] 14 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] 15 | {{/each}} 16 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateLandingTestReconciliation.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Reconciliation Test for {{targetDataObject.name}}. 4 | -- 5 | 6 | -- Test template 7 | DECLARE @TemplateId INT; 8 | EXEC [ut].[RegisterTestTemplate] 9 | @TemplateName = 'Generic-Reconciliation', 10 | @TemplateNotes = 'https://github.com/data-solution-automation-engine/testing-framework/blob/main/test-library/Test%20-%20Generic%20-%20Reconciliation.md', 11 | @TemplateId = @TemplateId OUTPUT; 12 | PRINT concat('The Test Template Id is: ', @TemplateId, '.'); 13 | 14 | -- Test registration 15 | DECLARE @TestId INT; 16 | EXEC [ut].[RegisterTest] 17 | -- Mandatory 18 | @TemplateId = @TemplateId, 19 | @Name = 'RECON_STG_{{sourceDataObjects.0.name}}', 20 | -- sample with test procedure 21 | @Debug='Y', 22 | @TestCode = 'BEGIN 23 | -- Framework required (added for unit testing) 24 | --DECLARE @TestResult VARCHAR(10) = ''Fail''; 25 | --DECLARE @TestOutput VARCHAR(MAX); 26 | -- Local 27 | DECLARE @Issues INT = 0; 28 | 29 | BEGIN TRY 30 | SELECT @Issues = COUNT(*) 31 | FROM ( 32 | SELECT {{#each dataItemMappings}} 33 | [{{sourceDataItems.0.name}}]{{#unless @last}},{{/unless}}{{/each}} 34 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] stg 35 | EXCEPT 36 | SELECT {{#each dataItemMappings}} 37 | [{{sourceDataItems.0.name}}]{{#unless @last}},{{/unless}}{{/each}} 38 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 39 | ) sub 40 | 41 | SET @TestOutput = CONVERT(VARCHAR(10),@Issues)+'' issues were found.'' 42 | 43 | IF @Issues=0 44 | BEGIN 45 | SET @TestResult=''Pass'' 46 | END 47 | END TRY 48 | BEGIN CATCH 49 | --THROW 50 | SET @TestOutput = ERROR_MESSAGE(); 51 | SET @TestResult=''Fail'' 52 | END CATCH 53 | 54 | SELECT @TestOutput AS [OUTPUT], @TestResult AS [RESULT] 55 | END ', 56 | @TestObject = '{{targetDataObject.dataObjectConnection.extensions.1.value}}.{{targetDataObject.name}}', 57 | -- Output 58 | @TestId = @TestId OUTPUT; 59 | 60 | -- Add DIRECT registration to run from control framework 61 | DECLARE @ModuleId INT; 62 | 63 | EXEC [omd].[RegisterModule] 64 | @ModuleCode = 'RECON_STG_{{sourceDataObjects.0.name}}' 65 | ,@ModuleAreaCode = 'Maintenance' 66 | ,@Executable = 'EXEC [ut].[RunTest] @TestName = ''RECON_STG_{{sourceDataObjects.0.name}}''' 67 | -- Non mandatory 68 | ,@ModuleDescription = 'Reconciliation Unit Test' 69 | ,@Debug = 'Y' 70 | ,@ModuleId = @ModuleId OUTPUT; 71 | 72 | PRINT 'The Module Id is: '+CONVERT(VARCHAR(10),@ModuleId)+'.'; 73 | 74 | -- Example - running the DIRECT Module 75 | EXEC [{{lookupExtension ../../extensions "controlFrameworkDataStore"}}].[{{lookupExtension ../../extensions "controlFrameworkLocation"}}].[RunModule] @ModuleCode = 'RECON_STG_{{sourceDataObjects.0.name}}', @Debug='Y', @ModuleInstanceIdColumnName='AUDIT_TRAIL_ID' 76 | {{/each}} 77 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateLandingWithDirect.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Landing process for {{targetDataObject.name}} using a DIRECT wrapper. 4 | -- 5 | 6 | IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'SP_{{targetDataObject.name}}') AND type in (N'P')) 7 | DROP PROCEDURE [SP_{{targetDataObject.name}}]; 8 | GO 9 | 10 | CREATE OR ALTER PROCEDURE [SP_{{targetDataObject.name}}] @{{../conventions.etlProcessAttribute}} INT 11 | AS 12 | 13 | TRUNCATE TABLE [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] 14 | 15 | INSERT INTO [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] 16 | ( 17 | [{{../conventions.etlProcessAttribute}}], 18 | [{{../conventions.eventDateTimeAttribute}}], 19 | [{{../conventions.changeDataCaptureAttribute}}], 20 | [{{../conventions.recordChecksumAttribute}}], 21 | {{#each dataItemMappings}} 22 | [{{sourceDataItems.0.name}}]{{#unless @last}},{{/unless}} 23 | {{/each}} 24 | ) 25 | SELECT 26 | @AUDIT_TRAIL_ID AS [{{../conventions.etlProcessAttribute}}] 27 | ,SYSDATETIME() AS [{{../conventions.eventDateTimeAttribute}}] 28 | ,'I' AS [{{../conventions.changeDataCaptureAttribute}}] 29 | ,0x00 AS [{{../conventions.recordChecksumAttribute}}] 30 | {{#each dataItemMappings}} 31 | ,[{{sourceDataItems.0.name}}] 32 | {{/each}} 33 | FROM [{{lookupExtension sourceDataObjects.0.extensions "datastore"}}].[{{lookupExtension sourceDataObjects.0.extensions "location"}}].[{{sourceDataObjects.0.name}}] 34 | GO 35 | 36 | -- DIRECT module registration 37 | 38 | EXEC [{{lookupExtension ../../extensions "controlFrameworkDataStore"}}].[{{lookupExtension ../../extensions "controlFrameworkLocation"}}].[RegisterModule] 39 | @ModuleCode = '100_{{targetDataObject.name}}' 40 | ,@ModuleAreaCode = 'STG' 41 | ,@Executable = 'EXEC [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[SP_{{targetDataObject.name}}] @AUDIT_TRAIL_ID = @ModuleInstanceId' 42 | ,@ModuleDescription = 'Source to landing for [{{sourceDataObjects.0.name}}]' 43 | ,@ModuleSourceDataObject = '[{{lookupExtension sourceDataObjects.0.extensions "datastore"}}].[{{lookupExtension sourceDataObjects.0.extensions "location"}}].[{{sourceDataObjects.0.name}}]' 44 | ,@ModuleTargetDataObject = '[{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}]' 45 | 46 | -- Run the process 47 | EXEC [{{lookupExtension ../../extensions "controlFrameworkDataStore"}}].[{{lookupExtension ../../extensions "controlFrameworkLocation"}}].[RunModule] @ModuleCode = '100_{{targetDataObject.name}}', @Debug='Y', @ModuleInstanceIdColumnName='AUDIT_TRAIL_ID' 48 | {{/each}} 49 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateLandingtoSilverFromView.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | 3 | update omd.PARAMETER set PARAMETER_VALUE_CODE='INSERT INTO DL_{{targetDataObject.name}} 4 | ( 5 | {{#each dataItemMappings}} 6 | {{sourceDataItems.0.name}}{{#unless @last}},{{/unless}} 7 | {{/each}} 8 | ,OMD_INSERT_MODULE_INSTANCE_ID 9 | ,OMD_FILE_LOCATION 10 | ,OMD_EVENT_DATETIME 11 | ,OMD_INSERT_DATETIME 12 | ,OMD_HASH_DIFF 13 | ,OMD_RECORD_SOURCE_ID 14 | 15 | ) 16 | 17 | SELECT {{#each dataItemMappings}} 18 | vw.{{sourceDataItems.0.name}}{{#unless @last}},{{/unless}} 19 | {{/each}} 20 | ,vw.OMD_INSERT_MODULE_INSTANCE_ID 21 | ,vw.OMD_FILE_LOCATION 22 | ,vw.OMD_EVENT_DATETIME 23 | ,vw.OMD_INSERT_DATETIME 24 | ,vw.OMD_HASH_DIFF 25 | ,vw.OMD_RECORD_SOURCE_ID 26 | FROM VW_DL_{{targetDataObject.name}} AS vw' where parameter_key_code='LANDING_TO_STAGING_SELECT_{{targetDataObject.name}}'; 27 | 28 | {{/each}} 29 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateLinkGenerateTables.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}}{{#if @first}} 2 | -- 3 | -- Table creation statement for {{targetDataObject.name}} 4 | -- 5 | 6 | IF OBJECT_ID('[{{targetDataObject.name}}]', 'U') IS NOT NULL 7 | DROP TABLE [{{targetDataObject.name}}]; 8 | 9 | CREATE TABLE [{{targetDataObject.name}}] 10 | ( 11 | [{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] binary(16) NOT NULL 12 | ,[{{../conventions.etlProcessAttribute}}] [int] NOT NULL 13 | ,[{{../conventions.loadDateTimeAttribute}}] [datetime2](7) NOT NULL 14 | ,[{{../conventions.recordSourceAttribute}}] [nvarchar](100) NOT NULL 15 | {{#each relatedDataObjects}} 16 | ,[{{StringReplace name "HUB_" ""}}_SK] binary(16) NOT NULL 17 | {{/each}} 18 | ){{/if}}{{/each}} 19 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateLinkInsertInto.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}}{{#if @first}} 2 | -- 3 | -- Link Insert Into statement for {{targetDataObject.name}} 4 | -- 5 | 6 | INSERT INTO [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] 7 | ( {{#each businessKeyDefinitions}} 8 | [{{surrogateKey}}],{{/each}}{{#each dataItemMappings}} 9 | [{{targetDataItem.name}}],{{/each}} 10 | [{{../conventions.etlProcessAttribute}}], 11 | [{{../conventions.loadDateTimeAttribute}}] 12 | ) 13 | SELECT {{#each businessKeyDefinitions}} 14 | link_view.[{{surrogateKey}}],{{/each}}{{#each dataItemMappings}} 15 | link_view.[{{targetDataItem.name}}],{{/each}} 16 | -1 AS [{{../conventions.etlProcessAttribute}}], 17 | link_view.[{{../conventions.loadDateTimeAttribute}}] 18 | FROM [{{../conventions.vdwSchemaName}}].[{{targetDataObject.name}}] link_view 19 | LEFT OUTER JOIN [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] link_table 20 | ON link_view.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] = link_table.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] 21 | WHERE link_table.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] IS NULL 22 | {{/if}} 23 | 24 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateLinkInsertIntoProcedure.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Link Insert Into statement for {{targetDataObject.name}} 4 | -- 5 | 6 | IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'INT.SP_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}') AND type in (N'P')) 7 | DROP PROCEDURE [INT].[SP_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}]; 8 | GO 9 | 10 | CREATE PROCEDURE [INT].[SP_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}] @{{../conventions.etlProcessAttribute}} int 11 | AS 12 | 13 | -- 14 | -- LNK Insert Into statement for [{{targetDataObject.name}}] 15 | -- Includes the GROUP BY clause 16 | 17 | INSERT INTO [INT].[{{targetDataObject.name}}] 18 | ( {{#each businessKeyDefinitions}} {{#if @first}} [{{surrogateKey}}], {{/if}}{{/each}} 19 | {{#each businessKeyDefinitions}}{{#unless @first}} 20 | [{{surrogateKey}}],{{/unless}} {{/each}} 21 | [{{../conventions.loadDateTimeAttribute}}], 22 | {{../conventions.recordSourceAttribute}}, 23 | {{../conventions.etlProcessAttribute}} 24 | ) 25 | 26 | SELECT 27 | CONVERT(CHAR,HASHBYTES('SHA1',UPPER( {{#each businessKeyDefinitions}} {{#unless @first}}{{!-- Create the Surrogate Key using the combined Business Keys --}} 28 | {{#each dataItemMappings}} RTRIM(ISNULL([{{targetDataItem.name}}], 'NA')) + '|'+{{/each}} 29 | {{#each businessKeyComponentMappings}} RTRIM(ISNULL([{{targetDataItem.name}}{{@../index }}], 'NA')) {{#unless @last}} + '|' +{{/unless}}{{/each}}{{#unless @last}} +'|'+{{/unless}}{{/unless}}{{/each}} 30 | {{#each dataItemMappings}} + RTRIM(ISNULL([{{targetDataItem.name}}], 'NA')) + '|'+{{/each}}{{#each businessKeyDefinitions}} {{#if @first}})),2) AS [{{surrogateKey}}],{{/if}}{{/each}} 31 | {{#each businessKeyDefinitions}}{{#unless @first}} {{!-- List out the separate (Hub) business keys, by skipping the first generic relationship key. --}} 32 | {{#unless businessKeyClassification}} 33 | CONVERT(CHAR,HASHBYTES('SHA1',UPPER( {{#each businessKeyComponentMappings}} RTRIM(ISNULL([{{targetDataItem.name}}{{@../index }}],'NA')) {{#unless @last}} +'|'+{{/unless}}{{/each}})),2) AS [{{surrogateKey}}], 34 | {{/unless}}{{#if businessKeyClassification}}{{#each businessKeyComponentMappings}}[{{targetDataItem.name}}{{@../index }}]{{/each}} AS [{{surrogateKey}}], 35 | {{/if}} {{/unless}} {{/each}}{{#each dataItemMappings}} 36 | [{{targetDataItem.name}}],{{/each}} 37 | [{{../conventions.loadDateTimeAttribute}}], 38 | {{../conventions.recordSourceAttribute}}, 39 | @{{../conventions.etlProcessAttribute}} 40 | FROM 41 | ( 42 | SELECT {{#each businessKeyDefinitions}} {{#unless @first}} {{#each businessKeyComponentMappings}} 43 | CAST({{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}[{{/stringcompare}}{{sourceDataItems.0.name}}{{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}]{{/stringcompare}} AS NVARCHAR(100)) AS [{{targetDataItem.name}}{{@../index }}],{{/each}}{{/unless}}{{/each}}{{#each dataItemMappings}} 44 | [{{sourceDataItems.0.name}}] AS [{{targetDataItem.name}}],{{/each}} 45 | min([{{../conventions.loadDateTimeAttribute}}]) AS [{{../conventions.loadDateTimeAttribute}}], 46 | min({{../conventions.recordSourceAttribute}}) AS {{../conventions.recordSourceAttribute}} 47 | FROM STG.[{{sourceDataObjects.0.name}}] 48 | GROUP BY {{#each businessKeyDefinitions}}{{#if @first}}{{#each businessKeyComponentMappings}}{{stringcompare sourceDataItems.0.isHardCodedValue 'True'}}{{#unless @first}},{{/unless}} [{{sourceDataItems.0.name}}]{{/stringcompare}}{{/each}}{{/if}}{{/each}} 49 | ) LINK_selection 50 | WHERE CONVERT(CHAR,HASHBYTES('SHA1',UPPER( {{#each businessKeyDefinitions}} {{#unless @first}}{{!-- Create the Surrogate Key using the combined Business Keys --}} 51 | {{#each dataItemMappings}} RTRIM(ISNULL([{{targetDataItem.name}}], 'NA')) + '|'+{{/each}} 52 | {{#each businessKeyComponentMappings}} RTRIM(ISNULL([{{targetDataItem.name}}{{@../index }}], 'NA')) {{#unless @last}} + '|' +{{/unless}}{{/each}}{{#unless @last}} +'|'+{{/unless}}{{/unless}}{{/each}} 53 | {{#each dataItemMappings}} + RTRIM(ISNULL([{{targetDataItem.name}}], 'NA')) + '|'+{{/each}}{{#each businessKeyDefinitions}} {{#if @first}})),2) {{/if}}{{/each}} NOT IN 54 | ( 55 | SELECT [{{targetDataObject.name}}_SK] 56 | FROM [INT].[{{targetDataObject.name}}] 57 | ); 58 | SELECT @@ROWCOUNT AS ROWS_INSERTED 59 | GO 60 | {{/each}} 61 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateLinkSatelliteDrivingKeyInsertInto.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}}{{#if @first}} 2 | -- 3 | -- Link-Satellite Insert Into statement for {{targetDataObject.name}}. 4 | -- 5 | -- This template inserts the data delta that has been detected into the landing area table. 6 | -- 7 | 8 | INSERT INTO [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] 9 | ( {{#each businessKeyDefinitions}}{{#if @first}} 10 | [{{surrogateKey}}],{{/if}}{{/each}} 11 | {{#each dataItemMappings}} 12 | [{{targetDataItem.name}}], 13 | {{/each}} 14 | [{{../conventions.loadDateTimeAttribute}}], 15 | [{{../conventions.sourceRowIdAttribute}}], 16 | [{{../conventions.changeDataCaptureAttribute}}], 17 | [{{../conventions.etlProcessAttribute}}], 18 | [{{../conventions.recordChecksumAttribute}}] 19 | ) 20 | SELECT {{#each businessKeyDefinitions}}{{#if @first}} 21 | sat_view.[{{surrogateKey}}],{{/if}}{{/each}} 22 | {{#each dataItemMappings}} 23 | sat_view.[{{targetDataItem.name}}], 24 | {{/each}} 25 | sat_view.[{{../conventions.loadDateTimeAttribute}}], 26 | sat_view.[{{../conventions.sourceRowIdAttribute}}], 27 | sat_view.[{{../conventions.changeDataCaptureAttribute}}], 28 | -1 AS [{{../conventions.etlProcessAttribute}}], 29 | sat_view.[{{../conventions.recordChecksumAttribute}}] 30 | FROM [{{../conventions.vdwSchemaName}}].[{{targetDataObject.name}}] sat_view 31 | LEFT OUTER JOIN [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] sat_table 32 | ON sat_view.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] = sat_table.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] 33 | AND sat_view.[{{../conventions.loadDateTimeAttribute}}] = sat_table.[{{../conventions.loadDateTimeAttribute}}] 34 | {{#each dataItemMappings}}{{#each targetDataItem.classifications}}{{#if classification}}AND sat_view.{{../targetDataItem.name}} = sat_table.{{../targetDataItem.name}}{{/if}}{{/each}}{{/each}} 35 | WHERE sat_table.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] IS NULL 36 | {{/if}} 37 | 38 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateLinkSatelliteInsertInto.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}}{{#if @first}} 2 | -- 3 | -- Link-Satellite Insert Into statement for {{targetDataObject.name}}. 4 | -- 5 | -- This template inserts the data delta that has been detected into the landing area table. 6 | -- 7 | 8 | INSERT INTO [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] 9 | ( {{#each businessKeyDefinitions}}{{#if @first}} 10 | [{{surrogateKey}}],{{/if}}{{/each}} 11 | {{#each dataItemMappings}} 12 | [{{targetDataItem.name}}], 13 | {{/each}} 14 | [{{../conventions.etlProcessAttribute}}], 15 | [{{../conventions.loadDateTimeAttribute}}], 16 | [{{../conventions.sourceRowIdAttribute}}], 17 | [{{../conventions.changeDataCaptureAttribute}}], 18 | [{{../conventions.recordChecksumAttribute}}] 19 | ) 20 | SELECT {{#each businessKeyDefinitions}}{{#if @first}} 21 | sat_view.[{{surrogateKey}}],{{/if}}{{/each}} 22 | {{#each dataItemMappings}} 23 | sat_view.[{{targetDataItem.name}}], 24 | {{/each}} 25 | -1 AS [{{../conventions.etlProcessAttribute}}], 26 | sat_view.[{{../conventions.loadDateTimeAttribute}}], 27 | sat_view.[{{../conventions.sourceRowIdAttribute}}], 28 | sat_view.[{{../conventions.changeDataCaptureAttribute}}], 29 | sat_view.[{{../conventions.recordChecksumAttribute}}] 30 | FROM [{{../conventions.vdwSchemaName}}].[{{targetDataObject.name}}] sat_view 31 | LEFT OUTER JOIN [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] sat_table 32 | ON sat_view.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] = sat_table.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] 33 | AND sat_view.[{{../conventions.loadDateTimeAttribute}}] = sat_table.[{{../conventions.loadDateTimeAttribute}}] 34 | {{#each dataItemMappings}}{{#each targetDataItem.classifications}}{{#if classification}}AND sat_view.{{../targetDataItem.name}} = sat_table.{{../targetDataItem.name}}{{/if}}{{/each}}{{/each}} 35 | WHERE sat_table.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] IS NULL 36 | {{/if}} 37 | 38 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templatePersistentStagingGenerateTables.Handlebars: -------------------------------------------------------------------------------- 1 | -- 2 | -- Presentation table creation statement for [{{dataObjectMappings.0.targetDataObject.name}}] 3 | -- 4 | 5 | DROP TABLE IF EXISTS [{{dataObjectMappings.0.targetDataObject.name}}] 6 | 7 | CREATE TABLE [{{lookupExtension dataObjectMappings.0.targetDataObject.extensions "datastore"}}].[{{lookupExtension dataObjectMappings.0.targetDataObject.extensions "location"}}].[{{dataObjectMappings.0.targetDataObject.name}}] 8 | ( 9 | {{#each dataObjectMappings.0.targetDataObject.dataItems}} 10 | [{{name}}] {{dataType}}{{#stringcompare dataType int}}{{else}}({{characterLength}}{{numericScale}}){{/stringcompare}}, 11 | {{/each}} 12 | [{{conventions.eventDateTimeAttribute}}] DATETIME2(7), 13 | [{{conventions.changeDataCaptureAttribute}}] CHAR(1), 14 | [{{conventions.etlProcessAttribute}}] INT, 15 | [{{conventions.sourceRowIdAttribute}}] INT, 16 | [{{conventions.recordChecksumAttribute}}] BINARY(16) 17 | ) 18 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templatePersistentStagingInsertInto.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Persistent Staging Area Insert Into statement for {{targetDataObject.name}} 4 | -- 5 | -- Generated from template '150 Persistent Staging Area Insert Into'. 6 | -- 7 | 8 | INSERT INTO [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] 9 | ( 10 | [{{../conventions.etlProcessAttribute}}], 11 | [{{../conventions.loadDateTimeAttribute}}], 12 | [{{../conventions.eventDateTimeAttribute}}], 13 | [{{../conventions.sourceRowIdAttribute}}], 14 | [{{../conventions.changeDataCaptureAttribute}}], 15 | [{{../conventions.recordChecksumAttribute}}], 16 | {{#each dataItemMappings}} 17 | [{{sourceDataItems.0.name}}]{{#unless @last}},{{/unless}} 18 | {{/each}} 19 | ) 20 | SELECT 21 | -1 AS [{{../conventions.etlProcessAttribute}}], 22 | [{{../conventions.loadDateTimeAttribute}}], 23 | [{{../conventions.eventDateTimeAttribute}}], 24 | [{{../conventions.sourceRowIdAttribute}}], 25 | [{{../conventions.changeDataCaptureAttribute}}], 26 | [{{../conventions.recordChecksumAttribute}}], 27 | {{#each dataItemMappings}} 28 | [{{sourceDataItems.0.name}}]{{#unless @last}},{{/unless}} 29 | {{/each}} 30 | FROM [{{../conventions.vdwSchemaName}}].[{{targetDataObject.name}}] 31 | 32 | {{/each}} 33 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templatePersistentStagingInsertIntoProcedure.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Persistent Staging Area Insert Into statement for [{{targetDataObject.name}}] 4 | -- 5 | 6 | IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'PSA.SP_{{targetDataObject.name}}') AND type in (N'P')) 7 | DROP PROCEDURE [PSA].[SP_{{targetDataObject.name}}] 8 | GO 9 | 10 | CREATE PROCEDURE [PSA].[SP_{{targetDataObject.name}}] @{{../conventions.etlProcessAttribute}} int 11 | AS 12 | INSERT INTO [PSA].[{{targetDataObject.name}}] 13 | ( 14 | [{{businessKeyDefinitions.0.surrogateKey}}], 15 | {{#each dataItemMappings}} 16 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}} 17 | {{/each}} 18 | ,{{../conventions.etlProcessAttribute}} 19 | ,OMD_FILE_LOCATION 20 | ,{{../conventions.eventDateTimeAttribute}} 21 | ,OMD_EFFECTIVE_DATETIME 22 | ,OMD_HASH_DIFF 23 | ,OMD_CURRENT_RECORD_INDICATOR 24 | ,OMD_DELETED_RECORD_INDICATOR 25 | ,OMD_EXPIRY_DATETIME 26 | ,{{../conventions.recordSourceAttribute}} 27 | ,{{../conventions.sourceRowIdAttribute}} 28 | ) 29 | SELECT 30 | CONVERT(CHAR,HASHBYTES('SHA1',UPPER({{#each businessKeyDefinitions}}{{#each businessKeyComponentMappings}} RTRIM(ISNULL([{{sourceDataItems.0.name}}],'NA')){{#unless @last}}+'|'+{{/unless}}{{/each}} {{/each}} )),2), 31 | {{#each dataItemMappings}} 32 | [{{sourceDataItems.0.name}}] as [{{targetDataItem.name}}]{{#unless @last}},{{/unless}} 33 | {{/each}} 34 | ,@{{../conventions.etlProcessAttribute}} 35 | ,OMD_FILE_LOCATION 36 | ,{{../conventions.eventDateTimeAttribute}} 37 | ,[{{../conventions.loadDateTimeAttribute}}] 38 | ,OMD_HASH_DIFF 39 | ,OMD_CURRENT_RECORD_INDICATOR 40 | ,OMD_DELETED_RECORD_INDICATOR 41 | ,OMD_EXPIRY_DATETIME 42 | ,{{../conventions.recordSourceAttribute}} 43 | ,{{../conventions.sourceRowIdAttribute}} 44 | 45 | FROM [STG].[{{sourceDataObjects.0.name}}] ; 46 | SELECT @@ROWCOUNT AS ROWS_INSERTED 47 | GO 48 | {{/each}} 49 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templatePersistentStagingViewSimplified.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Persistent Staging Area View definition for {{targetDataObject.name}} 4 | -- 5 | 6 | IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[{{../conventions.vdwSchemaName}}].[{{targetDataObject.name}}]') AND type in (N'V')) 7 | DROP VIEW [{{../conventions.vdwSchemaName}}].[{{targetDataObject.name}}] 8 | GO 9 | 10 | CREATE VIEW [{{../conventions.vdwSchemaName}}].[{{targetDataObject.name}}] AS 11 | 12 | SELECT 13 | {{#each dataItemMappings}} 14 | STG.[{{targetDataItem.name}}], 15 | {{/each}} 16 | STG.[{{../conventions.loadDateTimeAttribute}}], 17 | STG.[{{../conventions.eventDateTimeAttribute}}], 18 | --STG.[{{../conventions.recordSourceAttribute}}], 19 | STG.[{{../conventions.sourceRowIdAttribute}}], 20 | STG.[{{../conventions.changeDataCaptureAttribute}}], 21 | STG.[{{../conventions.recordChecksumAttribute}}] 22 | FROM [{{lookupExtension sourceDataObjects.0.extensions "datastore"}}].[{{lookupExtension sourceDataObjects.0.extensions "location"}}].[{{sourceDataObjects.0.name}}] STG 23 | LEFT OUTER JOIN -- Prevent reprocessing 24 | [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] HSTG 25 | ON{{#each businessKeyDefinitions}} {{#each businessKeyComponentMappings}} 26 | HSTG.[{{targetDataItem.name}}] = STG.[{{sourceDataItems.0.name}}] AND{{/each}} {{/each}} 27 | HSTG.[{{../conventions.sourceRowIdAttribute}}] = STG.[{{../conventions.sourceRowIdAttribute}}] AND 28 | HSTG.[{{../conventions.loadDateTimeAttribute}}] = STG.[{{../conventions.loadDateTimeAttribute}}] 29 | WHERE {{#each businessKeyDefinitions}} {{#each businessKeyComponentMappings}} {{#if @first}} 30 | HSTG.[{{targetDataItem.name}}] IS NULL -- prevent reprocessing{{/if}}{{/each}} {{/each}} 31 | 32 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templatePresentationGenerateTables.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}}{{#if @first}} 2 | -- 3 | -- Table creation statement for {{targetDataObject.name}} 4 | -- 5 | 6 | IF OBJECT_ID('[{{targetDataObject.name}}]', 'U') IS NOT NULL 7 | DROP TABLE [{{targetDataObject.name}}]; 8 | 9 | CREATE TABLE [{{targetDataObject.name}}] 10 | ({{/if}}{{/each}} 11 | {{#each dataObjectMappings}} 12 | {{#each dataItemMappings}} 13 | ,[{{targetDataItem.name}}] [nvarchar](1000) NOT NULL 14 | {{/each}} 15 | {{/each}} 16 | [SNAPSHOT_DATE] [datetime2](7) NOT NULL, 17 | [{{metadataConfiguration.etlProcessAttribute}}] [int] NOT NULL, 18 | [{{metadataConfiguration.loadDateTimeAttribute}}] [datetime2](7) NOT NULL, 19 | [{{metadataConfiguration.changeDataCaptureAttribute}}] [nvarchar](100) NOT NULL, 20 | [{{metadataConfiguration.eventDateTimeAttribute}}] [datetime2](7) NOT NULL, 21 | [{{metadataConfiguration.recordChecksumAttribute}}] [binary](16) NOT NULL 22 | ) 23 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateSatelliteGenerateTables.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}}{{#if @first}} 2 | -- 3 | -- Table creation statement for {{targetDataObject.name}} 4 | -- 5 | 6 | IF OBJECT_ID('[{{targetDataObject.name}}]', 'U') IS NOT NULL 7 | DROP TABLE [{{targetDataObject.name}}]; 8 | 9 | CREATE TABLE [{{targetDataObject.name}}] 10 | ( 11 | [{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] binary(16) NOT NULL 12 | ,[{{../conventions.etlProcessAttribute}}] [int] NOT NULL 13 | ,[{{../conventions.loadDateTimeAttribute}}] [datetime2](7) NOT NULL 14 | ,[{{../conventions.recordSourceAttribute}}] [nvarchar](100) NOT NULL 15 | {{#each relatedDataObjects}} 16 | {{/each}} 17 | {{#each dataItemMappings}} 18 | ,[{{targetDataItem.name}}] [nvarchar](1000) NOT NULL 19 | {{/each}} 20 | ){{/if}}{{/each}} 21 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateSatelliteInsertInto.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}}{{#if @first}} 2 | -- 3 | -- Satellite Insert Into statement for {{targetDataObject.name}}. 4 | -- 5 | -- This template inserts the data delta that has been detected by comparing the Satellite View to the existing data into the target Satellite table. 6 | -- 7 | 8 | INSERT INTO [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] 9 | ( {{#each businessKeyDefinitions}} 10 | [{{surrogateKey}}],{{/each}} 11 | {{#each dataItemMappings}} 12 | [{{targetDataItem.name}}], 13 | {{/each}} 14 | [{{../conventions.etlProcessAttribute}}], 15 | [{{../conventions.loadDateTimeAttribute}}], 16 | --[{{../conventions.recordSourceAttribute}}], 17 | [{{../conventions.changeDataCaptureAttribute}}], 18 | [{{../conventions.sourceRowIdAttribute}}], 19 | [{{../conventions.recordChecksumAttribute}}] 20 | ) 21 | SELECT {{#each businessKeyDefinitions}} 22 | sat_view.[{{surrogateKey}}],{{/each}} 23 | {{#each dataItemMappings}} 24 | sat_view.[{{targetDataItem.name}}], 25 | {{/each}} 26 | -1 AS [{{../conventions.etlProcessAttribute}}], 27 | sat_view.[{{../conventions.loadDateTimeAttribute}}], 28 | --sat_view.[{{../conventions.recordSourceAttribute}}], 29 | sat_view.[{{../conventions.changeDataCaptureAttribute}}], 30 | sat_view.[{{../conventions.sourceRowIdAttribute}}], 31 | sat_view.[{{../conventions.recordChecksumAttribute}}] 32 | FROM [{{../conventions.vdwSchemaName}}].[{{targetDataObject.name}}] sat_view 33 | LEFT OUTER JOIN [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] sat_table 34 | ON sat_view.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] = sat_table.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] 35 | AND sat_view.[{{../conventions.loadDateTimeAttribute}}] = sat_table.[{{../conventions.loadDateTimeAttribute}}] 36 | {{#each dataItemMappings}}{{#each targetDataItem.classifications}}{{#if classification}}AND sat_view.{{../targetDataItem.name}} = sat_table.{{../targetDataItem.name}}{{/if}}{{/each}}{{/each}} 37 | WHERE sat_table.[{{#each businessKeyDefinitions}}{{#if @first}}{{surrogateKey}}{{/if}}{{/each}}] IS NULL 38 | {{/if}} 39 | 40 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateSatelliteReferentialIntegrity.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}}{{#if @first}} 2 | -- 3 | -- Satellite Insert Referential Integrity check for {{targetDataObject.name}}. 4 | -- 5 | -- By convention, the parent (referencing) data object is the first object in the related data object list. 6 | -- 7 | 8 | SELECT 9 | COUNT(*) AS RI_ISSUES, 10 | '{{targetDataObject.name}}' 11 | FROM [{{../conventions.vdwSchemaName}}].{{targetDataObject.name}} sat 12 | WHERE NOT EXISTS 13 | ( 14 | SELECT 1 FROM [{{../conventions.vdwSchemaName}}].{{relatedDataObjects.0.name}} hub 15 | WHERE 1=1 16 | {{#each businessKeyDefinitions}}AND sat.{{surrogateKey}} = hub.{{surrogateKey}}{{/each}} 17 | ) 18 | {{/if}} 19 | 20 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateStagingGenerateTables.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Staging Area table creation statement for {{targetDataObject.name}}. 4 | -- 5 | -- DDL generation template that uses the data types (if available) from the mapped data items. 6 | -- 7 | 8 | IF OBJECT_ID('[{{targetDataObject.name}}]', 'U') IS NOT NULL 9 | DROP TABLE [{{targetDataObject.name}}] 10 | 11 | CREATE TABLE [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] 12 | ( 13 | [{{../conventions.loadDateTimeAttribute}}] [datetime2](7) NOT NULL DEFAULT (SYSDATETIME()), 14 | [{{../conventions.sourceRowIdAttribute}}] int IDENTITY(1,1) NOT NULL, 15 | [{{../conventions.eventDateTimeAttribute}}] [datetime2](7) NOT NULL, 16 | [{{../conventions.changeDataCaptureAttribute}}] [char](1) NOT NULL, 17 | [{{../conventions.etlProcessAttribute}}] [int] NOT NULL, 18 | --[{{../conventions.recordSourceAttribute}}] [nvarchar](100) NOT NULL, 19 | [{{../conventions.recordChecksumAttribute}}] binary(16) NOT NULL, 20 | {{#each sourceDataObjects.0.dataItems}} 21 | [{{name}}] [varchar](1000){{#unless @last}},{{/unless}} 22 | {{/each}} 23 | ) 24 | 25 | {{/each}} 26 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateStagingInsertInto.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Staging Area Insert Into statement for {{targetDataObject.name}}. 4 | -- 5 | -- This template inserts the data delta that has been detected into the landing area table. 6 | -- 7 | -- Generated from template '100 Staging Area Insert Into'. 8 | -- 9 | 10 | TRUNCATE TABLE [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] 11 | 12 | INSERT INTO [{{lookupExtension targetDataObject.extensions "datastore"}}].[{{lookupExtension targetDataObject.extensions "location"}}].[{{targetDataObject.name}}] 13 | ( 14 | [{{../conventions.etlProcessAttribute}}], 15 | [{{../conventions.eventDateTimeAttribute}}], 16 | [{{../conventions.changeDataCaptureAttribute}}], 17 | [{{../conventions.recordChecksumAttribute}}], 18 | {{#each dataItemMappings}} 19 | [{{sourceDataItems.0.name}}]{{#unless @last}},{{/unless}} 20 | {{/each}} 21 | ) 22 | SELECT 23 | -1 AS [{{../conventions.etlProcessAttribute}}], 24 | [{{../conventions.eventDateTimeAttribute}}], 25 | [{{../conventions.changeDataCaptureAttribute}}], 26 | [{{../conventions.recordChecksumAttribute}}], 27 | {{#each dataItemMappings}} 28 | [{{sourceDataItems.0.name}}]{{#unless @last}},{{/unless}} 29 | {{/each}} 30 | FROM [{{../conventions.vdwSchemaName}}].[{{targetDataObject.name}}] 31 | 32 | {{/each}} 33 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/templateStagingTestData.Handlebars: -------------------------------------------------------------------------------- 1 | {{#each dataObjectMappings}} 2 | -- 3 | -- Test data generation for {{targetDataObject.name}}. 4 | -- 5 | -- Generate x amount of records following the structure of the data object, and insert these. 6 | -- 7 | 8 | {{#replicate 5}} 9 | 10 | INSERT INTO [{{../targetDataObject.dataObjectConnection.extensions.0.value}}].[{{../targetDataObject.dataObjectConnection.extensions.1.value}}].[{{../targetDataObject.name}}] 11 | ( 12 | [{{../../conventions.etlProcessAttribute}}], 13 | [{{../../conventions.eventDateTimeAttribute}}], 14 | --[{{../../conventions.recordSourceAttribute}}], 15 | [{{../../conventions.changeDataCaptureAttribute}}], 16 | [{{../../conventions.recordChecksumAttribute}}], 17 | {{#each ../dataItemMappings}} 18 | [{{sourceDataItems.0.name}}]{{#unless @last}},{{/unless}} -- Data Type is {{sourceDataItems.0.dataType}} 19 | {{/each}} 20 | ) 21 | VALUES 22 | ( 23 | -1, 24 | GETDATE(), 25 | --'Test Cases', 26 | 'Insert', 27 | 0x00000000000000000000000000000000, 28 | {{#each ../dataItemMappings}} 29 | {{#stringcompare sourceDataItems.0.dataType "int"}}{{randomnumber 100000}}{{/stringcompare}}{{#stringcompare sourceDataItems.0.dataType "varchar"}}'{{randomstring 10}}'{{/stringcompare}}{{#stringcompare sourceDataItems.0.dataType "date"}}'{{randomdate 2000}}'{{/stringcompare}}{{#unless @last}},{{/unless}} 30 | {{/each}} 31 | ) 32 | {{/replicate}} 33 | 34 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_HUB_KeyNullCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 17; 3 | 4 | {{#each dataObjectMappings}} 5 | {{#if @first}} 6 | -- 7 | -- HUB UT unique key check for {{targetDataObject.name}} 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','ALL',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 'DV', 13 | 14 | 'WITH recordcount 15 | AS (SELECT Count(*) AS record_count 16 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 17 | testcount 18 | AS ( 19 | SELECT Count(1) AS test_count 20 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 21 | WHERE CONVERT(CHAR,HASHBYTES(''SHA1'',UPPER( TRIM(ISNULL(NULL,''NA'')) )),2) = [{{businessKeyDefinitions.0.surrogateKey}}] 22 | ) 23 | 24 | SELECT record_count, 25 | test_count, 26 | CASE 27 | WHEN test_count = 0 THEN ''Pass'' 28 | ELSE ''Fail'' 29 | END AS Status, 30 | Getdate() AS Test_Execution_DateTime, 31 | CURRENT_USER AS Test_Executor 32 | FROM recordcount, 33 | testcount; ' 34 | , 35 | 36 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}], [{{businessKeyDefinitions.0.businessKeyComponentMappings.0.targetDataItem.name}}], [OMD_INSERT_MODULE_INSTANCE_ID] 37 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 38 | WHERE CONVERT(CHAR,HASHBYTES(''SHA1'',UPPER( TRIM(ISNULL(NULL,''NA'')) )),2) = [{{businessKeyDefinitions.0.surrogateKey}}]' 39 | 40 | , 'N' 41 | , NULL 42 | 43 | FROM OMD.UT a 44 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 45 | and TARGET_TABLE = '{{targetDataObject.name}}' 46 | and SOURCE_TABLE = 'ALL' 47 | and REALM = 'DV' 48 | WHERE exist.UT_DETAIL_ID is null 49 | 50 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 51 | where [name] = 'UT_DETAIL_ID' 52 | 53 | {{/if}} 54 | 55 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 56 | SELECT DISTINCT 57 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 58 | m.MODULE_ID as MODULE_ID 59 | FROM OMD.UT ut 60 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 61 | and ud.UT_ID = @UT_ID 62 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 63 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 64 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 65 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 66 | 67 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_HUB_KeyTrimCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 16; 3 | 4 | {{#each dataObjectMappings}} 5 | {{#if @first}} 6 | -- 7 | -- HUB UT key trim check for {{targetDataObject.name}} 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','ALL',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 'DV', 13 | 14 | 'WITH recordcount 15 | AS (SELECT Count(*) AS Record_Count 16 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 17 | ), 18 | testcount 19 | AS (SELECT Count(*) AS test_count 20 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 21 | WHERE LEN([{{businessKeyDefinitions.0.businessKeyComponentMappings.0.targetDataItem.name}}]) != LEN(TRIM([{{businessKeyDefinitions.0.businessKeyComponentMappings.0.targetDataItem.name}}])) 22 | ) 23 | SELECT record_count, 24 | test_count, 25 | CASE 26 | WHEN test_count = 0 THEN ''Pass'' 27 | ELSE ''Fail'' 28 | END AS Status, 29 | Getdate() AS Test_Execution_DateTime, 30 | CURRENT_USER AS Test_Executor 31 | FROM recordcount, 32 | testcount; ' 33 | , 34 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}], [{{businessKeyDefinitions.0.businessKeyComponentMappings.0.targetDataItem.name}}], [OMD_INSERT_MODULE_INSTANCE_ID] 35 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 36 | WHERE LEN([{{businessKeyDefinitions.0.businessKeyComponentMappings.0.targetDataItem.name}}]) != LEN(TRIM([{{businessKeyDefinitions.0.businessKeyComponentMappings.0.targetDataItem.name}}]))' 37 | 38 | , 'N' 39 | , NULL 40 | 41 | FROM OMD.UT a 42 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 43 | and TARGET_TABLE = '{{targetDataObject.name}}' 44 | and SOURCE_TABLE = 'ALL' 45 | and REALM = 'DV' 46 | WHERE exist.UT_DETAIL_ID is null 47 | 48 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 49 | where [name] = 'UT_DETAIL_ID' 50 | 51 | {{/if}} 52 | 53 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 54 | SELECT DISTINCT 55 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 56 | m.MODULE_ID as MODULE_ID 57 | FROM OMD.UT ut 58 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 59 | and ud.UT_ID = @UT_ID 60 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 61 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 62 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 63 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 64 | 65 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_Hub_HasSatRecord.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 11; 3 | DECLARE @HUB nvarchar(1000); 4 | {{#each dataObjectMappings}} 5 | 6 | -- 7 | -- HUB has a SAT record UT check for {{targetDataObject.name}} and {{businessKeyDefinitions.0.surrogateKey}} 8 | -- 9 | 10 | SET @HUB = REPLACE('{{businessKeyDefinitions.0.surrogateKey}}','_SK',''); 11 | 12 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 13 | SELECT DISTINCT 14 | @UT_ID,@HUB,'{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 15 | 16 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 17 | WHEN 'STG' 18 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 19 | ELSE 'DV' 20 | END, 21 | 22 | 'WITH recordcount 23 | AS (SELECT Count(*) AS record_count 24 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].['+@HUB+']), 25 | testcount 26 | AS ( 27 | SELECT Count(*) AS test_count 28 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].['+@HUB+'] 29 | WHERE [{{businessKeyDefinitions.0.surrogateKey}}] 30 | NOT IN 31 | ( 32 | SELECT [{{businessKeyDefinitions.0.surrogateKey}}] 33 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 34 | ) 35 | ) 36 | SELECT record_count, 37 | test_count, 38 | CASE 39 | WHEN test_count = 0 THEN ''Pass'' 40 | ELSE ''Fail'' 41 | END AS Status, 42 | Getdate() AS Test_Execution_DateTime, 43 | CURRENT_USER AS Test_Executor 44 | FROM recordcount, 45 | testcount; ' 46 | , 47 | 48 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}] AS HUB_SK, [{{../conventions.etlProcessAttribute}}] 49 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].['+@HUB+'] 50 | WHERE [{{businessKeyDefinitions.0.surrogateKey}}] 51 | NOT IN 52 | ( 53 | SELECT [{{businessKeyDefinitions.0.surrogateKey}}] 54 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 55 | )' 56 | 57 | , 'Y' 58 | , 'This test is optional as HUB may contain keys which do not exist in particular SAT (if more than one SAT available for HUB), however all combined SATs must have all keys from the HUB' 59 | 60 | FROM OMD.UT a 61 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 62 | and TARGET_TABLE = @HUB 63 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 64 | WHERE exist.UT_DETAIL_ID is null 65 | 66 | 67 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 68 | where [name] = 'UT_DETAIL_ID' 69 | 70 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 71 | SELECT DISTINCT 72 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 73 | m.MODULE_ID as MODULE_ID 74 | FROM OMD.UT ut 75 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = @HUB 76 | and ud.SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 77 | and ud.UT_ID = @UT_ID 78 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_' + @HUB +'_{{sourceDataObjects.0.name}}' 79 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 80 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 81 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 82 | 83 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_Hub_UniqueKeyCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 10; 3 | 4 | {{#each dataObjectMappings}} 5 | {{#if @first}} 6 | -- 7 | -- HUB UT unique key check for {{targetDataObject.name}} 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','ALL',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 'DV', 13 | 14 | 'WITH recordcount 15 | AS (SELECT Count(*) AS Record_Count 16 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 17 | testcount 18 | AS (SELECT Count(DISTINCT [{{businessKeyDefinitions.0.surrogateKey}}]) AS Test_Count 19 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]) 20 | SELECT record_count, 21 | test_count, 22 | CASE 23 | WHEN record_count = test_count THEN ''Pass'' 24 | ELSE ''Fail'' 25 | END AS Status, 26 | Getdate() AS Test_Execution_DateTime, 27 | CURRENT_USER AS Test_Executor 28 | FROM recordcount, 29 | testcount; ' 30 | , 31 | 32 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}] as HUB_SK, [{{../conventions.etlProcessAttribute}}] 33 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 34 | GROUP BY [{{businessKeyDefinitions.0.surrogateKey}}], [{{../conventions.etlProcessAttribute}}] 35 | HAVING Count(*) > 1 ' 36 | 37 | , 'N' 38 | , NULL 39 | 40 | FROM OMD.UT a 41 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 42 | and TARGET_TABLE = '{{targetDataObject.name}}' 43 | and SOURCE_TABLE = 'ALL' 44 | and REALM = 'DV' 45 | WHERE exist.UT_DETAIL_ID is null 46 | 47 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 48 | where [name] = 'UT_DETAIL_ID' 49 | 50 | {{/if}} 51 | 52 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 53 | SELECT DISTINCT 54 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 55 | m.MODULE_ID as MODULE_ID 56 | FROM OMD.UT ut 57 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 58 | and ud.UT_ID = @UT_ID 59 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 60 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 61 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 62 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 63 | 64 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_LinkSatellite_DuplicateRecordCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 15; 3 | 4 | {{#each dataObjectMappings}} 5 | -- 6 | -- LSAT UT current record check for {{targetDataObject.name}} 7 | -- 8 | 9 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 10 | SELECT DISTINCT 11 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 12 | 13 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 14 | WHEN 'STG' 15 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 16 | ELSE 'DV' 17 | END, 18 | 19 | 'WITH recordcount 20 | AS (SELECT Count(*) AS Record_Count 21 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}]), 22 | 23 | testcount 24 | AS (SELECT Sum(test_count) AS Test_Count 25 | FROM (SELECT Count(DISTINCT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}]) AS test_count 26 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}} 27 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}] 28 | , OMD_HASH_DIFF 29 | HAVING Count(*) > 1) AS a) 30 | 31 | SELECT record_count, 32 | test_count, 33 | CASE 34 | WHEN test_count IS NULL THEN ''Pass'' 35 | ELSE ''Fail'' 36 | END AS STATUS, 37 | Getdate() AS Test_Execution_DateTime, 38 | CURRENT_USER AS Test_Executor 39 | FROM recordcount 40 | LEFT OUTER JOIN testcount 41 | ON 1 = 1; ' 42 | , 43 | 44 | 'SELECT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}] , [{{../conventions.etlProcessAttribute}}] 45 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}} 46 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [{{../conventions.etlProcessAttribute}}] 47 | , OMD_HASH_DIFF 48 | HAVING Count(*) > 1' 49 | 50 | , 'N' 51 | , NULL 52 | 53 | FROM OMD.UT a 54 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 55 | and TARGET_TABLE = '{{targetDataObject.name}}' 56 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 57 | and REALM = substring('{{targetDataObject.name}}',5,charindex('_','{{targetDataObject.name}}',5)-5) 58 | WHERE exist.UT_DETAIL_ID is null 59 | 60 | 61 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 62 | where [name] = 'UT_DETAIL_ID' 63 | 64 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 65 | SELECT DISTINCT 66 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 67 | m.MODULE_ID as MODULE_ID 68 | FROM OMD.UT ut 69 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' and ud.UT_ID = @UT_ID 70 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 71 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 72 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 73 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 74 | 75 | {{/each}} 76 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_LinkSatellite_ReferentialIntegrity.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 27; 3 | {{#each dataObjectMappings}} 4 | -- 5 | -- HUB has a LSAT record UT check for {{targetDataObject.name}} and {{businessKeyDefinitions.0.surrogateKey}} 6 | -- 7 | 8 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 9 | SELECT DISTINCT 10 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 11 | 12 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 13 | WHEN 'STG' 14 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 15 | ELSE 'DV' 16 | END, 17 | 18 | 'WITH recordcount 19 | AS (SELECT Count(*) AS record_count 20 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 21 | testcount 22 | AS ( 23 | SELECT Count(*) AS test_count 24 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 25 | WHERE [{{businessKeyDefinitions.0.surrogateKey}}] 26 | NOT IN 27 | ( 28 | SELECT [{{businessKeyDefinitions.0.surrogateKey}}] 29 | FROM [{{relatedDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{relatedDataObjects.0.name}}] 30 | ) 31 | ) 32 | SELECT record_count, 33 | test_count, 34 | CASE 35 | WHEN test_count = 0 THEN ''Pass'' 36 | ELSE ''Fail'' 37 | END AS Status, 38 | Getdate() AS Test_Execution_DateTime, 39 | CURRENT_USER AS Test_Executor 40 | FROM recordcount, 41 | testcount; ' 42 | , 43 | 44 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}] AS LSAT_SK, [{{../conventions.etlProcessAttribute}}] 45 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 46 | WHERE [{{businessKeyDefinitions.0.surrogateKey}}] 47 | NOT IN 48 | ( 49 | SELECT [{{businessKeyDefinitions.0.surrogateKey}}] 50 | FROM [{{relatedDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{relatedDataObjects.0.name}}] 51 | )' 52 | 53 | , 'N' 54 | , NULL 55 | 56 | FROM OMD.UT a 57 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 58 | and TARGET_TABLE = '{{targetDataObject.name}}' 59 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 60 | WHERE exist.UT_DETAIL_ID is null 61 | 62 | 63 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 64 | where [name] = 'UT_DETAIL_ID' 65 | 66 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 67 | SELECT DISTINCT 68 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 69 | m.MODULE_ID as MODULE_ID 70 | FROM OMD.UT ut 71 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 72 | and ud.SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 73 | and ud.UT_ID = @UT_ID 74 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 75 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 76 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 77 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 78 | 79 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_Link_KeyNullCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 21; 3 | 4 | {{#each dataObjectMappings}} 5 | {{#if @first}} 6 | -- 7 | -- LNK UT unique key check for {{targetDataObject.name}} 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','ALL',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 'DV', 13 | 14 | 'WITH recordcount 15 | AS (SELECT Count(*) AS record_count 16 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 17 | testcount 18 | AS ( 19 | SELECT Count(1) AS test_count 20 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 21 | WHERE CONVERT(CHAR,HASHBYTES(''SHA1'',UPPER( TRIM(ISNULL(NULL,''NA'')) )),2) = [{{businessKeyDefinitions.0.surrogateKey}}] 22 | ) 23 | 24 | SELECT record_count, 25 | test_count, 26 | CASE 27 | WHEN test_count = 0 THEN ''Pass'' 28 | ELSE ''Fail'' 29 | END AS Status, 30 | Getdate() AS Test_Execution_DateTime, 31 | CURRENT_USER AS Test_Executor 32 | FROM recordcount, 33 | testcount; ' 34 | , 35 | 36 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}], [{{businessKeyDefinitions.0.businessKeyComponentMappings.0.targetDataItem.name}}], [{{../conventions.etlProcessAttribute}}] 37 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 38 | WHERE CONVERT(CHAR,HASHBYTES(''SHA1'',UPPER( TRIM(ISNULL(NULL,''NA'')) )),2) = [{{businessKeyDefinitions.0.surrogateKey}}]' 39 | 40 | , 'N' 41 | , NULL 42 | 43 | FROM OMD.UT a 44 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 45 | and TARGET_TABLE = '{{targetDataObject.name}}' 46 | and SOURCE_TABLE = 'ALL' 47 | and REALM = 'DV' 48 | WHERE exist.UT_DETAIL_ID is null 49 | 50 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 51 | where [name] = 'UT_DETAIL_ID' 52 | 53 | {{/if}} 54 | 55 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 56 | SELECT DISTINCT 57 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 58 | m.MODULE_ID as MODULE_ID 59 | FROM OMD.UT ut 60 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 61 | and ud.UT_ID = @UT_ID 62 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 63 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 64 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 65 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 66 | 67 | {{/each}} 68 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_Link_ReferentialIntegrity.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 12; 3 | DECLARE @HUB nvarchar(1000); 4 | 5 | {{#each dataObjectMappings}} 6 | 7 | {{#each businessKeyDefinitions}} 8 | {{#unless @first}} 9 | -- 10 | -- LNK UT SK Generation check for {{../targetDataObject.name}} - {{surrogateKey}} pair 11 | -- 12 | 13 | SET @HUB = REPLACE('{{surrogateKey}}','_SK',''); 14 | 15 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 16 | SELECT DISTINCT 17 | @UT_ID,'{{../targetDataObject.name}}',@HUB,NULL, NULL, '{{../targetDataObject.dataObjectConnection.extensions.1.value}}','{{../targetDataObject.dataObjectConnection.extensions.1.value}}', 18 | 19 | CASE '{{../sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 20 | WHEN 'STG' 21 | THEN substring('{{../sourceDataObjects.0.name}}',5,charindex('_','{{../sourceDataObjects.0.name}}',5)-5) 22 | ELSE 'DV' 23 | END, 24 | 25 | 'WITH recordcount 26 | AS (SELECT Count(*) AS record_count 27 | FROM [{{../targetDataObject.dataObjectConnection.extensions.1.value}}].[{{../targetDataObject.name}}]), 28 | testcount 29 | AS ( 30 | SELECT Count(1) AS test_count 31 | FROM [{{../targetDataObject.dataObjectConnection.extensions.1.value}}].[{{../targetDataObject.name}}] [{{../targetDataObject.dataObjectConnection.extensions.1.value}}] 32 | WHERE {{surrogateKey}} 33 | NOT IN 34 | ( 35 | SELECT [{{surrogateKey}}] 36 | FROM [{{../targetDataObject.dataObjectConnection.extensions.1.value}}].[' + @HUB + '] 37 | ) 38 | ) 39 | SELECT record_count, 40 | test_count, 41 | CASE 42 | WHEN test_count = 0 THEN ''Pass'' 43 | ELSE ''Fail'' 44 | END AS Status, 45 | Getdate() AS Test_Execution_DateTime, 46 | CURRENT_USER AS Test_Executor 47 | FROM recordcount, 48 | testcount; ' 49 | , 50 | 51 | 'SELECT [{{surrogateKey}}], [{{../conventions.etlProcessAttribute}}] 52 | FROM [{{../targetDataObject.dataObjectConnection.extensions.1.value}}].[{{../targetDataObject.name}}] [{{../targetDataObject.dataObjectConnection.extensions.1.value}}] 53 | WHERE {{surrogateKey}} 54 | NOT IN 55 | ( 56 | SELECT [{{surrogateKey}}] 57 | FROM [{{../targetDataObject.dataObjectConnection.extensions.1.value}}].[' + @HUB + '] 58 | )' 59 | 60 | , 'N' 61 | , NULL 62 | 63 | FROM OMD.UT a 64 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 65 | and TARGET_TABLE = '{{../targetDataObject.name}}' 66 | and SOURCE_TABLE = @HUB 67 | WHERE exist.UT_DETAIL_ID is null 68 | 69 | {{/unless}} 70 | {{/each}} 71 | 72 | 73 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 74 | SELECT DISTINCT 75 | ud.UT_DETAIL_ID, 76 | m.MODULE_ID as MODULE_ID 77 | FROM OMD.UT ut 78 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 79 | and ud.UT_ID = @UT_ID 80 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 81 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 82 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 83 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 84 | 85 | {{/each}} 86 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_Link_UniqueKeyCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 13; 3 | 4 | {{#each dataObjectMappings}} 5 | {{#if @first}} 6 | -- 7 | -- LNK UT unique key check for {{targetDataObject.name}} 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','ALL',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 'DV', 13 | 14 | 'WITH recordcount 15 | AS (SELECT Count(*) AS Record_Count 16 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 17 | testcount 18 | AS (SELECT Count(DISTINCT [{{businessKeyDefinitions.0.surrogateKey}}]) AS Test_Count 19 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]) 20 | SELECT record_count, 21 | test_count, 22 | CASE 23 | WHEN record_count = test_count THEN ''Pass'' 24 | ELSE ''Fail'' 25 | END AS Status, 26 | Getdate() AS Test_Execution_DateTime, 27 | CURRENT_USER AS Test_Executor 28 | FROM recordcount, 29 | testcount; ' 30 | , 31 | 32 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}] as LNK_SK, [{{../conventions.etlProcessAttribute}}] 33 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 34 | GROUP BY [{{businessKeyDefinitions.0.surrogateKey}}], [{{../conventions.etlProcessAttribute}}] 35 | HAVING Count(*) > 1 ' 36 | 37 | , 'N' 38 | , NULL 39 | 40 | FROM OMD.UT a 41 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 42 | and TARGET_TABLE = '{{targetDataObject.name}}' 43 | and SOURCE_TABLE = 'ALL' 44 | and REALM = 'DV' 45 | WHERE exist.UT_DETAIL_ID is null 46 | 47 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 48 | where [name] = 'UT_DETAIL_ID' 49 | 50 | {{/if}} 51 | 52 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 53 | SELECT DISTINCT 54 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 55 | m.MODULE_ID as MODULE_ID 56 | FROM OMD.UT ut 57 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 58 | and ud.UT_ID = @UT_ID 59 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 60 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 61 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 62 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 63 | 64 | {{/each}} 65 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_PersistentStaging_DuplicateRecordCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 5; 3 | 4 | {{#each dataObjectMappings}} 5 | -- 6 | -- PSA UT current record check for {{targetDataObject.name}} 7 | -- 8 | 9 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 10 | SELECT DISTINCT 11 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', substring('{{targetDataObject.name}}',5,charindex('_','{{targetDataObject.name}}',5)-5), 12 | 13 | 'WITH recordcount 14 | AS (SELECT Count(*) AS Record_Count 15 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}]), 16 | 17 | testcount 18 | AS (SELECT Sum(test_count) AS Test_Count 19 | FROM (SELECT Count(DISTINCT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}]) AS test_count 20 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}} 21 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}] 22 | , OMD_HASH_DIFF 23 | HAVING Count(*) > 1) AS a) 24 | 25 | SELECT record_count, 26 | test_count, 27 | CASE 28 | WHEN test_count IS NULL THEN ''Pass'' 29 | ELSE ''Fail'' 30 | END AS STATUS, 31 | Getdate() AS Test_Execution_DateTime, 32 | CURRENT_USER AS Test_Executor 33 | FROM recordcount 34 | LEFT OUTER JOIN testcount 35 | ON 1 = 1; ' 36 | , 37 | 38 | 'SELECT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [{{../conventions.etlProcessAttribute}}] 39 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] {{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}} 40 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [{{../conventions.etlProcessAttribute}}] 41 | , OMD_HASH_DIFF 42 | HAVING Count(*) > 1' 43 | 44 | , 'N' 45 | , NULL 46 | 47 | FROM OMD.UT a 48 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 49 | and TARGET_TABLE = '{{targetDataObject.name}}' 50 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 51 | and REALM = substring('{{targetDataObject.name}}',5,charindex('_','{{targetDataObject.name}}',5)-5) 52 | WHERE exist.UT_DETAIL_ID is null 53 | 54 | 55 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 56 | where [name] = 'UT_DETAIL_ID' 57 | 58 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 59 | SELECT DISTINCT 60 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 61 | m.MODULE_ID as MODULE_ID 62 | FROM OMD.UT ut 63 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' and ud.UT_ID = @UT_ID 64 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_150_{{targetDataObject.name}}' 65 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 66 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 67 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 68 | 69 | {{/each}} 70 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_PersistentStaging_MappingCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 26; 3 | 4 | {{#each dataObjectMappings}} 5 | 6 | -- 7 | -- PSA UT Staging to PSA check for {{targetDataObject.name}} - {{sourceDataObjects.0.name}} pair 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 13 | 14 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 15 | WHEN 'STG' 16 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 17 | ELSE 'DV' 18 | END, 19 | 20 | 'WITH recordcount 21 | AS (SELECT Count(*) AS record_count 22 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 23 | 24 | testcount 25 | AS (select count(1) as test_count from ( 26 | SELECT {{#each dataItemMappings}} 27 | [{{sourceDataItems.0.name}}] AS [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 28 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] 29 | EXCEPT 30 | SELECT {{#each dataItemMappings}} 31 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 32 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 33 | ) t 34 | ) 35 | 36 | SELECT record_count, 37 | test_count, 38 | CASE 39 | WHEN test_count = 0 THEN ''Pass'' 40 | ELSE ''Fail'' 41 | END AS STATUS, 42 | Getdate() AS Test_Execution_DateTime, 43 | CURRENT_USER AS Test_Executor 44 | FROM recordcount 45 | , testcount 46 | ' 47 | , 48 | 49 | 'select {{#each dataItemMappings}} 50 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 51 | from ( 52 | SELECT {{#each dataItemMappings}} 53 | [{{sourceDataItems.0.name}}] AS [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 54 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] 55 | EXCEPT 56 | SELECT {{#each dataItemMappings}} 57 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 58 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 59 | ) t' 60 | 61 | , 'N' 62 | , NULL 63 | 64 | FROM OMD.UT a 65 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 66 | and TARGET_TABLE = '{{targetDataObject.name}}' 67 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 68 | and REALM = substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 69 | WHERE exist.UT_DETAIL_ID is null 70 | 71 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 72 | where [name] = 'UT_DETAIL_ID' 73 | 74 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 75 | SELECT DISTINCT 76 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 77 | m.MODULE_ID as MODULE_ID 78 | FROM OMD.UT ut 79 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 80 | and ud.SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 81 | and ud.UT_ID = @UT_ID 82 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_150_{{targetDataObject.name}}' 83 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 84 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 85 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 86 | 87 | {{/each}} 88 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_SAT_StagingMappingCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 6; 3 | 4 | {{#each dataObjectMappings}} 5 | 6 | -- 7 | -- SAT UT Staging to Satellite check for {{targetDataObject.name}} - {{sourceDataObjects.0.name}} pair 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 13 | 14 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 15 | WHEN 'STG' 16 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 17 | ELSE 'DV' 18 | END, 19 | 20 | 'WITH recordcount 21 | AS (SELECT Count(*) AS record_count 22 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 23 | 24 | testcount 25 | AS (select count(1) as test_count from ( 26 | SELECT {{#each dataItemMappings}} 27 | [{{sourceDataItems.0.name}}] AS [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 28 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] 29 | EXCEPT 30 | SELECT {{#each dataItemMappings}} 31 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 32 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 33 | ) t 34 | ) 35 | 36 | SELECT record_count, 37 | test_count, 38 | CASE 39 | WHEN test_count = 0 THEN ''Pass'' 40 | ELSE ''Fail'' 41 | END AS STATUS, 42 | Getdate() AS Test_Execution_DateTime, 43 | CURRENT_USER AS Test_Executor 44 | FROM recordcount 45 | , testcount 46 | ' 47 | , 48 | 49 | 'select {{#each dataItemMappings}} 50 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 51 | from ( 52 | SELECT {{#each dataItemMappings}} 53 | [{{sourceDataItems.0.name}}] AS [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 54 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] 55 | EXCEPT 56 | SELECT {{#each dataItemMappings}} 57 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 58 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 59 | ) t' 60 | 61 | , 'N' 62 | , NULL 63 | 64 | FROM OMD.UT a 65 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 66 | and TARGET_TABLE = '{{targetDataObject.name}}' 67 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 68 | and REALM = substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 69 | WHERE exist.UT_DETAIL_ID is null 70 | 71 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 72 | where [name] = 'UT_DETAIL_ID' 73 | 74 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 75 | SELECT DISTINCT 76 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 77 | m.MODULE_ID as MODULE_ID 78 | FROM OMD.UT ut 79 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 80 | and ud.SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 81 | and ud.UT_ID = @UT_ID 82 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 83 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 84 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 85 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 86 | 87 | {{/each}} 88 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_Satellite_CurrentRecordUniqueCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 7; 3 | 4 | {{#each dataObjectMappings}} 5 | -- 6 | -- SAT UT current record check for {{targetDataObject.name}} 7 | -- 8 | 9 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 10 | SELECT DISTINCT 11 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 12 | 13 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 14 | WHEN 'STG' 15 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 16 | ELSE 'DV' 17 | END, 18 | 19 | 'WITH recordcount 20 | AS (SELECT Count(*) AS Record_Count 21 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}]), 22 | 23 | testcount 24 | AS (SELECT Count(test_count) AS Test_Count 25 | FROM (SELECT Count(*) AS Test_Count 26 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}] 27 | WHERE omd_current_record_indicator = ''Y'' 28 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}] 29 | HAVING Count(*) > 1) AS a) 30 | 31 | SELECT record_count, 32 | test_count, 33 | CASE 34 | WHEN test_count = 0 THEN ''Pass'' 35 | ELSE ''Fail'' 36 | END AS STATUS, 37 | Getdate() AS Test_Execution_DateTime, 38 | CURRENT_USER AS Test_Executor 39 | FROM recordcount 40 | LEFT OUTER JOIN testcount 41 | ON 1 = 1; ' 42 | , 43 | 44 | 'SELECT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [{{../conventions.etlProcessAttribute}}] 45 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}] 46 | WHERE omd_current_record_indicator = ''Y'' 47 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [{{../conventions.etlProcessAttribute}}] 48 | HAVING Count(*) > 1' 49 | 50 | , 'N' 51 | , NULL 52 | 53 | FROM OMD.UT a 54 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 55 | and TARGET_TABLE = '{{targetDataObject.name}}' 56 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 57 | and REALM = substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 58 | WHERE exist.UT_DETAIL_ID is null 59 | 60 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 61 | where [name] = 'UT_DETAIL_ID' 62 | 63 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 64 | SELECT DISTINCT 65 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 66 | m.MODULE_ID as MODULE_ID 67 | FROM OMD.UT ut 68 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' and ud.UT_ID = @UT_ID 69 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 70 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 71 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 72 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 73 | 74 | {{/each}} 75 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_Satellite_DuplicateRecordCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 8; 3 | 4 | {{#each dataObjectMappings}} 5 | -- 6 | -- SAT UT current record check for {{targetDataObject.name}} 7 | -- 8 | 9 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 10 | SELECT DISTINCT 11 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 12 | 13 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 14 | WHEN 'STG' 15 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 16 | ELSE 'DV' 17 | END, 18 | 19 | 'WITH recordcount 20 | AS (SELECT Count(*) AS Record_Count 21 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}]), 22 | 23 | testcount 24 | AS (SELECT Sum(test_count) AS Test_Count 25 | FROM (SELECT Count(DISTINCT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}]) AS test_count 26 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}] 27 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}] 28 | , OMD_HASH_DIFF 29 | HAVING Count(*) > 1) AS a) 30 | 31 | SELECT record_count, 32 | test_count, 33 | CASE 34 | WHEN test_count IS NULL THEN ''Pass'' 35 | ELSE ''Fail'' 36 | END AS STATUS, 37 | Getdate() AS Test_Execution_DateTime, 38 | CURRENT_USER AS Test_Executor 39 | FROM recordcount 40 | LEFT OUTER JOIN testcount 41 | ON 1 = 1; ' 42 | , 43 | 44 | 'SELECT [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [{{../conventions.etlProcessAttribute}}] 45 | FROM [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{Name}}] [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}] 46 | GROUP BY [{{../dataObjectMappings.0.targetDataObject.dataObjectConnection.extensions.1.value}}].[{{businessKeyDefinitions.0.surrogateKey}}], [{{../conventions.etlProcessAttribute}}] 47 | , OMD_HASH_DIFF 48 | HAVING Count(*) > 1' 49 | 50 | , 'N' 51 | , NULL 52 | 53 | FROM OMD.UT a 54 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 55 | and TARGET_TABLE = '{{targetDataObject.name}}' 56 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 57 | and REALM = substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 58 | WHERE exist.UT_DETAIL_ID is null 59 | 60 | 61 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 62 | where [name] = 'UT_DETAIL_ID' 63 | 64 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 65 | SELECT DISTINCT 66 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 67 | m.MODULE_ID as MODULE_ID 68 | FROM OMD.UT ut 69 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' and ud.UT_ID = @UT_ID 70 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 71 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 72 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 73 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 74 | 75 | {{/each}} 76 | -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_Satellite_KeyNullCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 24; 3 | 4 | {{#each dataObjectMappings}} 5 | 6 | -- 7 | -- SAT UT unique key check for {{targetDataObject.name}} - {{sourceDataObjects.0.name}} pair 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 13 | 14 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 15 | WHEN 'STG' 16 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 17 | ELSE 'DV' 18 | END, 19 | 20 | 'WITH recordcount 21 | AS (SELECT Count(*) AS record_count 22 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 23 | testcount 24 | AS ( 25 | SELECT Count(1) AS test_count 26 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 27 | WHERE CONVERT(CHAR,HASHBYTES(''SHA1'',UPPER( TRIM(ISNULL(NULL,''NA'')) )),2) = [{{businessKeyDefinitions.0.surrogateKey}}] 28 | ) 29 | 30 | SELECT record_count, 31 | test_count, 32 | CASE 33 | WHEN test_count = 0 THEN ''Pass'' 34 | ELSE ''Fail'' 35 | END AS Status, 36 | Getdate() AS Test_Execution_DateTime, 37 | CURRENT_USER AS Test_Executor 38 | FROM recordcount, 39 | testcount; ' 40 | , 41 | 42 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}], [{{../conventions.etlProcessAttribute}}] 43 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 44 | WHERE CONVERT(CHAR,HASHBYTES(''SHA1'',UPPER( TRIM(ISNULL(NULL,''NA'')) )),2) = [{{businessKeyDefinitions.0.surrogateKey}}]' 45 | 46 | , 'N' 47 | , NULL 48 | 49 | FROM OMD.UT a 50 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 51 | and TARGET_TABLE = '{{targetDataObject.name}}' 52 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 53 | and REALM = 'DV' 54 | WHERE exist.UT_DETAIL_ID is null 55 | 56 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 57 | where [name] = 'UT_DETAIL_ID' 58 | 59 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 60 | SELECT DISTINCT 61 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 62 | m.MODULE_ID as MODULE_ID 63 | FROM OMD.UT ut 64 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 65 | and ud.SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 66 | and ud.UT_ID = @UT_ID 67 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 68 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 69 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 70 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 71 | 72 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_Satellite_ReferentialIntegrity.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 9; 3 | DECLARE @HUB nvarchar(1000); 4 | {{#each dataObjectMappings}} 5 | 6 | -- 7 | -- HUB has a SAT record UT check for {{targetDataObject.name}} and {{businessKeyDefinitions.0.surrogateKey}} 8 | -- 9 | 10 | SET @HUB = REPLACE('{{businessKeyDefinitions.0.surrogateKey}}','_SK',''); 11 | 12 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 13 | SELECT DISTINCT 14 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 15 | 16 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 17 | WHEN 'STG' 18 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 19 | ELSE 'DV' 20 | END, 21 | 22 | 'WITH recordcount 23 | AS (SELECT Count(*) AS record_count 24 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 25 | testcount 26 | AS ( 27 | SELECT Count(*) AS test_count 28 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 29 | WHERE [{{businessKeyDefinitions.0.surrogateKey}}] 30 | NOT IN 31 | ( 32 | SELECT [{{businessKeyDefinitions.0.surrogateKey}}] 33 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].['+@HUB+'] 34 | ) 35 | ) 36 | SELECT record_count, 37 | test_count, 38 | CASE 39 | WHEN test_count = 0 THEN ''Pass'' 40 | ELSE ''Fail'' 41 | END AS Status, 42 | Getdate() AS Test_Execution_DateTime, 43 | CURRENT_USER AS Test_Executor 44 | FROM recordcount, 45 | testcount; ' 46 | , 47 | 48 | 'SELECT [{{businessKeyDefinitions.0.surrogateKey}}] AS SAT_SK, [{{../conventions.etlProcessAttribute}}] 49 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 50 | WHERE [{{businessKeyDefinitions.0.surrogateKey}}] 51 | NOT IN 52 | ( 53 | SELECT [{{businessKeyDefinitions.0.surrogateKey}}] 54 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].['+@HUB+'] 55 | )' 56 | 57 | , 'N' 58 | , NULL 59 | 60 | FROM OMD.UT a 61 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 62 | and TARGET_TABLE = '{{targetDataObject.name}}' 63 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 64 | WHERE exist.UT_DETAIL_ID is null 65 | 66 | 67 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 68 | where [name] = 'UT_DETAIL_ID' 69 | 70 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 71 | SELECT DISTINCT 72 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 73 | m.MODULE_ID as MODULE_ID 74 | FROM OMD.UT ut 75 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 76 | and ud.SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 77 | and ud.UT_ID = @UT_ID 78 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 79 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 80 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 81 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 82 | 83 | {{/each}} -------------------------------------------------------------------------------- /Virtual_EDW/Templates/template_test_Satellite_StagingMappingCheck.Handlebars: -------------------------------------------------------------------------------- 1 | DECLARE @LAST_UT_DETAIL_ID nvarchar(1000); 2 | DECLARE @UT_ID int = 6; 3 | 4 | {{#each dataObjectMappings}} 5 | 6 | -- 7 | -- SAT UT Staging to Sattelite check for {{targetDataObject.name}} - {{sourceDataObjects.0.name}} pair 8 | -- 9 | 10 | INSERT INTO OMD.UT_DETAIL (UT_ID, TARGET_TABLE, SOURCE_TABLE, TARGET_COLUMN, SOURCE_COLUMN, TARGET_SCHEMA, SOURCE_SCHEMA, REALM, UT_TEST_SQL, UT_DETAIL_SQL, INACTIVE_INDICATOR, NOTES) 11 | SELECT DISTINCT 12 | @UT_ID,'{{targetDataObject.name}}','{{sourceDataObjects.0.name}}',NULL, NULL, '{{targetDataObject.dataObjectConnection.extensions.1.value}}','{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}', 13 | 14 | CASE '{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}' 15 | WHEN 'STG' 16 | THEN substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 17 | ELSE 'DV' 18 | END, 19 | 20 | 'WITH recordcount 21 | AS (SELECT Count(*) AS record_count 22 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}]), 23 | 24 | testcount 25 | AS (select count(1) as test_count from ( 26 | SELECT {{#each dataItemMappings}} 27 | [{{sourceDataItems.0.name}}] AS [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 28 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] 29 | EXCEPT 30 | SELECT {{#each dataItemMappings}} 31 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 32 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 33 | ) t 34 | ) 35 | 36 | SELECT record_count, 37 | test_count, 38 | CASE 39 | WHEN test_count = 0 THEN ''Pass'' 40 | ELSE ''Fail'' 41 | END AS STATUS, 42 | Getdate() AS Test_Execution_DateTime, 43 | CURRENT_USER AS Test_Executor 44 | FROM recordcount 45 | , testcount 46 | ' 47 | , 48 | 49 | 'select {{#each dataItemMappings}} 50 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 51 | from ( 52 | SELECT {{#each dataItemMappings}} 53 | [{{sourceDataItems.0.name}}] AS [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 54 | FROM [{{sourceDataObjects.0.dataObjectConnection.extensions.1.value}}].[{{sourceDataObjects.0.name}}] 55 | EXCEPT 56 | SELECT {{#each dataItemMappings}} 57 | [{{targetDataItem.name}}]{{#unless @last}},{{/unless}}{{/each}} 58 | FROM [{{targetDataObject.dataObjectConnection.extensions.1.value}}].[{{targetDataObject.name}}] 59 | ) t' 60 | 61 | , 'N' 62 | , NULL 63 | 64 | FROM OMD.UT a 65 | LEFT JOIN OMD.UT_DETAIL exist ON exist.UT_ID = @UT_ID 66 | and TARGET_TABLE = '{{targetDataObject.name}}' 67 | and SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 68 | and REALM = substring('{{sourceDataObjects.0.name}}',5,charindex('_','{{sourceDataObjects.0.name}}',5)-5) 69 | WHERE exist.UT_DETAIL_ID is null 70 | 71 | SELECT @LAST_UT_DETAIL_ID = CONVERT(nvarchar(1000), last_value) from sys.identity_columns 72 | where [name] = 'UT_DETAIL_ID' 73 | 74 | INSERT INTO OMD.UT_DETAIL_MODULE (UT_DETAIL_ID, MODULE_ID) 75 | SELECT DISTINCT 76 | @LAST_UT_DETAIL_ID as UT_DETAIL_ID, 77 | m.MODULE_ID as MODULE_ID 78 | FROM OMD.UT ut 79 | INNER JOIN OMD.UT_DETAIL ud ON ud.TARGET_TABLE = '{{targetDataObject.name}}' 80 | and ud.SOURCE_TABLE = '{{sourceDataObjects.0.name}}' 81 | and ud.UT_ID = @UT_ID 82 | INNER JOIN OMD.MODULE m ON m.MODULE_CODE = 'm_200_{{targetDataObject.name}}_{{sourceDataObjects.0.name}}' 83 | LEFT JOIN OMD.UT_DETAIL_MODULE udm ON udm.MODULE_ID = m.MODULE_ID 84 | and udm.UT_DETAIL_ID = ud.UT_DETAIL_ID 85 | WHERE udm.UT_DETAIL_MODULE_ID IS NULL 86 | 87 | {{/each}} 88 | -------------------------------------------------------------------------------- /Virtual_EDW/Virtual_Data_Warehouse.csproj.DotSettings: -------------------------------------------------------------------------------- 1 |  2 | No -------------------------------------------------------------------------------- /Virtual_Enterprise_Data_Warehouse.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.5.33516.290 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Virtual_Data_Warehouse", "Virtual_EDW\Virtual_Data_Warehouse.csproj", "{73971441-2782-43CF-862C-1F636528858A}" 7 | EndProject 8 | Project("{54435603-DBB4-11D2-8724-00A0C9A8B90C}") = "Virtual_Data_Warehouse_Installer", "Virtual_EDW_Installer\Virtual_EDW_Installer.vdproj", "{FFFAA8D4-DD12-4659-A020-1B76795C6491}" 9 | EndProject 10 | Global 11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 12 | CD_ROM|Any CPU = CD_ROM|Any CPU 13 | CD_ROM|x64 = CD_ROM|x64 14 | Debug|Any CPU = Debug|Any CPU 15 | Debug|x64 = Debug|x64 16 | DVD-5|Any CPU = DVD-5|Any CPU 17 | DVD-5|x64 = DVD-5|x64 18 | Release|Any CPU = Release|Any CPU 19 | Release|x64 = Release|x64 20 | SingleImage|Any CPU = SingleImage|Any CPU 21 | SingleImage|x64 = SingleImage|x64 22 | EndGlobalSection 23 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 24 | {73971441-2782-43CF-862C-1F636528858A}.CD_ROM|Any CPU.ActiveCfg = Release|Any CPU 25 | {73971441-2782-43CF-862C-1F636528858A}.CD_ROM|Any CPU.Build.0 = Release|Any CPU 26 | {73971441-2782-43CF-862C-1F636528858A}.CD_ROM|x64.ActiveCfg = Debug|x64 27 | {73971441-2782-43CF-862C-1F636528858A}.CD_ROM|x64.Build.0 = Debug|x64 28 | {73971441-2782-43CF-862C-1F636528858A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 29 | {73971441-2782-43CF-862C-1F636528858A}.Debug|Any CPU.Build.0 = Debug|Any CPU 30 | {73971441-2782-43CF-862C-1F636528858A}.Debug|Any CPU.Deploy.0 = Debug|Any CPU 31 | {73971441-2782-43CF-862C-1F636528858A}.Debug|x64.ActiveCfg = Debug|x64 32 | {73971441-2782-43CF-862C-1F636528858A}.Debug|x64.Build.0 = Debug|x64 33 | {73971441-2782-43CF-862C-1F636528858A}.DVD-5|Any CPU.ActiveCfg = Debug|Any CPU 34 | {73971441-2782-43CF-862C-1F636528858A}.DVD-5|Any CPU.Build.0 = Debug|Any CPU 35 | {73971441-2782-43CF-862C-1F636528858A}.DVD-5|x64.ActiveCfg = Debug|x64 36 | {73971441-2782-43CF-862C-1F636528858A}.DVD-5|x64.Build.0 = Debug|x64 37 | {73971441-2782-43CF-862C-1F636528858A}.Release|Any CPU.ActiveCfg = Release|Any CPU 38 | {73971441-2782-43CF-862C-1F636528858A}.Release|Any CPU.Build.0 = Release|Any CPU 39 | {73971441-2782-43CF-862C-1F636528858A}.Release|x64.ActiveCfg = Release|x64 40 | {73971441-2782-43CF-862C-1F636528858A}.Release|x64.Build.0 = Release|x64 41 | {73971441-2782-43CF-862C-1F636528858A}.SingleImage|Any CPU.ActiveCfg = Release|Any CPU 42 | {73971441-2782-43CF-862C-1F636528858A}.SingleImage|Any CPU.Build.0 = Release|Any CPU 43 | {73971441-2782-43CF-862C-1F636528858A}.SingleImage|x64.ActiveCfg = Debug|x64 44 | {73971441-2782-43CF-862C-1F636528858A}.SingleImage|x64.Build.0 = Debug|x64 45 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.CD_ROM|Any CPU.ActiveCfg = Release 46 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.CD_ROM|x64.ActiveCfg = Debug 47 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.CD_ROM|x64.Build.0 = Debug 48 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.Debug|Any CPU.ActiveCfg = Debug 49 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.Debug|x64.ActiveCfg = Debug 50 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.Debug|x64.Build.0 = Debug 51 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.DVD-5|Any CPU.ActiveCfg = Debug 52 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.DVD-5|x64.ActiveCfg = Debug 53 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.DVD-5|x64.Build.0 = Debug 54 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.Release|Any CPU.ActiveCfg = Release 55 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.Release|x64.ActiveCfg = Release 56 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.Release|x64.Build.0 = Release 57 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.SingleImage|Any CPU.ActiveCfg = Release 58 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.SingleImage|x64.ActiveCfg = Debug 59 | {FFFAA8D4-DD12-4659-A020-1B76795C6491}.SingleImage|x64.Build.0 = Debug 60 | EndGlobalSection 61 | GlobalSection(SolutionProperties) = preSolution 62 | HideSolutionNode = FALSE 63 | EndGlobalSection 64 | GlobalSection(ExtensibilityGlobals) = postSolution 65 | SolutionGuid = {D9FE2AD3-87E8-4C5F-A1DF-5B7A6B0784D2} 66 | EndGlobalSection 67 | EndGlobal 68 | --------------------------------------------------------------------------------