├── .github └── workflows │ └── ControlDB-deployment.yml ├── .gitignore ├── LICENSE ├── README.md ├── devops ├── azure-pipeline-cicd-controldb.yml ├── azure-pipeline-cicd-db.yml └── templates │ ├── build-sqldb.yml │ └── deploy-sqldb.yml ├── elt-framework └── ControlDB │ ├── ControlDB.sln │ ├── ControlDB.sqlproj │ ├── ELT │ ├── Constraints │ │ ├── CC_IngestDefinition_DataMapping.sql │ │ ├── CC_IngestDefinition_MaxIntervalMinutes.sql │ │ ├── CC_IngestDefinition_MaxIntervalNumber.sql │ │ ├── CC_IngestDefinition_SourceStructure.sql │ │ ├── CC_IngestInstance_IngestStatus.sql │ │ ├── CC_L1TransformDefinition_CustomParameters.sql │ │ ├── CC_L1TransformDefinition_OutputDWTableWriteMode.sql │ │ ├── CC_L1TransformDefinition_OutputL1CuratedFileWriteMode.sql │ │ ├── CC_L1TransformInstance_L1TransformStatus.sql │ │ ├── CC_L1TransformInstance_OutputDWTableWriteMode.sql │ │ ├── CC_L1TransformInstance_OutputL1CuratedFileWriteMode.sql │ │ ├── CC_L2TransformDefinition_CustomParameters.sql │ │ ├── CC_L2TransformDefinition_InputType.sql │ │ ├── CC_L2TransformDefinition_MaxIntervalMinutes.sql │ │ ├── CC_L2TransformDefinition_MaxIntervalNumber.sql │ │ ├── CC_L2TransformDefinition_OutputDWTableWriteMode.sql │ │ ├── CC_L2TransformDefinition_OutputL2CuratedFileWriteMode.sql │ │ ├── CC_L2TransformInstance_L2TransformStatus.sql │ │ ├── CC_L2TransformInstance_OutputDWTableWriteMode.sql │ │ ├── CC_L2TransformInstance_OutputL2CuratedFileWriteMode.sql │ │ ├── DC_Attribute_CreatedBy.sql │ │ ├── DC_Attribute_CreatedTimestamp.sql │ │ ├── DC_Attribute_ModifiedBy.sql │ │ ├── DC_Attribute_ModifiedTimestamp.sql │ │ ├── DC_IngestDefinition_CreatedBy.sql │ │ ├── DC_IngestDefinition_CreatedTimestamp.sql │ │ ├── DC_IngestDefinition_DelayL1TransformationFlag.sql │ │ ├── DC_IngestDefinition_DelayL2TransformationFlag.sql │ │ ├── DC_IngestDefinition_MaxRetries.sql │ │ ├── DC_IngestDefinition_ModifiedBy.sql │ │ ├── DC_IngestDefinition_ModifiedTimestamp.sql │ │ ├── DC_IngestDefinition_RunSequence.sql │ │ ├── DC_L1TransformDefinition_CreatedBy.sql │ │ ├── DC_L1TransformDefinition_CreatedTimestamp.sql │ │ ├── DC_L1TransformDefinition_MaxRetries.sql │ │ ├── DC_L1TransformDefinition_ModifiedBy.sql │ │ ├── DC_L1TransformDefinition_ModifiedTimestamp.sql │ │ ├── DC_L2TransformDefinition.RunSequence.sql │ │ ├── DC_L2TransformDefinition_CreatedBy.sql │ │ ├── DC_L2TransformDefinition_CreatedTimestamp.sql │ │ ├── DC_L2TransformDefinition_MaxRetries.sql │ │ ├── DC_L2TransformDefinition_ModifiedBy.sql │ │ ├── DC_L2TransformDefinition_ModifiedTimestamp.sql │ │ └── PK_L1TransformInstance.sql │ ├── Database.sqlproj │ ├── Functions │ │ ├── uf_GetAestDateTime.sql │ │ ├── uf_GetColumnMapping.sql │ │ ├── uf_GetIngestionColumnList.sql │ │ ├── uf_GetSalesforceQuery.sql │ │ └── uf_GetTabularTranslatorMappingJson.sql │ ├── Indexes │ │ ├── UI_IngestDefinition.sql │ │ ├── UI_IngestInstance.sql │ │ ├── UI_L1TransformDefinition.sql │ │ ├── UI_L1TransformInstance.sql │ │ ├── UI_L2TransformDefinition.sql │ │ └── UI_L2TransformInstance.sql │ ├── Keys │ │ ├── FK_Attribute_IngestID.sql │ │ ├── FK_Attribute_L1TransformID.sql │ │ ├── FK_Attribute_L2TransformID.sql │ │ ├── FK_IngestInstance_IngestID.sql │ │ ├── FK_L1TransformDefinition_IngestID.sql │ │ ├── FK_L1TransformInstance_IngestID.sql │ │ ├── FK_L1TransformInstance_L1TransformID.sql │ │ ├── FK_L2TransformDefinition_IngestID.sql │ │ ├── FK_L2TransformInstance_IngestID.sql │ │ ├── FK_L2TransformInstance_L1TransformID.sql │ │ ├── FK_L2TransformInstance_L2TransformID.sql │ │ ├── PK_Attribute.sql │ │ ├── PK_IngestDefinition.sql │ │ ├── PK_IngestInstance.sql │ │ ├── PK_L1TransformDefinition.sql │ │ ├── PK_L2TransformDefinition.sql │ │ └── PK_L2TransformInstance.sql │ ├── Stored Procedures │ │ ├── GetIngestDefinition.sql │ │ ├── GetIngestDefinition_FileDrop.sql │ │ ├── GetRunningTransformInstances.sql │ │ ├── GetTransformDefinition_L1.sql │ │ ├── GetTransformDefinition_L2.sql │ │ ├── GetTransformInstance_L1.sql │ │ ├── GetTransformInstance_L2.sql │ │ ├── InsertIngestInstance.sql │ │ ├── InsertTransformInstance_L1.sql │ │ ├── InsertTransformInstance_L2.sql │ │ ├── UpdateIngestDefinition.sql │ │ ├── UpdateIngestInstance.sql │ │ ├── UpdateTransformDefinition_L2.sql │ │ ├── UpdateTransformInstance_L1.sql │ │ └── UpdateTransformInstance_L2.sql │ ├── Tables │ │ ├── ColumnMapping.sql │ │ ├── IngestDefinition.sql │ │ ├── IngestInstance.sql │ │ ├── L1TransformDefinition.sql │ │ ├── L1TransformInstance.sql │ │ ├── L2TransformDefinition.sql │ │ └── L2TransformInstance.sql │ └── elt.sql │ ├── Post Deployment Script │ ├── Script.PostDeployment.sql │ └── datasources │ │ ├── RestAPI │ │ ├── IngestDefinition │ │ │ ├── AzureRestAPI.sql │ │ │ └── PurviewRestAPI.sql │ │ └── L1TransformDefinition │ │ │ ├── L1T_AzureRestAPI.sql │ │ │ └── L1T_PurviewRestAPI.sql │ │ └── UploadFiles │ │ ├── IngestDefinition │ │ └── CustomFormRecognizerModel.sql │ │ └── L1TransformDefinition │ │ └── L1T_CustomFormRecognizerModel.sql │ └── Pre Deployment Script │ ├── DBRoles.PreDeployment.sql │ └── Script.PreDeployment.sql └── pictures ├── Azure Modern Data Platform - Ingest Definition.png ├── Azure Modern Data Platform - L1 Transform Definition.png ├── Azure Modern Data Platform - L2 Transform Definition.png ├── Azure Modern Data Platform.drawio ├── Azure Modern Data Platform.png ├── ELT_Framework_Dataflow.png ├── Ingestion_Pipeline.png ├── L1Transformation_Pipeline.png └── L2Transformation_Pipeline.png /.github/workflows/ControlDB-deployment.yml: -------------------------------------------------------------------------------- 1 | name: ControlDB Deployment 2 | on: workflow_dispatch 3 | jobs: 4 | release: 5 | runs-on: windows-latest 6 | steps: 7 | - name: Checkout code 8 | uses: actions/checkout@v3 9 | - name: Display checkout files and folders 10 | run: Get-ChildItem ${{github.workspace}} -Recurse 11 | - name: Azure Login 12 | uses: Azure/login@v1 13 | with: 14 | creds: '{"clientId":"${{ secrets.CLIENT_ID }}","clientSecret":"${{ secrets.CLIENT_SECRET }}","subscriptionId":"${{ secrets.SUBSCRIPTION_ID }}","tenantId":"${{ secrets.TENANT_ID }}"}' 15 | - name: Build and Deploy SQL Database 16 | uses: Azure/sql-action@v2 17 | with: 18 | connection-string: ${{secrets.CONTROLDB_CONNECTIONSTRING}} 19 | path: ${{github.workspace}}\elt-framework\ControlDB\ELT\Database.sqlproj 20 | action: 'Publish' 21 | arguments: '/p:DropPermissionsNotInSource=false /p:BlockOnPossibleDataLoss=false /p:DropRoleMembersNotInSource=false /p:DropObjectsNotInSource=false /p:IgnoreColumnOrder=true /p:IgnorePermissions=true' 22 | - name: Azure Logout 23 | run: | 24 | az logout 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Ww][Ii][Nn]32/ 27 | [Aa][Rr][Mm]/ 28 | [Aa][Rr][Mm]64/ 29 | bld/ 30 | [Bb]in/ 31 | [Oo]bj/ 32 | [Ll]og/ 33 | [Ll]ogs/ 34 | 35 | # Visual Studio 2015/2017 cache/options directory 36 | .vs/ 37 | # Uncomment if you have tasks that create the project's static files in wwwroot 38 | #wwwroot/ 39 | 40 | # Visual Studio 2017 auto generated files 41 | Generated\ Files/ 42 | 43 | # MSTest test Results 44 | [Tt]est[Rr]esult*/ 45 | [Bb]uild[Ll]og.* 46 | 47 | # NUnit 48 | *.VisualState.xml 49 | TestResult.xml 50 | nunit-*.xml 51 | 52 | # Build Results of an ATL Project 53 | [Dd]ebugPS/ 54 | [Rr]eleasePS/ 55 | dlldata.c 56 | 57 | # Benchmark Results 58 | BenchmarkDotNet.Artifacts/ 59 | 60 | # .NET Core 61 | project.lock.json 62 | project.fragment.lock.json 63 | artifacts/ 64 | 65 | # ASP.NET Scaffolding 66 | ScaffoldingReadMe.txt 67 | 68 | # StyleCop 69 | StyleCopReport.xml 70 | 71 | # Files built by Visual Studio 72 | *_i.c 73 | *_p.c 74 | *_h.h 75 | *.ilk 76 | *.meta 77 | *.obj 78 | *.iobj 79 | *.pch 80 | *.pdb 81 | *.ipdb 82 | *.pgc 83 | *.pgd 84 | *.rsp 85 | *.sbr 86 | *.tlb 87 | *.tli 88 | *.tlh 89 | *.tmp 90 | *.tmp_proj 91 | *_wpftmp.csproj 92 | *.log 93 | *.tlog 94 | *.vspscc 95 | *.vssscc 96 | .builds 97 | *.pidb 98 | *.svclog 99 | *.scc 100 | 101 | # Chutzpah Test files 102 | _Chutzpah* 103 | 104 | # Visual C++ cache files 105 | ipch/ 106 | *.aps 107 | *.ncb 108 | *.opendb 109 | *.opensdf 110 | *.sdf 111 | *.cachefile 112 | *.VC.db 113 | *.VC.VC.opendb 114 | 115 | # Visual Studio profiler 116 | *.psess 117 | *.vsp 118 | *.vspx 119 | *.sap 120 | 121 | # Visual Studio Trace Files 122 | *.e2e 123 | 124 | # TFS 2012 Local Workspace 125 | $tf/ 126 | 127 | # Guidance Automation Toolkit 128 | *.gpState 129 | 130 | # ReSharper is a .NET coding add-in 131 | _ReSharper*/ 132 | *.[Rr]e[Ss]harper 133 | *.DotSettings.user 134 | 135 | # TeamCity is a build add-in 136 | _TeamCity* 137 | 138 | # DotCover is a Code Coverage Tool 139 | *.dotCover 140 | 141 | # AxoCover is a Code Coverage Tool 142 | .axoCover/* 143 | !.axoCover/settings.json 144 | 145 | # Coverlet is a free, cross platform Code Coverage Tool 146 | coverage*.json 147 | coverage*.xml 148 | coverage*.info 149 | 150 | # Visual Studio code coverage results 151 | *.coverage 152 | *.coveragexml 153 | 154 | # NCrunch 155 | _NCrunch_* 156 | .*crunch*.local.xml 157 | nCrunchTemp_* 158 | 159 | # MightyMoose 160 | *.mm.* 161 | AutoTest.Net/ 162 | 163 | # Web workbench (sass) 164 | .sass-cache/ 165 | 166 | # Installshield output folder 167 | [Ee]xpress/ 168 | 169 | # DocProject is a documentation generator add-in 170 | DocProject/buildhelp/ 171 | DocProject/Help/*.HxT 172 | DocProject/Help/*.HxC 173 | DocProject/Help/*.hhc 174 | DocProject/Help/*.hhk 175 | DocProject/Help/*.hhp 176 | DocProject/Help/Html2 177 | DocProject/Help/html 178 | 179 | # Click-Once directory 180 | publish/ 181 | 182 | # Publish Web Output 183 | *.[Pp]ublish.xml 184 | *.azurePubxml 185 | # Note: Comment the next line if you want to checkin your web deploy settings, 186 | # but database connection strings (with potential passwords) will be unencrypted 187 | *.pubxml 188 | *.publishproj 189 | 190 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 191 | # checkin your Azure Web App publish settings, but sensitive information contained 192 | # in these scripts will be unencrypted 193 | PublishScripts/ 194 | 195 | # NuGet Packages 196 | *.nupkg 197 | # NuGet Symbol Packages 198 | *.snupkg 199 | # The packages folder can be ignored because of Package Restore 200 | **/[Pp]ackages/* 201 | # except build/, which is used as an MSBuild target. 202 | !**/[Pp]ackages/build/ 203 | # Uncomment if necessary however generally it will be regenerated when needed 204 | #!**/[Pp]ackages/repositories.config 205 | # NuGet v3's project.json files produces more ignorable files 206 | *.nuget.props 207 | *.nuget.targets 208 | 209 | # Microsoft Azure Build Output 210 | csx/ 211 | *.build.csdef 212 | 213 | # Microsoft Azure Emulator 214 | ecf/ 215 | rcf/ 216 | 217 | # Windows Store app package directories and files 218 | AppPackages/ 219 | BundleArtifacts/ 220 | Package.StoreAssociation.xml 221 | _pkginfo.txt 222 | *.appx 223 | *.appxbundle 224 | *.appxupload 225 | 226 | # Visual Studio cache files 227 | # files ending in .cache can be ignored 228 | *.[Cc]ache 229 | # but keep track of directories ending in .cache 230 | !?*.[Cc]ache/ 231 | 232 | # Others 233 | ClientBin/ 234 | ~$* 235 | *~ 236 | *.dbmdl 237 | *.dbproj.schemaview 238 | *.jfm 239 | *.pfx 240 | *.publishsettings 241 | orleans.codegen.cs 242 | 243 | # Including strong name files can present a security risk 244 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 245 | #*.snk 246 | 247 | # Since there are multiple workflows, uncomment next line to ignore bower_components 248 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 249 | #bower_components/ 250 | 251 | # RIA/Silverlight projects 252 | Generated_Code/ 253 | 254 | # Backup & report files from converting an old project file 255 | # to a newer Visual Studio version. Backup files are not needed, 256 | # because we have git ;-) 257 | _UpgradeReport_Files/ 258 | Backup*/ 259 | UpgradeLog*.XML 260 | UpgradeLog*.htm 261 | ServiceFabricBackup/ 262 | *.rptproj.bak 263 | 264 | # SQL Server files 265 | *.mdf 266 | *.ldf 267 | *.ndf 268 | 269 | # Business Intelligence projects 270 | *.rdl.data 271 | *.bim.layout 272 | *.bim_*.settings 273 | *.rptproj.rsuser 274 | *- [Bb]ackup.rdl 275 | *- [Bb]ackup ([0-9]).rdl 276 | *- [Bb]ackup ([0-9][0-9]).rdl 277 | 278 | # Microsoft Fakes 279 | FakesAssemblies/ 280 | 281 | # GhostDoc plugin setting file 282 | *.GhostDoc.xml 283 | 284 | # Node.js Tools for Visual Studio 285 | .ntvs_analysis.dat 286 | node_modules/ 287 | 288 | # Visual Studio 6 build log 289 | *.plg 290 | 291 | # Visual Studio 6 workspace options file 292 | *.opt 293 | 294 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 295 | *.vbw 296 | 297 | # Visual Studio 6 auto-generated project file (contains which files were open etc.) 298 | *.vbp 299 | 300 | # Visual Studio 6 workspace and project file (working project files containing files to include in project) 301 | *.dsw 302 | *.dsp 303 | 304 | # Visual Studio 6 technical files 305 | *.ncb 306 | *.aps 307 | 308 | # Visual Studio LightSwitch build output 309 | **/*.HTMLClient/GeneratedArtifacts 310 | **/*.DesktopClient/GeneratedArtifacts 311 | **/*.DesktopClient/ModelManifest.xml 312 | **/*.Server/GeneratedArtifacts 313 | **/*.Server/ModelManifest.xml 314 | _Pvt_Extensions 315 | 316 | # Paket dependency manager 317 | .paket/paket.exe 318 | paket-files/ 319 | 320 | # FAKE - F# Make 321 | .fake/ 322 | 323 | # CodeRush personal settings 324 | .cr/personal 325 | 326 | # Python Tools for Visual Studio (PTVS) 327 | __pycache__/ 328 | *.pyc 329 | 330 | # Cake - Uncomment if you are using it 331 | # tools/** 332 | # !tools/packages.config 333 | 334 | # Tabs Studio 335 | *.tss 336 | 337 | # Telerik's JustMock configuration file 338 | *.jmconfig 339 | 340 | # BizTalk build output 341 | *.btp.cs 342 | *.btm.cs 343 | *.odx.cs 344 | *.xsd.cs 345 | 346 | # OpenCover UI analysis results 347 | OpenCover/ 348 | 349 | # Azure Stream Analytics local run output 350 | ASALocalRun/ 351 | 352 | # MSBuild Binary and Structured Log 353 | *.binlog 354 | 355 | # NVidia Nsight GPU debugger configuration file 356 | *.nvuser 357 | 358 | # MFractors (Xamarin productivity tool) working folder 359 | .mfractor/ 360 | 361 | # Local History for Visual Studio 362 | .localhistory/ 363 | 364 | # Visual Studio History (VSHistory) files 365 | .vshistory/ 366 | 367 | # BeatPulse healthcheck temp database 368 | healthchecksdb 369 | 370 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 371 | MigrationBackup/ 372 | 373 | # Ionide (cross platform F# VS Code tools) working folder 374 | .ionide/ 375 | 376 | # Fody - auto-generated XML schema 377 | FodyWeavers.xsd 378 | 379 | # VS Code files for those working on multiple tools 380 | .vscode/* 381 | !.vscode/settings.json 382 | !.vscode/tasks.json 383 | !.vscode/launch.json 384 | !.vscode/extensions.json 385 | *.code-workspace 386 | 387 | # Local History for Visual Studio Code 388 | .history/ 389 | 390 | # Windows Installer files from build outputs 391 | *.cab 392 | *.msi 393 | *.msix 394 | *.msm 395 | *.msp 396 | 397 | # JetBrains Rider 398 | *.sln.iml 399 | /elt-framework/.azuredatastudio 400 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # elt-framework 2 | The Extract Load Transform (ELT) framework is a metadata-driven orchestration framework designed for modern cloud data platforms. It simplifies ingestion and transformation pipelines, ensuring a consistent development experience and ease of maintenance. The framework supports batch ingestion and has been extensively tested with **[Microsoft Fabric](https://github.com/bennyaustin/fabric-dataplatform)** and Azure managed services like **[Azure Databricks](https://github.com/rorymcmanus87/databricks-dataplatform)** and **[Azure Synapse](https://github.com/bennyaustin/synapse-dataplatform)**. It utilizes an ANSI-compatible control database as the metadata repository. 3 | 4 | ## Key Features: 5 | * **Configurable and Extendable:** Easily adapt the framework to meet specific needs. 6 | * **Data Source Agnostic:** Ingest data from various sources such as databases, Delta Lake, REST API, flat files, JSON, XML, without storing connection strings as metadata. 7 | * **Delta and Full Loads:** Support for both incremental and full data loads. 8 | * **Re-run and Retry Capability:** Automatically handle failures without manual intervention. 9 | * **In-built Audit Tracking:** Track data processing activities with built-in audit capabilities. 10 | * **Extended Audit Capability:** Enhance audit tracking with Azure PaaS services like Diagnostic Logging. 11 | * **Eliminates Manual Data Patching:** Streamline data processing by removing the need for manual interventions. 12 | * **Data Lineage Support:** Maintain data lineage throughout the data lifecycle. 13 | * **Level1 and Level2 Transformations:** Support for one-to-many and many-to-many transformations. 14 | * **On-demand Pipeline and Transformation Management:** Enable or disable pipelines and transformations as needed. 15 | 16 | ## Documentation 17 | Key concepts and configuration metadata explained in detail at **[Wiki](https://github.com/bennyaustin/elt-framework/wiki)** 18 | 19 | ## Getting Started 20 | To get started follow these steps: 21 | 1. **Clone or Fork the Repository**: Start by cloning or forking the repository from [github.com/bennyaustin/elt-framework](https://github.com/bennyaustin/elt-framework). 22 | 23 | 2. **Deploy ControlDB**: The GitHub Action [workflows/ControlDB-deployment.yml](https://github.com/bennyaustin/elt-framework/blob/main/.github/workflows/ControlDB-deployment.yml) executes the workflow to deploy controlDB objects. 24 | 25 | **Pre-Requisites** 26 | * controlDB is already provisioned by IaC process like [07-IaC-Bicep](https://github.com/bennyaustin/fabric-accelerator/wiki/07-IaC-Bicep) or [iac-synapse-dataplatform](https://github.com/bennyaustin/iac-synapse-dataplatform) 27 | * Create [1](https://learn.microsoft.com/en-us/entra/identity-platform/howto-create-service-principal-portal) or reuse an existing Service Principal. Take note of the Application (client) ID and the secret, they will be required later. 28 | * Grant _db_owner_ permission to the Service Principal 29 | ```sql 30 | CREATE USER [] FROM EXTERNAL PROVIDER 31 | GO 32 | EXEC sp_addrolemember 'db_owner', [] 33 | GO 34 | ``` 35 | 36 | This GitHub Action requires the following repository secrets: 37 | 38 | 1. **CLIENT_ID**: Client/Application ID of the Service Principal. 39 | 1. **CLIENT_SECRET**: Service Principal Secret. 40 | 1. **SUBSCRIPTION_ID**: Azure Subscription ID of controlDB 41 | 1. **TENANT_ID**: Entra Tenant ID of controlDB 42 | 1. **CONTROLDB_CONNECTIONSTRING**: controlDB connection string in service principal authentication format. 43 | ``` 44 | Server=;Authentication=Active Directory Service Principal; Encrypt=True;Database=controlDB;User Id=;Password= 45 | ``` 46 | Now, hit the Run Workflow on GitHub action [ControlDB-deployment.yml](https://github.com/bennyaustin/elt-framework/actions/workflows/ControlDB-deployment.yml) to deploy database objects. 47 | 48 | ## Implementation References 49 | * **[Microsoft Fabric data platform using ELT Framework](https://github.com/bennyaustin/fabric-dataplatform)** 50 | * **[Azure Databricks data platform using ELT Framework](https://github.com/bennyaustin/synapse-dataplatform)** 51 | * **[Azure Synapse data platform using ELT Framework](https://github.com/bennyaustin/synapse-dataplatform)** 52 | 53 | ## Collaborate 54 | You can collaborate in various ways, including: 55 | - Pull Requests 56 | - Update/Enrich [Wiki documentation](https://github.com/bennyaustin/elt-framework/wiki) 57 | - Raise [issues](https://github.com/bennyaustin/elt-framework/issues) when you spot one 58 | - Answer questions in the [discussion](https://github.com/bennyaustin/elt-framework/discussions) forum 59 | 60 | Please contact me to be added as a contributor. 61 | 62 | ## Contact 63 | If you have any questions or need support, please contact the maintainer: 64 | - [Benny Austin](https://github.com/bennyaustin) 65 | -------------------------------------------------------------------------------- /devops/azure-pipeline-cicd-controldb.yml: -------------------------------------------------------------------------------- 1 | # Pipeline to build and deploy controlDB for ELT Framework https://github.com/bennyaustin/elt-framework 2 | # Add steps that build, run tests, deploy, and more: 3 | # https://aka.ms/yaml 4 | 5 | trigger: 6 | - main 7 | 8 | pool: 9 | name: Azure Pipelines 10 | demands: 11 | - msbuild 12 | - visualstudio 13 | vmImage: windows-latest 14 | 15 | steps: 16 | - task: VSBuild@1 17 | displayName: Build database project 18 | inputs: 19 | solution: '**\ControlDB.sln' 20 | 21 | - publish: $(System.DefaultWorkingDirectory) 22 | artifact: dacpac 23 | displayName: Publish Dacpac 24 | 25 | - task: AzureKeyVault@2 26 | displayName: Get keyvault secrets 27 | inputs: 28 | connectedServiceName: ba-devops-spn 29 | KeyVaultName: ba-keyvault1 30 | SecretsFilter: ba-sqlserver1-sqlusername,ba-sqlserver1-password 31 | RunAsPreJob: true 32 | - task: SqlAzureDacpacDeployment@1 33 | displayName: Deploy database 34 | inputs: 35 | ConnectedServiceNameARM: ba-devops-spn 36 | AuthenticationType: server 37 | ServerName: ba-sqlserver1.database.windows.net,1433 38 | DatabaseName: ControlDB 39 | SqlUsername: $(ba-sqlserver1-sqlusername) 40 | SqlPassword: $(ba-sqlserver1-password) 41 | DacpacFile: elt-framework\ControlDB\bin\Release\ControlDB.dacpac 42 | IpDetectionMethod: AutoDetect 43 | DeleteFirewallRule: true 44 | AdditionalArguments: /p:DropPermissionsNotInSource=false /p:DropRoleMembersNotInSource=false /p:DropObjectsNotInSource=true /p:IgnoreColumnOrder=true /p:IgnorePermissions=true -------------------------------------------------------------------------------- /devops/azure-pipeline-cicd-db.yml: -------------------------------------------------------------------------------- 1 | # Pipeline to build and deploy controlDB for ELT Framework https://github.com/bennyaustin/elt-framework 2 | # Add steps that build, run tests, deploy, and more: 3 | # https://aka.ms/yaml 4 | 5 | trigger: 6 | branches: 7 | include: 8 | - main 9 | paths: 10 | include: 11 | - elt-framework/ControlDB 12 | 13 | pool: 14 | name: Azure Pipelines 15 | demands: 16 | - msbuild 17 | - visualstudio 18 | vmImage: windows-latest 19 | 20 | stages: 21 | - stage: ci 22 | displayName: Build Stage 23 | jobs: 24 | - template: templates/build-sqldb.yml 25 | parameters: 26 | solutionName: ControlDB 27 | buildPath: elt-framework\ControlDB\bin\Release 28 | 29 | - stage: cd_nonprod 30 | displayName: Deploy Stage - Non Prod 31 | dependsOn: ci 32 | condition: succeeded() 33 | jobs: 34 | - template: templates/deploy-sqldb.yml 35 | parameters: 36 | serviceConnectionName: devops-contoso-spn 37 | keyvaultName: ba-kv01-kn3acb6lw3vr4 38 | buildArtifactName: dacpac 39 | targetDatabaseServer: ba-sql01-kn3acb6lw3vr4.database.windows.net,1433 40 | targetDatabase: controlDB 41 | 42 | - stage: cd_prod 43 | displayName: Deploy Stage - Prod 44 | dependsOn: ci 45 | condition: succeeded() 46 | jobs: 47 | - template: templates/deploy-sqldb.yml 48 | parameters: 49 | serviceConnectionName: devops-contoso-spn 50 | keyvaultName: ba-kv01-o5rium4bfcf6a 51 | buildArtifactName: dacpac 52 | targetDatabaseServer: ba-sql01-o5rium4bfcf6a.database.windows.net,1433 53 | targetDatabase: controlDB -------------------------------------------------------------------------------- /devops/templates/build-sqldb.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | solutionName: 3 | buildPath: 4 | 5 | 6 | jobs: 7 | - job: 8 | displayName: Build SQL DB Job 9 | steps: 10 | - task: VSBuild@1 11 | displayName: Build db solution ${{parameters.solutionName}} 12 | inputs: 13 | solution: '**\${{parameters.solutionName}}.sln' 14 | 15 | - publish: $(System.DefaultWorkingDirectory)/${{parameters.buildPath}} 16 | artifact: dacpac 17 | displayName: Publish ${{parameters.solutionName}} dacpac -------------------------------------------------------------------------------- /devops/templates/deploy-sqldb.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | serviceConnectionName: 3 | keyvaultName: 4 | buildArtifactName: 5 | targetDatabaseServer: 6 | targetDatabase: 7 | 8 | 9 | jobs: 10 | - job: Deploy 11 | displayName: Deploy SQL DB Job 12 | steps: 13 | - task: AzureKeyVault@2 14 | displayName: Get secrets 15 | inputs: 16 | connectedServiceName: ${{parameters.serviceConnectionName}} 17 | KeyVaultName: ${{parameters.keyvaultName}} 18 | SecretsFilter: sqlserver-admin-username,sqlserver-admin-password 19 | RunAsPreJob: true 20 | 21 | - task: DownloadPipelineArtifact@2 22 | displayName: Download dacpac 23 | inputs: 24 | source: current 25 | artifactName: ${{parameters.buildArtifactName}} 26 | path: $(System.DefaultWorkingDirectory) 27 | 28 | - task: SqlAzureDacpacDeployment@1 29 | displayName: Deploy database - ${{parameters.targetDatabase}} 30 | inputs: 31 | ConnectedServiceNameARM: ${{parameters.serviceConnectionName}} 32 | AuthenticationType: server 33 | ServerName: ${{parameters.targetDatabaseServer}} 34 | DatabaseName: ${{parameters.targetDatabase}} 35 | SqlUsername: $(sqlserver-admin-username) 36 | SqlPassword: $(sqlserver-admin-password) 37 | DacpacFile: ${{parameters.targetDatabase}}.dacpac 38 | IpDetectionMethod: AutoDetect 39 | DeleteFirewallRule: true 40 | AdditionalArguments: /p:DropPermissionsNotInSource=false /p:DropRoleMembersNotInSource=false /p:DropObjectsNotInSource=true /p:IgnoreColumnOrder=true /p:IgnorePermissions=true /p:DoNotDropObjectTypes=Users;Credentials;Permissions;RoleMembership;SecurityPolicies;Logins;ServerRoleMembership;ServerRoles -------------------------------------------------------------------------------- /elt-framework/ControlDB/ControlDB.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.2.32602.215 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{00D1A9C2-B5F0-4AF3-8072-F6C62B433612}") = "ControlDB", "ControlDB.sqlproj", "{B4521477-9A6C-4EBC-A067-02EE9F0B9978}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {B4521477-9A6C-4EBC-A067-02EE9F0B9978}.Debug|Any CPU.ActiveCfg = Release|Any CPU 15 | {B4521477-9A6C-4EBC-A067-02EE9F0B9978}.Debug|Any CPU.Build.0 = Release|Any CPU 16 | {B4521477-9A6C-4EBC-A067-02EE9F0B9978}.Debug|Any CPU.Deploy.0 = Release|Any CPU 17 | {B4521477-9A6C-4EBC-A067-02EE9F0B9978}.Release|Any CPU.ActiveCfg = Release|Any CPU 18 | {B4521477-9A6C-4EBC-A067-02EE9F0B9978}.Release|Any CPU.Build.0 = Release|Any CPU 19 | {B4521477-9A6C-4EBC-A067-02EE9F0B9978}.Release|Any CPU.Deploy.0 = Release|Any CPU 20 | EndGlobalSection 21 | GlobalSection(SolutionProperties) = preSolution 22 | HideSolutionNode = FALSE 23 | EndGlobalSection 24 | GlobalSection(ExtensibilityGlobals) = postSolution 25 | SolutionGuid = {A4971EE5-99CC-4196-B086-305A02EF4317} 26 | EndGlobalSection 27 | EndGlobal 28 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ControlDB.sqlproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Debug 5 | AnyCPU 6 | ControlDB 7 | 2.0 8 | 4.1 9 | {b4521477-9a6c-4ebc-a067-02ee9f0b9978} 10 | Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider 11 | Database 12 | 13 | 14 | ControlDB 15 | ControlDB 16 | 1033,CI 17 | BySchemaAndSchemaType 18 | True 19 | v4.7.2 20 | CS 21 | Properties 22 | False 23 | True 24 | True 25 | dbo 26 | 27 | SQL_Latin1_General_CP1_CI_AS 28 | False 29 | 30 | 31 | bin\Release\ 32 | $(MSBuildProjectName).sql 33 | False 34 | pdbonly 35 | true 36 | false 37 | true 38 | prompt 39 | 4 40 | 41 | 42 | bin\Debug\ 43 | $(MSBuildProjectName).sql 44 | false 45 | true 46 | full 47 | false 48 | true 49 | true 50 | prompt 51 | 4 52 | 53 | 54 | False 55 | 56 | 57 | 11.0 58 | 59 | True 60 | 11.0 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | Always 170 | 171 | 172 | Always 173 | 174 | 175 | Always 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | Always 195 | 196 | 197 | Always 198 | 199 | 200 | 201 | 202 | 203 | Always 204 | 205 | 206 | 207 | 208 | 1.0.0 209 | All 210 | 211 | 212 | 213 | 214 | 215 | 216 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_IngestDefinition_DataMapping.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [CC_IngestDefinition_DataMapping] 3 | CHECK (ISJSON([DataMapping]) = 1) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_IngestDefinition_MaxIntervalMinutes.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [CC_IngestDefinition_MaxIntervalMinutes] 3 | CHECK ([MaxIntervalMinutes] IS NULL OR [MaxIntervalMinutes] > 0) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_IngestDefinition_MaxIntervalNumber.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [CC_IngestDefinition_MaxIntervalNumber] 3 | CHECK ([MaxIntervalNumber] IS NULL OR [MaxIntervalNumber] > 0) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_IngestDefinition_SourceStructure.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [CC_IngestDefinition_SourceStructure] 3 | CHECK (ISJSON([SourceStructure]) = 1) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_IngestInstance_IngestStatus.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestInstance] 2 | ADD CONSTRAINT [CC_IngestInstance_IngestStatus] 3 | CHECK ([IngestStatus]='ReRunFailure' OR [IngestStatus]='ReRunSuccess' OR [IngestStatus]='Running' OR [IngestStatus]='Failure' OR [IngestStatus]='Success') -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L1TransformDefinition_CustomParameters.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformDefinition] 2 | ADD CONSTRAINT [CC_L1TransformDefinition_CustomParameters] 3 | CHECK (ISJSON([CustomParameters]) = 1) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L1TransformDefinition_OutputDWTableWriteMode.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformDefinition] 2 | ADD CONSTRAINT [CC_L1TransformDefinition_OutputDWTableWriteMode] 3 | CHECK ([OutputDWTableWriteMode]='append' OR [OutputDWTableWriteMode]='overwrite' OR [OutputDWTableWriteMode]='error' OR [OutputDWTableWriteMode]='errorifexists' OR [OutputDWTableWriteMode]='ignore') 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L1TransformDefinition_OutputL1CuratedFileWriteMode.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformDefinition] 2 | ADD CONSTRAINT [CC_L1TransformDefinition_OutputL1CuratedFileWriteMode] 3 | CHECK ([OutputL1CuratedFileWriteMode] ='append' OR [OutputL1CuratedFileWriteMode] ='overwrite' OR [OutputL1CuratedFileWriteMode] ='ignore' OR [OutputL1CuratedFileWriteMode] = 'error' OR [OutputL1CuratedFileWriteMode]='errorifexists') 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L1TransformInstance_L1TransformStatus.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformInstance] 2 | ADD CONSTRAINT [CC_L1TransformInstance_L1TransformStatus] 3 | CHECK ([L1TransformStatus] ='ReRunFailure' OR [L1TransformStatus] ='ReRunSuccess' OR [L1TransformStatus] ='Running' OR [L1TransformStatus] ='DWUpload' OR [L1TransformStatus]='Failure' OR [L1TransformStatus]='Success') 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L1TransformInstance_OutputDWTableWriteMode.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformInstance] 2 | ADD CONSTRAINT [CC_L1TransformInstance_OutputDWTableWriteMode] 3 | CHECK ([OutputDWTableWriteMode]='append' OR [OutputDWTableWriteMode]='overwrite' OR [OutputDWTableWriteMode]='error' OR [OutputDWTableWriteMode]='errorifexists' OR [OutputDWTableWriteMode]='ignore') 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L1TransformInstance_OutputL1CuratedFileWriteMode.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformInstance] 2 | ADD CONSTRAINT [CC_L1TransformInstance_OutputL1CuratedFileWriteMode] 3 | CHECK ([OutputL1CuratedFileWriteMode] ='append' OR [OutputL1CuratedFileWriteMode] ='overwrite' OR [OutputL1CuratedFileWriteMode] ='ignore' OR [OutputL1CuratedFileWriteMode] = 'error' OR [OutputL1CuratedFileWriteMode]='errorifexists') 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L2TransformDefinition_CustomParameters.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [CC_L2TransformDefinition_CustomParameters] 3 | CHECK (ISJSON([CustomParameters]) = 1) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L2TransformDefinition_InputType.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [CC_L2TransformDefinition_InputType] 3 | CHECK ([InputType] ='Raw' OR [InputType] ='Curated' OR [InputType] ='Datawarehouse') -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L2TransformDefinition_MaxIntervalMinutes.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [CC_L2TransformDefinition_MaxIntervalMinutes] 3 | CHECK ([MaxIntervalMinutes] IS NULL OR [MaxIntervalMinutes] > 0) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L2TransformDefinition_MaxIntervalNumber.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [CC_L2TransformDefinition_MaxIntervalNumber] 3 | CHECK ([MaxIntervalNumber] IS NULL OR [MaxIntervalNumber] > 0) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L2TransformDefinition_OutputDWTableWriteMode.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [CC_L2TransformDefinition_OutputDWTableWriteMode] 3 | CHECK ([OutputDWTableWriteMode]='append' OR [OutputDWTableWriteMode]='overwrite' OR [OutputDWTableWriteMode]='ignore' OR [OutputDWTableWriteMode]='error' OR [OutputDWTableWriteMode]='errorifexists') 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L2TransformDefinition_OutputL2CuratedFileWriteMode.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [CC_L2TransformDefinition_OutputL2CuratedFileWriteMode] 3 | CHECK ([OutputL2CuratedFileWriteMode]='append' OR [OutputL2CuratedFileWriteMode]='overwrite' OR [OutputL2CuratedFileWriteMode]='ignore' OR [OutputL2CuratedFileWriteMode]='error' OR [OutputL2CuratedFileWriteMode]='errorifexists') 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L2TransformInstance_L2TransformStatus.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformInstance] 2 | ADD CONSTRAINT [CC_L2TransformInstance_L2TransformStatus] 3 | CHECK ([L2TransformStatus] ='ReRunFailure' OR [L2TransformStatus] ='ReRunSuccess' OR [L2TransformStatus] ='Running' OR [L2TransformStatus] ='DWUpload' OR [L2TransformStatus]='Failure' OR [L2TransformStatus]='Success') 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L2TransformInstance_OutputDWTableWriteMode.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformInstance] 2 | ADD CONSTRAINT [CC_L2TransformInstance_OutputDWTableWriteMode] 3 | CHECK ([OutputDWTableWriteMode]='append' OR [OutputDWTableWriteMode]= 'overwrite' OR [OutputDWTableWriteMode]= 'ignore' OR [OutputDWTableWriteMode]= 'error' OR [OutputDWTableWriteMode]= 'errorifexists') 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/CC_L2TransformInstance_OutputL2CuratedFileWriteMode.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformInstance] 2 | ADD CONSTRAINT [CC_L2TransformInstance_OutputL2CuratedFileWriteMode] 3 | CHECK ([OutputL2CuratedFileWriteMode]='append' OR [OutputL2CuratedFileWriteMode]= 'overwrite' OR [OutputL2CuratedFileWriteMode]= 'ignore' OR [OutputL2CuratedFileWriteMode]= 'error' OR [OutputL2CuratedFileWriteMode]= 'errorifexists') 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_Attribute_CreatedBy.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[ColumnMapping] 2 | ADD CONSTRAINT [DC_Attribute_CreatedBy] 3 | DEFAULT (suser_sname()) 4 | FOR [CreatedBy] -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_Attribute_CreatedTimestamp.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[ColumnMapping] 2 | ADD CONSTRAINT [DC_Attribute_CreatedTimestamp] 3 | DEFAULT (CONVERT([datetime],(CONVERT([datetimeoffset],getdate()) AT TIME ZONE 'AUS Eastern Standard Time'))) 4 | FOR [CreatedTimestamp] -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_Attribute_ModifiedBy.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[ColumnMapping] 2 | ADD CONSTRAINT [DC_Attribute_ModifiedBy] 3 | DEFAULT (suser_sname()) 4 | FOR [ModifiedBy] -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_Attribute_ModifiedTimestamp.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[ColumnMapping] 2 | ADD CONSTRAINT [DC_Attribute_ModifiedTimestamp] 3 | DEFAULT (CONVERT([datetime],(CONVERT([datetimeoffset],getdate()) AT TIME ZONE 'AUS Eastern Standard Time'))) 4 | FOR [ModifiedTimestamp] -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_CreatedBy.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [DC_IngestDefinition_CreatedBy] 3 | DEFAULT suser_sname() 4 | FOR [CreatedBy] -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_CreatedTimestamp.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [DC_IngestDefinition_CreatedTimestamp] 3 | DEFAULT CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 4 | FOR [CreatedTimestamp] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_DelayL1TransformationFlag.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [DC_IngestDefinition_DelayL1TransformationFlag] 3 | DEFAULT 1 4 | FOR [DelayL1TransformationFlag] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_DelayL2TransformationFlag.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [DC_IngestDefinition_DelayL2TransformationFlag] 3 | DEFAULT 1 4 | FOR [DelayL2TransformationFlag] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_MaxRetries.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [DC_IngestDefinition_MaxRetries] 3 | DEFAULT 3 4 | FOR [MaxRetries] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_ModifiedBy.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [DC_IngestDefinition_ModifiedBy] 3 | DEFAULT suser_sname() 4 | FOR [ModifiedBy] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_ModifiedTimestamp.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [DC_IngestDefinition_ModifiedTimestamp] 3 | DEFAULT CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 4 | FOR [ModifiedTimestamp] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_RunSequence.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [DC_IngestDefinition_RunSequence] 3 | DEFAULT 100 4 | FOR [RunSequence] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_L1TransformDefinition_CreatedBy.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformDefinition] 2 | ADD CONSTRAINT [DC_L1TransformDefinition_CreatedBy] 3 | DEFAULT suser_sname() 4 | FOR [CreatedBy] -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_L1TransformDefinition_CreatedTimestamp.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformDefinition] 2 | ADD CONSTRAINT [DC_L1TransformDefinition_CreatedTimestamp] 3 | DEFAULT CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 4 | FOR [CreatedTimestamp] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_L1TransformDefinition_MaxRetries.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformDefinition] 2 | ADD CONSTRAINT [DC_L1TransformDefinition_MaxRetries] 3 | DEFAULT 3 4 | FOR [MaxRetries] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_L1TransformDefinition_ModifiedBy.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformDefinition] 2 | ADD CONSTRAINT [DC_L1TransformDefinition_ModifiedBy] 3 | DEFAULT suser_sname() 4 | FOR [ModifiedBy] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_L1TransformDefinition_ModifiedTimestamp.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformDefinition] 2 | ADD CONSTRAINT [DC_L1TransformDefinition_ModifiedTimestamp] 3 | DEFAULT CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 4 | FOR [ModifiedTimestamp] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_L2TransformDefinition.RunSequence.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [DC_L2TransformDefinition_RunSequence] 3 | DEFAULT 100 4 | FOR [RunSequence] -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_L2TransformDefinition_CreatedBy.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [DC_L2TransformDefinition_CreatedBy] 3 | DEFAULT suser_sname() 4 | FOR [CreatedBy] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_L2TransformDefinition_CreatedTimestamp.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [DC_L2TransformDefinition_CreatedTimestamp] 3 | DEFAULT CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 4 | FOR [CreatedTimestamp] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_L2TransformDefinition_MaxRetries.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [DC_L2TransformDefinition_MaxRetries] 3 | DEFAULT 3 4 | FOR [MaxRetries] -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_L2TransformDefinition_ModifiedBy.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [DC_L2TransformDefinition_ModifiedBy] 3 | DEFAULT suser_sname() 4 | FOR [ModifiedBy] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/DC_L2TransformDefinition_ModifiedTimestamp.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [DC_L2TransformDefinition_ModifiedTimestamp] 3 | DEFAULT CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 4 | FOR [ModifiedTimestamp] 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Constraints/PK_L1TransformInstance.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformInstance] 2 | ADD CONSTRAINT [PK_L1TransformInstance] 3 | PRIMARY KEY ([L1TransformInstanceID]) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Database.sqlproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | reactions 6 | Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider 7 | 1033, CI 8 | {00000000-0000-0000-0000-000000000000} 9 | 10 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Functions/uf_GetAestDateTime.sql: -------------------------------------------------------------------------------- 1 | CREATE FUNCTION ELT.[uf_GetAestDateTime]() 2 | RETURNS DATETIME 3 | WITH EXECUTE AS CALLER 4 | AS 5 | BEGIN 6 | RETURN CONVERT(datetime,CONVERT(datetimeoffset, getdate()) AT TIME ZONE 'AUS Eastern Standard Time') 7 | END 8 | GO -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Functions/uf_GetColumnMapping.sql: -------------------------------------------------------------------------------- 1 | CREATE FUNCTION [ELT].[uf_GetColumnMapping] 2 | ( 3 | @IngestID int NULL, 4 | @L1TransformID int NULL, 5 | @L2TransformID int NULL 6 | ) 7 | RETURNS varchar(max) 8 | AS BEGIN 9 | DECLARE @mapping NVARCHAR(MAX) 10 | BEGIN 11 | WITH 12 | cte 13 | AS 14 | ( 15 | select 16 | Mapping = CASE 17 | WHEN SourceName IS NULL THEN NULL 18 | ELSE '{"source":{"name":"' + SourceName + '"},"sink":{"name":"' + TargetName + '"}},' 19 | END 20 | from 21 | [ELT].[ColumnMapping] 22 | WHERE 23 | IngestID = @IngestID 24 | or L1TransformID = @L1TransformID 25 | or L2TransformID = @L2TransformID 26 | and ActiveFlag = 1 27 | ) 28 | 29 | SELECT 30 | @mapping = CASE 31 | WHEN STRING_AGG (Mapping, '') IS NULL THEN NULL 32 | ELSE COALESCE(concat('{"type":"TabularTranslator","mappings":[', SUBSTRING(STRING_AGG (Mapping, ''),1,LEN(STRING_AGG (Mapping, ''))-1), ']}'), NULL) 33 | END 34 | FROM cte; 35 | END 36 | 37 | -- Return the result of the function 38 | Return @mapping 39 | 40 | END 41 | GO 42 | 43 | 44 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Functions/uf_GetIngestionColumnList.sql: -------------------------------------------------------------------------------- 1 | CREATE FUNCTION [ELT].[uf_GetIngestionColumnList] 2 | ( 3 | @IngestID INT 4 | ) 5 | 6 | RETURNS varchar(max) 7 | AS BEGIN 8 | DECLARE @Columns NVARCHAR(MAX) 9 | 10 | --Columns 11 | SET @Columns = ( 12 | SELECT 13 | DISTINCT 14 | stuff(( 15 | SELECT ', ' + SourceName + ' as ' + TargetName 16 | FROM 17 | [ELT].[ColumnMapping] 18 | WHERE IngestID = @IngestID 19 | and ActiveFlag = 1 20 | ORDER BY TargetOrdinalPosition ASC 21 | FOR XML PATH('') 22 | ),1,1,'') as Columns 23 | FROM [ELT].[ColumnMapping] 24 | WHERE IngestID = @IngestID 25 | and ActiveFlag = 1 26 | GROUP BY SourceName 27 | ) 28 | SET @Columns = COALESCE(@Columns, ' * ') 29 | 30 | -- Return the columns of the function 31 | RETURN @Columns 32 | END 33 | GO 34 | 35 | 36 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Functions/uf_GetSalesforceQuery.sql: -------------------------------------------------------------------------------- 1 | CREATE FUNCTION [ELT].[uf_GetSalesforceQuery] 2 | ( 3 | @QueryType varchar(20), --SourceQuery|StatQuery 4 | @IngestID INT, 5 | @EntityName varchar(100), 6 | @WatermarkColName varchar(20), 7 | @FromDate datetime2=NULL, 8 | @ToDate datetime2=NULL, 9 | @MaxIntervalMinutes int=NULL 10 | ) 11 | RETURNS varchar(MAX) 12 | AS 13 | BEGIN 14 | Declare @Query varchar(MAX) 15 | DECLARE @MappingExists INT 16 | DECLARE @Columns NVARCHAR(MAX) 17 | Declare @From datetime2 18 | DECLARE @FromStr varchar(27), @ToStr varchar(27) 19 | 20 | SET @From = (CASE 21 | WHEN @FromDate is NULL THEN '1900-01-01 00:00:00' 22 | ELSE @FromDate 23 | END) 24 | 25 | --Datetime Strings 26 | SET @ToStr = FORMAT(CAST(( 27 | CASE 28 | WHEN @ToDate IS NOT NULL 29 | THEN @ToDate 30 | WHEN @ToDate is NULL AND @MaxIntervalMinutes IS NULL 31 | THEN GETDATE() 32 | WHEN @ToDate is NULL AND @MaxIntervalMinutes IS NOT NULL AND DATEADD(DAY,@MaxIntervalMinutes/1440,@From) > GETDATE() 33 | THEN GETDATE() 34 | WHEN @ToDate is NULL AND @MaxIntervalMinutes IS NOT NULL AND DATEADD(DAY,@MaxIntervalMinutes/1440,@From) <= GETDATE() 35 | THEN DATEADD(DAY,@MaxIntervalMinutes/1440,@From) 36 | ELSE GETDATE() 37 | END 38 | ) 39 | as datetime), 'yyyy-MM-ddTHH:mm:ssZ') 40 | 41 | SET @FromStr = (CASE 42 | WHEN @FromDate is NULL THEN '1900-01-01T00:00:00Z' 43 | ELSE FORMAT(CAST(@FromDate as datetime), 'yyyy-MM-ddTHH:mm:ssZ') 44 | END) 45 | 46 | --Set Columns 47 | SET @Columns = ( 48 | SELECT 49 | DISTINCT 50 | STUFF(( 51 | SELECT 52 | ', ' + SourceName 53 | FROM 54 | [ELT].[ColumnMapping] 55 | WHERE IngestID = @IngestID 56 | AND ActiveFlag = 1 57 | ORDER BY TargetOrdinalPosition ASC 58 | FOR XML PATH('') 59 | ),1,1,'') AS ColumnList 60 | FROM 61 | [ELT].[ColumnMapping] 62 | WHERE IngestID = @IngestID 63 | and ActiveFlag = 1 64 | GROUP BY SourceName 65 | ) 66 | 67 | --SourceQuery 68 | IF @QueryType ='SourceQuery' 69 | BEGIN 70 | SET @Query = 71 | 'SELECT ' + COALESCE(@Columns, ' * ') 72 | + ' FROM ' + @EntityName 73 | + CASE 74 | WHEN @WatermarkColName IS NOT NULL 75 | THEN ' WHERE ' + @WatermarkColName + ' > ' + @FromStr + ' AND ' + @WatermarkColName + ' <= ' + @ToStr 76 | ELSE '' 77 | END 78 | END 79 | 80 | --StatQuery 81 | IF @QueryType ='StatQuery' 82 | BEGIN 83 | SET @Query = CASE 84 | WHEN @WatermarkColName IS NOT NULL 85 | THEN 'SELECT MIN('+@WatermarkColName+') AS DataFromTimestamp,' + ' MAX('+@WatermarkColName+') AS DataToTimestamp,'+ 'COUNT(*) AS SourceCount' 86 | WHEN @WatermarkColName IS NULL 87 | THEN 'SELECT ''1900-01-01T00:00:00Z'' AS DataFromTimestamp, '''+FORMAT(CAST(GETDATE() as datetime), 'yyyy-MM-ddTHH:mm:ssZ')+''' AS DataToTimestamp, COUNT(*) AS SourceCount ' 88 | END 89 | + ' FROM ' + @EntityName 90 | + CASE 91 | WHEN @WatermarkColName IS NOT NULL 92 | THEN ' WHERE ' + @WatermarkColName + ' > ' + '''' + FORMAT(CAST(@FromStr as datetime), 'yyyy-MM-dd HH:mm:ss') + '''' + ' AND ' + @WatermarkColName + ' <= ' + '''' + FORMAT(CAST(@ToStr as datetime), 'yyyy-MM-dd HH:mm:ss') + '''' 93 | ELSE '' 94 | END 95 | END 96 | 97 | -- Return the result of the function 98 | Return @Query 99 | 100 | END 101 | 102 | 103 | GO 104 | 105 | 106 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Functions/uf_GetTabularTranslatorMappingJson.sql: -------------------------------------------------------------------------------- 1 | CREATE FUNCTION [ELT].[uf_GetTabularTranslatorMappingJson] 2 | ( 3 | @DataMappingJson VARCHAR(MAX) 4 | ) 5 | RETURNS VARCHAR(MAX) 6 | AS 7 | BEGIN 8 | -- Uses standard ADF Explicit Schema Mapping 9 | -- https://docs.microsoft.com/en-us/azure/data-factory/copy-activity-schema-and-type-mapping 10 | DECLARE @TabularTranslatorJson VARCHAR(MAX) = '{ 11 | "type": "TabularTranslator", 12 | "mappings": 13 | }' 14 | RETURN REPLACE(@TabularTranslatorJson, '', @DataMappingJson) 15 | END 16 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Indexes/UI_IngestDefinition.sql: -------------------------------------------------------------------------------- 1 | CREATE UNIQUE INDEX [UI_IngestDefinition] 2 | ON [ELT].[IngestDefinition] 3 | ([SourceSystemName],[StreamName]) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Indexes/UI_IngestInstance.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX [UI_IngestInstance] 2 | ON [ELT].[IngestInstance] 3 | ([DestinationRawFileSystem],[DestinationRawFolder],[DestinationRawFile]) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Indexes/UI_L1TransformDefinition.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX [UI_L1TransformDefinition] 2 | ON [ELT].[L1TransformDefinition] 3 | ([InputRawFileSystem],[InputRawFileFolder],[InputRawFile],[OutputL1CurateFileSystem],[OutputL1CuratedFolder],[OutputL1CuratedFile]) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Indexes/UI_L1TransformInstance.sql: -------------------------------------------------------------------------------- 1 | CREATE UNIQUE INDEX [UI_L1TransformInstance] 2 | ON [ELT].[L1TransformInstance] 3 | ([InputRawFileSystem],[InputRawFileFolder],[InputRawFile],[OutputL1CurateFileSystem],[OutputL1CuratedFolder],[OutputL1CuratedFile]) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Indexes/UI_L2TransformDefinition.sql: -------------------------------------------------------------------------------- 1 | CREATE UNIQUE INDEX [UI_L2TransformDefinition] 2 | ON [ELT].[L2TransformDefinition] 3 | ([InputFileSystem],[InputFileFolder],[InputFile],[InputDWTable],[OutputL2CurateFileSystem],[OutputL2CuratedFolder],[OutputL2CuratedFile]) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Indexes/UI_L2TransformInstance.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX [UI_L2TransformInstance] 2 | ON [ELT].[L2TransformInstance] 3 | ([InputFileSystem],[InputFileFolder],[InputFile],[InputDWTable],[OutputL2CurateFileSystem],[OutputL2CuratedFolder],[OutputL2CuratedFile]) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/FK_Attribute_IngestID.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[ColumnMapping] 2 | ADD CONSTRAINT [FK_Attribute_IngestID] 3 | FOREIGN KEY([IngestID]) 4 | REFERENCES [ELT].[IngestDefinition] ([IngestID]) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/FK_Attribute_L1TransformID.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[ColumnMapping] 2 | ADD CONSTRAINT [FK_Attribute_L1TransformID] 3 | FOREIGN KEY([L1TransformID]) 4 | REFERENCES [ELT].[L1TransformDefinition] ([L1TransformID]) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/FK_Attribute_L2TransformID.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[ColumnMapping] 2 | ADD CONSTRAINT [FK_Attribute_L2TransformID] 3 | FOREIGN KEY([L2TransformID]) 4 | REFERENCES [ELT].[L2TransformDefinition] ([L2TransformID]) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/FK_IngestInstance_IngestID.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestInstance] 2 | ADD CONSTRAINT [FK_IngestInstance_IngestID] 3 | FOREIGN KEY (IngestID) 4 | REFERENCES [ELT].[IngestDefinition] (IngestID) 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/FK_L1TransformDefinition_IngestID.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformDefinition] 2 | ADD CONSTRAINT [FK_L1TransformDefinition_IngestID] 3 | FOREIGN KEY ([IngestID]) 4 | REFERENCES [ELT].[IngestDefinition] ([IngestID]) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/FK_L1TransformInstance_IngestID.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformInstance] 2 | ADD CONSTRAINT [FK_L1TransformInstance_IngestID] 3 | FOREIGN KEY ([IngestID]) 4 | REFERENCES [ELT].[IngestDefinition] ([IngestID]) 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/FK_L1TransformInstance_L1TransformID.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformInstance] 2 | ADD CONSTRAINT [FK_L1TransformInstance_L1TransformID] 3 | FOREIGN KEY (L1TransformID) 4 | REFERENCES [ELT].[L1TransformDefinition] (L1TransformID) 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/FK_L2TransformDefinition_IngestID.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [FK_L2TransformDefinition_IngestID] 3 | FOREIGN KEY (IngestID) 4 | REFERENCES [ELT].[IngestDefinition] (IngestID) 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/FK_L2TransformInstance_IngestID.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformInstance] 2 | ADD CONSTRAINT [FK_L2TransformInstance_IngestID] 3 | FOREIGN KEY (IngestID) 4 | REFERENCES [ELT].[IngestDefinition] (IngestID) 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/FK_L2TransformInstance_L1TransformID.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformInstance] 2 | ADD CONSTRAINT [FK_L2TransformInstance_L1TransformID] 3 | FOREIGN KEY (L1TransformID) 4 | REFERENCES [ELT].[L1TransformDefinition] (L1TransformID) 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/FK_L2TransformInstance_L2TransformID.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformInstance] 2 | ADD CONSTRAINT [FK_L2TransformInstance_L2TransformID] 3 | FOREIGN KEY (L2TransformID) 4 | REFERENCES [ELT].[L2TransformDefinition] (L2TransformID) 5 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/PK_Attribute.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[ColumnMapping] 2 | ADD CONSTRAINT [PK_Attribute] 3 | PRIMARY KEY CLUSTERED ([MappingID]) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/PK_IngestDefinition.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestDefinition] 2 | ADD CONSTRAINT [PK_IngestDefinition] 3 | PRIMARY KEY ([IngestID]) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/PK_IngestInstance.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[IngestInstance] 2 | ADD CONSTRAINT [PK_IngestInstance] 3 | PRIMARY KEY ([IngestInstanceID]) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/PK_L1TransformDefinition.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L1TransformDefinition] 2 | ADD CONSTRAINT [PK_L1TransformDefinition] 3 | PRIMARY KEY ([L1TransformID]) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/PK_L2TransformDefinition.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformDefinition] 2 | ADD CONSTRAINT [PK_L2TransformDefinition] 3 | PRIMARY KEY (L2TransformID) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Keys/PK_L2TransformInstance.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE [ELT].[L2TransformInstance] 2 | ADD CONSTRAINT [PK_L2TransformInstance] 3 | PRIMARY KEY ([L2TransformInstanceID]) 4 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/GetIngestDefinition.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[GetIngestDefinition] 2 | @SourceSystemName varchar(20), 3 | @StreamName VARCHAR(100) = '%', --Default =All Streams 4 | @MaxIngestInstance INT = 10 5 | 6 | AS 7 | BEGIN 8 | 9 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time'); 10 | 11 | with 12 | cte 13 | as 14 | 15 | ( --Normal Run 16 | SELECT 17 | [IngestID] 18 | ,[SourceSystemName] 19 | ,[StreamName] 20 | ,[Backend] 21 | ,[EntityName] 22 | ,[WatermarkColName] 23 | 24 | --Delta Dates 25 | ,[LastDeltaDate] 26 | ,[DataFromTimestamp] = 27 | CASE 28 | WHEN ([EntityName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL) THEN [LastDeltaDate] 29 | ELSE CAST('1900-01-01' AS DateTime) 30 | END 31 | ,[DataToTimestamp] = 32 | CASE 33 | WHEN ([EntityName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL AND [MaxIntervalMinutes] IS NOT NULL AND datediff_big(minute,[LastDeltaDate],@localdate) > [MaxIntervalMinutes]) THEN DateAdd(minute,[MaxIntervalMinutes],[LastDeltaDate]) 34 | WHEN ([EntityName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL AND [MaxIntervalMinutes] IS NOT NULL AND datediff_big(minute,[LastDeltaDate],@localdate) <= [MaxIntervalMinutes]) THEN CONVERT(VARCHAR(30),@localdate,120) 35 | ELSE CONVERT(VARCHAR(30),@localdate,120) 36 | END 37 | 38 | --Delta Numbers 39 | ,[LastDeltaNumber] 40 | ,[DataFromNumber] = 41 | CASE 42 | WHEN ([EntityName] IS NOT NULL AND [LastDeltaNumber] IS NOT NULL) THEN [LastDeltaNumber] 43 | END 44 | ,[DataToNumber] = 45 | CASE 46 | WHEN ([EntityName] IS NOT NULL AND [LastDeltaNumber] IS NOT NULL) THEN ([LastDeltaNumber] + [MaxIntervalNumber]) 47 | END 48 | 49 | ,[DataFormat] 50 | ,[SourceStructure] 51 | ,[MaxIntervalMinutes] 52 | ,[MaxIntervalNumber] 53 | ,[DataMapping] 54 | ,[RunSequence] 55 | ,[ActiveFlag] 56 | ,[L1TransformationReqdFlag] 57 | ,[L2TransformationReqdFlag] 58 | ,[DelayL1TransformationFlag] 59 | ,[DelayL2TransformationFlag] 60 | ,[DestinationRawFileSystem] 61 | 62 | --Derived Fields 63 | ,[DestinationRawFolder] = 64 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE([DestinationRawFolder] COLLATE SQL_Latin1_General_CP1_CS_AS 65 | ,'YYYY',CAST(Year(COALESCE([LastDeltaDate],@localdate)) as varchar(4))) 66 | ,'MM',Right('0'+ CAST(Month(COALESCE([LastDeltaDate],@localdate)) AS varchar(2)),2)) 67 | ,'DD',Right('0'+Cast(Day(COALESCE([LastDeltaDate],@localdate)) as varchar(2)),2)) 68 | ,'HH',Right('0'+ CAST(DatePart(hh,COALESCE([LastDeltaDate],@localdate)) as varchar(2)),2)) 69 | ,'MI',Right('0'+ CAST(DatePart(mi,COALESCE([LastDeltaDate],@localdate)) as varchar(2)),2)) 70 | ,'SS',Right('0'+ CAST(DatePart(ss,COALESCE([LastDeltaDate],@localdate)) as varchar(2)),2)) 71 | 72 | ,[DestinationRawFile] = 73 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE([DestinationRawFile] COLLATE SQL_Latin1_General_CP1_CS_AS 74 | ,'YYYY',CAST(Year(COALESCE([LastDeltaDate],@localdate)) AS varchar(4))) 75 | ,'MM',Right('0'+ CAST(Month(COALESCE([LastDeltaDate],@localdate)) AS varchar(2)),2)) 76 | ,'DD',Right('0'+Cast(Day(COALESCE([LastDeltaDate],@localdate)) as varchar(2)),2)) 77 | ,'HH',Right('0'+ CAST(DatePart(hh,COALESCE([LastDeltaDate],@localdate)) AS varchar(2)),2)) 78 | ,'MI',Right('0'+ CAST(DatePart(mi,COALESCE([LastDeltaDate],@localdate)) AS varchar(2)),2)) 79 | ,'SS',Right('0'+ CAST(DatePart(ss,COALESCE([LastDeltaDate],@localdate)) AS varchar(2)),2)) 80 | 81 | 82 | --Query 83 | ,SourceSQL = 84 | CASE 85 | -- Customized for simple Purview ATLAS API 86 | WHEN Backend IN ('ATLAS REST API','AZURE REST API') THEN EntityName 87 | 88 | --DEFAULT ANSI SQL for Delta Table 89 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL 90 | THEN 91 | CASE 92 | WHEN datediff_big(minute,[LastDeltaDate],@localdate) > [MaxIntervalMinutes] 93 | THEN 94 | 'SELECT * FROM ' + [EntityName] + ' WHERE ' 95 | + [WatermarkColName] + ' > ' + ''''+CONVERT(VARCHAR(30),[LastDeltaDate],121) +''''+ ' AND ' + [WatermarkColName] + '<=' + ''''+ CONVERT(VARCHAR(30), DATEADD(minute,[MaxIntervalMinutes],[LastDeltaDate]),121) +'''' 96 | ELSE 97 | 'SELECT * FROM ' + [EntityName] + ' WHERE ' 98 | + [WatermarkColName] + ' > ' + ''''+ CONVERT(VARCHAR(30),[LastDeltaDate],121) +''''+ ' AND ' + [WatermarkColName] + '<=' + ''''+ CONVERT(VARCHAR(30), @localdate,120) +'''' 99 | END 100 | --DEFAULT ANSI SQL for Full Table 101 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NULL 102 | THEN 103 | 'SELECT * FROM ' + [EntityName] 104 | --Running Number 105 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaNumber] IS NOT NULL 106 | THEN 'SELECT * FROM ' + [EntityName] + ' WHERE ' 107 | + [WatermarkColName] + ' > ' + ''''+CONVERT(VARCHAR,[LastDeltaNumber]) +'''' + [WatermarkColName] + ' <= ' + ''''+CONVERT(VARCHAR,([LastDeltaNumber] + [MaxIntervalNumber])) +'''' 108 | ELSE NULL 109 | END 110 | 111 | --Stats Query 112 | ,StatSQL = 113 | 114 | CASE 115 | -- Customized for simple Purview ATLAS API 116 | WHEN Backend IN ('ATLAS REST API','AZURE REST API') THEN EntityName 117 | 118 | --DEFAULT ANSI SQL For Delta Table 119 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL 120 | THEN 121 | CASE 122 | WHEN datediff_big(minute,[LastDeltaDate],@localdate) > [MaxIntervalMinutes] 123 | THEN 124 | 'SELECT MIN('+[WatermarkColName]+') AS DataFromTimestamp, MAX('+[WatermarkColName]+') AS DataToTimestamp, count(1) as SourceCount FROM ' + [EntityName] + ' WHERE ' 125 | + [WatermarkColName] + ' > ' + ''''+CONVERT(varchar(30),LastDeltaDate,121)+''''+ ' AND ' + [WatermarkColName] + ' <= ' + ''''+CONVERT(varchar(30), DATEADD(minute,[MaxIntervalMinutes],[LastDeltaDate]),121)+'''' 126 | ELSE 127 | 'SELECT MIN('+[WatermarkColName]+') AS DataFromTimestamp, MAX('+[WatermarkColName]+') AS DataToTimestamp, count(1) as SourceCount FROM ' + [EntityName] + ' WHERE ' 128 | + [WatermarkColName] + ' > ' + ''''+CONVERT(varchar(30),[LastDeltaDate],121) +''''+ ' AND ' + [WatermarkColName] + ' <= ' + ''''+ CONVERT(varchar(30),(@localdate),120)+'''' 129 | END 130 | --Common No Delta 131 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NULL 132 | THEN 'SELECT ''1900-01-01 00:00:00'' AS DataFromTimestamp, '''+CONVERT(VARCHAR(30),ELT.uf_GetAestDateTime(),120)+''' AS DataToTimestamp, COUNT(*) AS SourceCount FROM ' + [EntityName] 133 | --Running Number 134 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaNumber] IS NOT NULL 135 | THEN 'SELECT MIN('+[WatermarkColName]+') AS DataFromTimestamp,' + ' MAX('+[WatermarkColName]+') AS DataToTimestamp,'+ 'COUNT(*) AS SourceCount FROM ' + [EntityName] 136 | + [WatermarkColName] + ' > ' + ''''+CONVERT(VARCHAR,[LastDeltaNumber])+'''' + ' AND ' + [WatermarkColName] + ' <= ' + ''''+CONVERT(VARCHAR,([LastDeltaNumber] + [MaxIntervalNumber]))+'''' 137 | ELSE NULL 138 | END 139 | 140 | , CAST(0 AS BIT) AS [ReloadFlag] 141 | , NULL AS [ADFPipelineRunID] 142 | FROM 143 | [ELT].[IngestDefinition] 144 | WHERE 145 | [SourceSystemName]=@SourceSystemName 146 | AND [StreamName] LIKE COALESCE(@StreamName, [StreamName]) 147 | AND [ActiveFlag]=1 148 | 149 | --ReRun 150 | UNION 151 | SELECT 152 | [ID].[IngestID] 153 | ,[SourceSystemName] 154 | ,[StreamName] 155 | ,[Backend] 156 | ,[EntityName] 157 | ,[WatermarkColName] 158 | ,[LastDeltaDate] 159 | ,II.[DataFromTimestamp] 160 | ,II.[DataToTimestamp] 161 | ,ID.[LastDeltaNumber] 162 | ,II.[DataFromNumber] 163 | ,II.[DataToNumber] 164 | ,[DataFormat] 165 | ,[SourceStructure] 166 | ,ID.[MaxIntervalMinutes] 167 | ,ID.[MaxIntervalNumber] 168 | ,ID.[DataMapping] 169 | ,ID.[RunSequence] 170 | ,[ActiveFlag] 171 | ,[L1TransformationReqdFlag] 172 | ,[L2TransformationReqdFlag] 173 | ,[DelayL1TransformationFlag] 174 | ,[DelayL2TransformationFlag] 175 | ,II.[DestinationRawFileSystem] 176 | ,II.[DestinationRawFolder] 177 | ,II.[DestinationRawFile] 178 | 179 | --Derived Fields 180 | ,SourceSQL = 181 | CASE 182 | -- Customized for simple Purview ATLAS API 183 | WHEN Backend IN ('ATLAS REST API','AZURE REST API') THEN EntityName 184 | --DEFAULT ANSI SQL for Delta Table 185 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL 186 | THEN 'SELECT * FROM ' + [EntityName] + ' WHERE ' 187 | + [WatermarkColName] + ' > ' + ''''+ CONVERT(varchar(30),II.[DataFromTimestamp],121)+''''+ ' AND ' + [WatermarkColName] + ' <= ' + ''''+ CONVERT(varchar(30),II.[DataToTimestamp],121)+'''' 188 | --Common No Delta 189 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NULL 190 | THEN 'SELECT * FROM ' + [EntityName] 191 | --Running Number 192 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaNumber] IS NOT NULL 193 | THEN 'SELECT * FROM ' + [EntityName] + ' WHERE ' 194 | + [WatermarkColName] + ' > ' + ''''+CONVERT(VARCHAR,II.[DataFromNumber])+'''' + ' AND ' + [WatermarkColName] + ' <= ' + ''''+CONVERT(VARCHAR,II.[DataToNumber])+'''' 195 | 196 | ELSE NULL 197 | END 198 | 199 | ,StatSQL = 200 | CASE 201 | -- Customized for simple Purview ATLAS API 202 | WHEN Backend IN ('ATLAS REST API','AZURE REST API') THEN EntityName 203 | 204 | --DEFAULT ANSI SQL for Delta Table 205 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL THEN 206 | 'SELECT MIN('+[WatermarkColName]+') AS DataFromTimestamp, MAX('+[WatermarkColName]+') AS DataToTimestamp, count(1) as SourceCount FROM ' 207 | + [EntityName] + ' WHERE ' + [WatermarkColName] + '>' + ''''+CONVERT(varchar(30),II.DataFromTimestamp,121)+''''+ ' AND ' + [WatermarkColName] + '<='+ ''''+CONVERT(varchar(30),II.[DataToTimestamp],121)+'''' 208 | --Common No Delta 209 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NULL AND [LastDeltaDate] IS NOT NULL 210 | THEN 'SELECT MIN('+[WatermarkColName]+') AS DataFromTimestamp,' + ' MAX('+[WatermarkColName]+') AS DataToTimestamp,'+ 'COUNT(*) AS SourceCount FROM ' + [EntityName] 211 | --Common No Delta 212 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NULL 213 | THEN 'SELECT SELECT ''1900-01-01 00:00:00'' AS DataFromTimestamp, '''+CONVERT(VARCHAR(30),ELT.uf_GetAestDateTime(),120)+''' AS DataToTimestamp, COUNT(*) AS SourceCount FROM ' + [EntityName] 214 | --Running Number 215 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaNumber] IS NOT NULL 216 | THEN 'SELECT MIN('+[WatermarkColName]+') AS DataFromTimestamp,' + ' MAX('+[WatermarkColName]+') AS DataToTimestamp,'+ 'COUNT(*) AS SourceCount FROM ' + [EntityName] 217 | + [WatermarkColName] + ' > ' + ''''+CONVERT(VARCHAR,II.[DataFromNumber]) +'''' + ' AND ' + [WatermarkColName] + ' <= ' + ''''+CONVERT(VARCHAR,II.[DataToNumber])+'''' 218 | ELSE NULL 219 | END 220 | 221 | ,II.[ReloadFlag] 222 | , II.[ADFIngestPipelineRunID] 223 | FROM 224 | [ELT].[IngestDefinition] ID 225 | INNER JOIN [ELT].[IngestInstance] AS II 226 | ON II.[IngestID]= ID.[IngestID] 227 | AND II.[ReloadFlag]=1 228 | AND (II.[IngestStatus] is NULL OR II.[IngestStatus] != 'Running') --Fetch new instances and ignore instances that are currently running 229 | WHERE 230 | ID.[SourceSystemName]=@SourceSystemName 231 | AND ID.[StreamName] LIKE COALESCE(@StreamName, [StreamName]) 232 | AND ID.[ActiveFlag]=1 233 | AND ISNULL(II.RetryCount,0) <= ID.MaxRetries 234 | 235 | ) 236 | SELECT 237 | TOP (@MaxIngestInstance) * 238 | FROM CTE 239 | ORDER BY 240 | [RunSequence] ASC, [DataFromTimestamp] DESC, [DataToTimestamp] DESC 241 | 242 | END -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/GetIngestDefinition_FileDrop.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[GetIngestDefinition_FileDrop] 2 | @SourceSystemName varchar(20), 3 | @StreamName VARCHAR(100), 4 | @SourceFolder VARCHAR(250), --where SourceFolder=Container/Folder 5 | @SourceFile VARCHAR(200) 6 | AS 7 | --/* 8 | --Generic Stored Procedure to be used for File Drop Pipelines instead of the [ELT].[GetIngestDefinition] procedure which is relevant for database ingest pipelines 9 | --*/ 10 | BEGIN 11 | --For Testing 12 | 13 | 14 | DECLARE @IngestID INT 15 | SELECT @IngestID = [IngestID] FROM [ELT].[IngestDefinition] WHERE [SourceSystemName]=@SourceSystemName AND [StreamName]=@StreamName and [ActiveFlag]=1 16 | 17 | --If the Source File already exists (Reload) 18 | IF EXISTS ( SELECT 1 FROM [ELT].[IngestInstance] 19 | WHERE ( 20 | CASE 21 | WHEN [SourceFileDropFileSystem] IS NULL 22 | THEN [SourceFileDropFolder] --In most cases container will be null as it is part of folder name from event trigger 23 | WHEN [SourceFileDropFileSystem] IS NOT NULL AND LEN(RTRIM([SourceFileDropFolder])) =0 24 | THEN [SourceFileDropFolder] 25 | ELSE [SourceFileDropFileSystem] +'/' + [SourceFileDropFolder] 26 | END ) =@SourceFolder 27 | AND [SourceFileDropFile]=@SourceFile) 28 | 29 | BEGIN 30 | SELECT 31 | top 1 32 | II.[IngestID] 33 | , ID.[SourceSystemName] 34 | , ID.[StreamName] 35 | ,ID.[Backend] 36 | ,ID.[DataFormat] 37 | , II.[SourceFileDropFileSystem] 38 | , REPLACE(II.[SourceFileDropFolder],ID.[SourceFileDropFileSystem] +'/','') AS [SourceFileDropFolder] --Remove container name from folder path 39 | , II.[SourceFileDropFile] 40 | , ID.[SourceFileDelimiter] 41 | , ID.[SourceFileHeaderFlag] 42 | , II.[DestinationRawFileSystem] 43 | , II.[DestinationRawFolder] 44 | , II.[DestinationRawFile] 45 | , ID.[DataMapping] 46 | , CAST(1 AS BIT) AS [ReloadFlag] 47 | , ID.[L1TransformationReqdFlag] 48 | , ID.[L2TransformationReqdFlag] 49 | , ID.[DelayL1TransformationFlag] 50 | , ID.[DelayL2TransformationFlag] 51 | , II.[ADFIngestPipelineRunID] 52 | FROM 53 | [ELT].[IngestInstance] AS II 54 | INNER JOIN [ELT].[IngestDefinition] AS ID 55 | ON II.[IngestID] =ID.[IngestID] 56 | AND II.[IngestID]=@IngestID 57 | AND ( 58 | CASE 59 | WHEN II.[SourceFileDropFileSystem] IS NULL 60 | THEN II.[SourceFileDropFolder] --In most cases container will be null as it is part of folder name from event trigger 61 | WHEN II.[SourceFileDropFolder] IS NOT NULL AND LEN(RTRIM(II.[SourceFileDropFileSystem])) =0 62 | THEN II.[SourceFileDropFolder] 63 | ELSE II.[SourceFileDropFileSystem] +'/' + II.[SourceFileDropFolder] 64 | END ) = @SourceFolder 65 | AND II.[SourceFileDropFile]=@SourceFile 66 | ORDER BY [IngestInstanceID] DESC 67 | END 68 | ELSE 69 | --If this is a new file 70 | BEGIN 71 | 72 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 73 | 74 | SELECT 75 | [IngestID] 76 | , [SourceSystemName] 77 | , [StreamName] 78 | ,[Backend] 79 | ,[DataFormat] 80 | , [SourceFileDropFileSystem] 81 | , REPLACE(@SourceFolder,[SourceFileDropFileSystem] +'/','') AS [SourceFileDropFolder] --Remove container name from folder path 82 | , @SourceFile AS [SourceFileDropFile] 83 | , [SourceFileDelimiter] 84 | , [SourceFileHeaderFlag] 85 | , [DestinationRawFileSystem] 86 | , [DestinationRawFolder] = 87 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE([DestinationRawFolder] COLLATE SQL_Latin1_General_CP1_CS_AS 88 | ,'YYYY',CAST(Year(@localdate) as varchar(4))) 89 | ,'MM',Right('0'+ CAST(Month(@localdate) AS varchar(2)),2)) 90 | ,'DD',Right('0'+Cast(Day(@localdate) as varchar(2)),2)) 91 | ,'HH',Right('0'+ CAST(DatePart(hh,@localdate) as varchar(2)),2)) 92 | ,'MI',Right('0'+ CAST(DatePart(mi,@localdate) as varchar(2)),2)) 93 | ,'SS',Right('0'+ CAST(DatePart(ss,@localdate) as varchar(2)),2)) 94 | 95 | , [DestinationRawFile] = 96 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE([DestinationRawFile] COLLATE SQL_Latin1_General_CP1_CS_AS 97 | ,'YYYY',CAST(Year(@localdate) AS varchar(4))) 98 | ,'MM',Right('0'+ CAST(Month(@localdate) AS varchar(2)),2)) 99 | ,'DD',Right('0'+Cast(Day(@localdate) as varchar(2)),2)) 100 | ,'HH',Right('0'+ CAST(DatePart(hh,@localdate) AS varchar(2)),2)) 101 | ,'MI',Right('0'+ CAST(DatePart(mi,@localdate) AS varchar(2)),2)) 102 | ,'SS',Right('0'+ CAST(DatePart(ss,@localdate) AS varchar(2)),2)) 103 | , [DataMapping] 104 | , CAST(0 AS BIT) AS [ReloadFlag] 105 | , [L1TransformationReqdFlag] 106 | , [L2TransformationReqdFlag] 107 | , [DelayL1TransformationFlag] 108 | , [DelayL2TransformationFlag] 109 | , NULL AS [ADFPipelineRunID] 110 | 111 | FROM 112 | [ELT].[IngestDefinition] 113 | WHERE 114 | [IngestID]=@IngestID 115 | END 116 | END 117 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/GetRunningTransformInstances.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[GetRunningTransformInstances] 2 | @timespanHrs int = 18 --Default Timespan 18hrs 3 | AS 4 | --Get the count of transform instances (L1 and L2) still active and running in last @timespanHrs 5 | DECLARE @L1Count INT, @L2Count INT 6 | 7 | IF @timespanHrs IS NULL OR @timespanHrs<=0 8 | SET @timespanHrs=24 9 | 10 | SELECT @L1Count = COUNT(1) 11 | FROM ELT.L1TransformInstance 12 | WHERE L1TransformStatus='Running' 13 | AND DateDiff(hour,L1TransformStartTimestamp,getdate()) <= @timespanHrs 14 | 15 | SELECT @L2Count= COUNT(1) 16 | FROM ELT.L2TransformInstance 17 | WHERE L2TransformStatus='Running' 18 | AND DateDiff(hour,L2TransformStartTimestamp,getdate()) <= @timespanHrs 19 | 20 | SELECT (@L1Count + @L2Count) AS RunningCount 21 | 22 | RETURN 23 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/GetTransformDefinition_L1.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[GetTransformDefinition_L1] 2 | @IngestID int, 3 | @DeltaDate datetime = null 4 | AS 5 | --declare @IngestID int 6 | DECLARE @localdate datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) AT TIME ZONE 'AUS Eastern Standard Time') 7 | DECLARE @CuratedDate datetime 8 | SET @CuratedDate = COALESCE(@DeltaDate,@localdate) 9 | 10 | 11 | SELECT 12 | --PK/FK 13 | TD.[L1TransformID] 14 | , TD.[IngestID] 15 | 16 | 17 | --Databricks 18 | , TD.[ComputeName] 19 | , TD.[ComputePath] 20 | 21 | --Custom 22 | , TD.[CustomParameters] 23 | 24 | --Raw 25 | ,TD.[InputRawFileDelimiter] 26 | ,TD.[InputFileHeaderFlag] 27 | 28 | --Curated File 29 | --Need to test how this performs with a file drop and filedrop reload. 30 | --Possibly use date drop file was dropped? 31 | ,TD.[OutputL1CurateFileSystem] 32 | 33 | ,[OutputL1CuratedFolder] = 34 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(TD.[OutputL1CuratedFolder] COLLATE SQL_Latin1_General_CP1_CS_AS 35 | ,'YYYY',CAST(Year(@CuratedDate) as varchar(4))) 36 | ,'MM',Right('0'+ CAST(Month(@CuratedDate) AS varchar(2)),2)) 37 | ,'DD',Right('0'+Cast(Day(@CuratedDate) as varchar(2)),2)) 38 | ,'HH',Right('0'+ CAST(DatePart(hh,@CuratedDate) as varchar(2)),2)) 39 | ,'MI',Right('0'+ CAST(DatePart(mi,@CuratedDate) as varchar(2)),2)) 40 | ,'SS',Right('0'+ CAST(DatePart(ss,@CuratedDate) as varchar(2)),2)) 41 | 42 | ,[OutputL1CuratedFile] = 43 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(TD.[OutputL1CuratedFile] COLLATE SQL_Latin1_General_CP1_CS_AS 44 | ,'YYYY',CAST(Year(@CuratedDate) as varchar(4))) 45 | ,'MM',Right('0'+ CAST(Month(@CuratedDate) AS varchar(2)),2)) 46 | ,'DD',Right('0'+Cast(Day(@CuratedDate) as varchar(2)),2)) 47 | ,'HH',Right('0'+ CAST(DatePart(hh,@CuratedDate) as varchar(2)),2)) 48 | ,'MI',Right('0'+ CAST(DatePart(mi,@CuratedDate) as varchar(2)),2)) 49 | ,'SS',Right('0'+ CAST(DatePart(ss,@CuratedDate) as varchar(2)),2)) 50 | 51 | , TD.[OutputL1CuratedFileDelimiter] 52 | , TD.[OutputL1CuratedFileFormat] 53 | , TD.[OutputL1CuratedFileWriteMode] 54 | 55 | --SQL 56 | , [LookupColumns] 57 | , TD.[OutputDWStagingTable] 58 | , TD.[OutputDWTable] 59 | , TD.[OutputDWTableWriteMode] 60 | 61 | --Max Retries 62 | , TD.[MaxRetries] 63 | 64 | 65 | FROM 66 | [ELT].[L1TransformDefinition] TD 67 | LEFT JOIN [ELT].[IngestDefinition] ID 68 | ON TD.[IngestID] = ID.[IngestID] 69 | WHERE 70 | TD.[IngestID] = @IngestID and 71 | TD.[ActiveFlag] = 1 72 | and ID.[ActiveFlag] = 1 73 | and ID.[L1TransformationReqdFlag] =1 74 | 75 | GO 76 | 77 | 78 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/GetTransformDefinition_L2.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[GetTransformDefinition_L2] 2 | @IngestID int, 3 | @DeltaDate datetime = null, 4 | @InputType varchar(15) = '%' 5 | AS 6 | --declare @IngestID int 7 | DECLARE @localdate datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 8 | 9 | --Should be using L2DeltaTransformDate, if null then LocalDate 10 | SELECT 11 | --PK/FK 12 | TD.[L2TransformID] 13 | , TD.[IngestID] 14 | , TD.[L1TransformID] 15 | 16 | --Databricks 17 | , TD.[ComputeName] 18 | , TD.[ComputePath] 19 | 20 | --Custom 21 | , TD.[CustomParameters] 22 | 23 | --InputType 24 | ,TD.[InputType] 25 | 26 | --Raw 27 | ,TD.[InputFileSystem] 28 | ,TD.[InputFileFolder] 29 | ,TD.[InputFile] 30 | ,TD.[InputFileDelimiter] 31 | ,TD.[InputFileHeaderFlag] 32 | ,TD.[InputDWTable] 33 | 34 | --Deltas 35 | ,TD.[WatermarkColName] 36 | ,[DataFromTimestamp] = NULL 37 | ,[DataToTimestamp] = NULL 38 | ,[DataFromNumber] = NULL 39 | ,[DataToNumber] = NULL 40 | ,TD.[MaxIntervalMinutes] 41 | ,TD.[MaxIntervalNumber] 42 | 43 | 44 | --Retry 45 | ,TD.[MaxRetries] 46 | 47 | --Curated File 48 | --Need to test how this performs with a file drop and filedrop reload. Possibly use date drop file was dropped? 49 | ,TD.[OutputL2CurateFileSystem] 50 | 51 | ,[OutputL2CuratedFolder] = 52 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(TD.[OutputL2CuratedFolder] COLLATE SQL_Latin1_General_CP1_CS_AS 53 | ,'YYYY',CAST(Year(COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(4))) 54 | ,'MM',Right('0'+ CAST(Month(COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) AS varchar(2)),2)) 55 | ,'DD',Right('0'+Cast(Day(COALESCE(TD.[LastDeltaDate],@localdate)) as varchar(2)),2)) 56 | ,'HH',Right('0'+ CAST(DatePart(hh,COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2)) 57 | ,'MI',Right('0'+ CAST(DatePart(mi,COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2)) 58 | ,'SS',Right('0'+ CAST(DatePart(ss,COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2)) 59 | 60 | ,[OutputL2CuratedFile] = 61 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(TD.[OutputL2CuratedFile] COLLATE SQL_Latin1_General_CP1_CS_AS 62 | ,'YYYY',CAST(Year(COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(4))) 63 | ,'MM',Right('0'+ CAST(Month(COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) AS varchar(2)),2)) 64 | ,'DD',Right('0'+Cast(Day(COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2)) 65 | ,'HH',Right('0'+ CAST(DatePart(hh,COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2)) 66 | ,'MI',Right('0'+ CAST(DatePart(mi,COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2)) 67 | ,'SS',Right('0'+ CAST(DatePart(ss,COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2)) 68 | 69 | , TD.[OutputL2CuratedFileDelimiter] 70 | , TD.[OutputL2CuratedFileFormat] 71 | , TD.[OutputL2CuratedFileWriteMode] 72 | 73 | --SQL 74 | ,TD.[OutputDWStagingTable] 75 | ,TD.[LookupColumns] 76 | ,TD.[OutputDWTable] 77 | ,TD.[OutputDWTableWriteMode] 78 | 79 | 80 | FROM 81 | [ELT].[L2TransformDefinition] TD 82 | LEFT JOIN [ELT].[IngestDefinition] ID 83 | ON TD.[IngestID] = ID.[IngestID] 84 | WHERE 85 | TD.[IngestID] = @IngestID and 86 | TD.[ActiveFlag] = 1 87 | and ID.[ActiveFlag] = 1 88 | and ID.[L2TransformationReqdFlag] =1 89 | and TD.[InputType] like COALESCE(@InputType,TD.[InputType]) 90 | 91 | GO 92 | 93 | 94 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/GetTransformInstance_L1.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[GetTransformInstance_L1] 2 | @SourceSystemName varchar(20), 3 | @StreamName varchar(100) = '%', 4 | @MaxTransformInstance int = 10, 5 | @L1TransformInstanceId INT = NULL, --To fetch all transform instances set Parameter as NULL otherwise provide a specific instance id 6 | @DelayL1TransformationFlag INT=NULL --Pass @DelayL1TransformationFlag=0 to fetch all instances that needs to be transformed in the current pipeline (usually the ingestion pipeline). Pass @DelayL1TransformationFlag=1 to fetch all transformations that are scheduled for a later time. 7 | AS 8 | begin 9 | --Limit Number of Transform Instances to prevent queuing at DWH 10 | SELECT top (@MaxTransformInstance) 11 | L1TI.[L1TransformInstanceID] 12 | , L1TI.[L1TransformID] 13 | , L1TI.[IngestID] 14 | , L1TI.[ComputeName] 15 | , L1TI.[ComputePath] 16 | , L1TI.[CustomParameters] 17 | , L1TI.[InputRawFileSystem] 18 | , L1TI.[InputRawFileFolder] 19 | , L1TI.[InputRawFile] 20 | , L1TI.[InputRawFileDelimiter] 21 | , L1TI.[InputFileHeaderFlag] 22 | , L1TI.[OutputL1CurateFileSystem] 23 | , L1TI.[OutputL1CuratedFolder] 24 | , L1TI.[OutputL1CuratedFile] 25 | , L1TI.[OutputL1CuratedFileDelimiter] 26 | , L1TI.[OutputL1CuratedFileFormat] 27 | , L1TI.[OutputL1CuratedFileWriteMode] 28 | , L1TI.[OutputDWStagingTable] 29 | , L1TI.[LookupColumns] 30 | , L1TI.[OutputDWTable] 31 | , L1TI.[OutputDWTableWriteMode] 32 | , L1TI.[ReRunL1TransformFlag] 33 | , L1TD.[MaxRetries] 34 | , L1TD.[WatermarkColName] 35 | 36 | FROM 37 | [ELT].[L1TransformInstance] as L1TI 38 | LEFT JOIN [ELT].[L1TransformDefinition] as L1TD 39 | ON L1TI.[L1TransformID] = L1TD.[L1TransformID] 40 | LEFT JOIN [ELT].[IngestDefinition] as ID 41 | ON id.[IngestID]= L1TD.[IngestID] 42 | WHERE 43 | ID.[SourceSystemName] =@SourceSystemName 44 | AND ID.[StreamName] like COALESCE(@StreamName,id.[StreamName]) 45 | AND (ID.ActiveFlag = 1 AND L1TD.ActiveFlag = 1) 46 | AND (L1TI.[ActiveFlag]=1 OR L1TI.[ReRunL1TransformFlag]=1) 47 | AND L1TI.[L1TransformInstanceID] = COALESCE(@L1TransformInstanceId, L1TI.[L1TransformInstanceID]) 48 | AND (L1TI.[L1TransformStatus] IS NULL OR L1TI.[L1TransformStatus] NOT IN ('Running','DWUpload')) --Fetch new instances and ignore instances that are currently running 49 | AND ID.[DelayL1TransformationFlag] = COALESCE(@DelayL1TransformationFlag,ID.[DelayL1TransformationFlag]) 50 | AND ISNULL(L1TI.RetryCount,0) <= L1TD.MaxRetries 51 | ORDER BY L1TI.[L1TransformInstanceID] ASC 52 | END -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/GetTransformInstance_L2.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[GetTransformInstance_L2] 2 | @SourceSystemName varchar(20), 3 | @StreamName varchar(100) = '%', 4 | @MaxTransformInstance int = 10, 5 | @L2TransformInstanceId INT = NULL, --To fetch all transform instances set Parameter as NULL otherwise provide a specific instance id 6 | @DelayL2TransformationFlag INT=NULL, --Pass @DelayL2TransformationFlag=0 to fetch all instances that needs to be transformed in the current pipeline (usually the ingestion pipeline). Pass =1 to fetch @DelayL2TransformationFlagh all transformations that are scheduled for a later time. 7 | @InputType varchar(15) = '%', 8 | @L2TransformID INT=0 --To Fetch all transformations for a specific stream pass 0 otherwise provide TransformId prersent in L2Transsformdefinition 9 | AS 10 | begin 11 | --Limit Number of Transform Instances to prevent queuing at DWH 12 | SELECT TOP 13 | (@MaxTransformInstance) 14 | L2TI.[L2TransformInstanceID] 15 | , L2TI.[L2TransformID] 16 | , L2TI.[IngestID] 17 | , L2TI.[L1TransformID] 18 | , L2TI.[ComputePath] 19 | , L2TI.[ComputeName] 20 | , L2TI.[CustomParameters] 21 | , L2TD.[InputType] 22 | , L2TI.[InputFileSystem] 23 | , L2TI.[InputFileFolder] 24 | , L2TI.[InputFile] 25 | , L2TI.[InputFileDelimiter] 26 | , L2TI.[InputFileHeaderFlag] 27 | , L2TI.[InputDWTable] 28 | , L2TI.[WatermarkColName] 29 | , L2TI.[DataFromTimestamp] 30 | , L2TI.[DataToTimestamp] 31 | , L2TI.[DataFromNumber] 32 | , L2TI.[DataToNumber] 33 | , L2TI.[OutputL2CurateFileSystem] 34 | , L2TI.[OutputL2CuratedFolder] 35 | , L2TI.[OutputL2CuratedFile] 36 | , L2TI.[OutputL2CuratedFileDelimiter] 37 | , L2TI.[OutputL2CuratedFileFormat] 38 | , L2TI.[OutputL2CuratedFileWriteMode] 39 | , L2TI.[OutputDWStagingTable] 40 | , L2TI.[LookupColumns] 41 | , L2TI.[OutputDWTable] 42 | , L2TI.[OutputDWTableWriteMode] 43 | , L2TI.[ReRunL2TransformFlag] 44 | , L2TD.[MaxRetries] 45 | 46 | FROM 47 | [ELT].[L2TransformInstance] as L2TI 48 | LEFT JOIN [ELT].[L2TransformDefinition] as L2TD 49 | ON L2TI.[L2TransformID] = L2TD.[L2TransformID] 50 | LEFT JOIN [ELT].[IngestDefinition] as ID 51 | ON id.[IngestID]= L2TD.[IngestID] 52 | WHERE 53 | ID.[SourceSystemName] =@SourceSystemName 54 | AND ID.[StreamName] like COALESCE(@StreamName,id.[StreamName]) 55 | AND (ID.ActiveFlag = 1 AND L2TD.ActiveFlag = 1) 56 | AND (L2TI.[ActiveFlag]=1 OR L2TI.[ReRunL2TransformFlag]=1) 57 | AND L2TI.[L2TransformInstanceID] = COALESCE(@L2TransformInstanceId, L2TI.[L2TransformInstanceID]) 58 | AND (L2TI.[L2TransformStatus] IS NULL OR L2TI.[L2TransformStatus] NOT IN ('Running','DWUpload')) --Fetch new instances and ignore instances that are currently running 59 | AND ID.[DelayL2TransformationFlag] = COALESCE(@DelayL2TransformationFlag,ID.[DelayL2TransformationFlag]) 60 | AND L2TD.[InputType] like COALESCE(@InputType,L2TD.[InputType]) 61 | --AND ISNULL(L2TI.RetryCount,0) <= L2TD.MaxRetries 62 | AND L2TI.[L2TransformID]= (CASE WHEN @L2TransformID=0 then L2TI.[L2TransformID] ELSE @L2TransformID END) 63 | ORDER BY L2TD.RunSequence ASC, L2TI.[L2TransformInstanceID] ASC 64 | END -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/InsertIngestInstance.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[InsertIngestInstance] 2 | @IngestID INT 3 | ,@SourceFileDropFileSystem varchar(50)=null 4 | ,@SourceFileDropFolder varchar(200)=null 5 | ,@SourceFileDropFile varchar(200)=null 6 | ,@DestinationRawFileSystem varchar(50)=null 7 | ,@DestinationRawFolder varchar(200)=null 8 | ,@DestinationRawFile varchar(200)=null 9 | ,@ReloadFlag bit =0 10 | ,@ADFPipelineRunID UniqueIdentifier=null 11 | AS 12 | 13 | BEGIN 14 | 15 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 16 | 17 | 18 | --NOTE:Potential enhancements for lookup. 19 | -- 1. Job to Purge the Instance Tables to Blob after 3 months 20 | -- 2. Use the AdfPipeline Instance Run ID for the lookup, though this needs to be the individual. Lookup Including Index on the lookup columns. 21 | -- 3. Create a Hash Column on DestinationFileSystem/Folder/File and use as the lookup. Including Index on the lookup columns. 22 | 23 | --Normal Run 24 | IF (@ReloadFlag=0 AND NOT EXISTS ( 25 | SELECT 1 26 | FROM [ELT].[IngestInstance] 27 | WHERE [DestinationRawFileSystem] = @DestinationRawFileSystem 28 | AND [DestinationRawFolder] = @DestinationRawFolder 29 | AND [DestinationRawFile] = @DestinationRawFile 30 | ) 31 | ) 32 | BEGIN 33 | INSERT INTO [ELT].[IngestInstance] 34 | ([IngestID] 35 | ,[SourceFileDropFileSystem] 36 | ,[SourceFileDropFolder] 37 | ,[SourceFileDropFile] 38 | ,[DestinationRawFileSystem] 39 | ,[DestinationRawFolder] 40 | ,[DestinationRawFile] 41 | ,[IngestStartTimestamp] 42 | ,[IngestEndTimestamp] 43 | ,[IngestStatus] 44 | ,[RetryCount] 45 | ,[ReloadFlag] 46 | ,[CreatedBy] 47 | ,[CreatedTimestamp] 48 | ,[ModifiedBy] 49 | ,[ModifiedTimestamp] 50 | ,ADFIngestPipelineRunID) 51 | VALUES 52 | (@IngestID 53 | ,@SourceFileDropFileSystem 54 | ,@SourceFileDropFolder 55 | ,@SourceFileDropFile 56 | ,@DestinationRawFileSystem 57 | ,@DestinationRawFolder 58 | ,@DestinationRawFile 59 | ,@localdate 60 | ,NULL 61 | ,'Running' 62 | ,0 63 | ,0 64 | ,suser_sname() 65 | ,@localdate 66 | ,NULL 67 | ,NULL 68 | ,@ADFPipelineRunID) 69 | END 70 | 71 | --Re-load 72 | IF (@ReloadFlag=1 73 | OR EXISTS (SELECT 1 FROM [ELT].[IngestInstance] 74 | WHERE [DestinationRawFileSystem] = @DestinationRawFileSystem 75 | AND [DestinationRawFolder] = @DestinationRawFolder 76 | AND [DestinationRawFile] = @DestinationRawFile) 77 | ) 78 | BEGIN 79 | Update [ELT].[IngestInstance] 80 | SET [IngestStartTimestamp] = @localdate 81 | ,[IngestEndTimestamp] = NULL 82 | ,[SourceCount]=NULL 83 | ,[IngestCount]=NULL 84 | ,[IngestStatus]='Running' 85 | ,[ModifiedBy]=suser_sname() 86 | ,[ModifiedTimestamp] = @localdate 87 | ,ADFIngestPipelineRunID = @ADFPipelineRunID 88 | --Unique Keys 89 | WHERE [DestinationRawFileSystem] = @DestinationRawFileSystem 90 | AND [DestinationRawFolder] = @DestinationRawFolder 91 | AND [DestinationRawFile] = @DestinationRawFile 92 | END 93 | END 94 | 95 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/InsertTransformInstance_L1.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[InsertTransformInstance_L1] 2 | --PK/FK 3 | @L1TransformID int = null, 4 | @IngestInstanceID int=null, 5 | @IngestID int, 6 | 7 | --Databricks Notebook 8 | @ComputeName varchar(100) = null, 9 | @ComputePath varchar(200) = null, 10 | 11 | --Custom 12 | @CustomParameters varchar(max) = null, 13 | 14 | --Input File 15 | @InputRawFileSystem varchar(50) = null, 16 | @InputRawFileFolder varchar(200) = null, 17 | @InputRawFile varchar(200) = null, 18 | @InputRawFileDelimiter char(1) = null, 19 | @InputFileHeaderFlag bit = null, 20 | 21 | --Curated File 22 | @OutputL1CurateFileSystem varchar(50) = null, 23 | @OutputL1CuratedFolder varchar(200) = null, 24 | @OutputL1CuratedFile varchar(200) = null, 25 | @OutputL1CuratedFileDelimiter char(1) = null, 26 | @OutputL1CuratedFileFormat varchar(10) = null, 27 | @OutputL1CuratedFileWriteMode varchar(20) = null, 28 | 29 | --SQL 30 | @OutputDWStagingTable varchar(200) = null, 31 | @LookupColumns varchar(4000) = null, 32 | @OutputDWTable varchar(200) = null, 33 | @OutputDWTableWriteMode varchar(20) = null, 34 | @IngestCount int = null, 35 | 36 | --ADF Pipeline IDs 37 | @IngestADFPipelineRunID uniqueidentifier = null 38 | 39 | 40 | 41 | 42 | AS 43 | BEGIN 44 | 45 | 46 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 47 | 48 | --Check if Transformation records already exists for the input file for same transformation e.g it's a reload 49 | IF NOT EXISTS 50 | ( 51 | SELECT 1 52 | FROM 53 | [ELT].[L1TransformInstance] 54 | WHERE 55 | [IngestID] = @IngestID 56 | AND L1TransformID = @L1TransformID 57 | AND InputRawFileSystem = @InputRawFileSystem 58 | AND InputRawFileFolder = @InputRawFileFolder 59 | AND InputRawFile = @InputRawFile 60 | 61 | ) 62 | 63 | 64 | BEGIN 65 | --If this is a new transformation 66 | INSERT INTO [ELT].[L1TransformInstance] 67 | ( 68 | [L1TransformID] 69 | ,[IngestInstanceID] 70 | ,[IngestID] 71 | ,[ComputeName] 72 | ,[ComputePath] 73 | ,[CustomParameters] 74 | ,[InputRawFileSystem] 75 | ,[InputRawFileFolder] 76 | ,[InputRawFile] 77 | ,[InputRawFileDelimiter] 78 | ,[InputFileHeaderFlag] 79 | ,[OutputL1CurateFileSystem] 80 | ,[OutputL1CuratedFolder] 81 | ,[OutputL1CuratedFile] 82 | ,[OutputL1CuratedFileDelimiter] 83 | ,[OutputL1CuratedFileFormat] 84 | ,[OutputL1CuratedFileWriteMode] 85 | ,[OutputDWStagingTable] 86 | ,[LookupColumns] 87 | ,[OutputDWTable] 88 | ,[OutputDWTableWriteMode] 89 | ,[RetryCount] 90 | ,[ActiveFlag] 91 | ,[IngestCount] 92 | ,[IngestADFPipelineRunID] 93 | ,[CreatedBy] 94 | ,[CreatedTimestamp] 95 | 96 | ) 97 | VALUES 98 | ( 99 | @L1TransformID 100 | ,@IngestInstanceID 101 | ,@IngestID 102 | ,@ComputeName 103 | ,@ComputePath 104 | ,@CustomParameters 105 | ,@InputRawFileSystem 106 | ,@InputRawFileFolder 107 | ,@InputRawFile 108 | ,@InputRawFileDelimiter 109 | ,@InputFileHeaderFlag 110 | ,@OutputL1CurateFileSystem 111 | ,@OutputL1CuratedFolder 112 | ,@OutputL1CuratedFile 113 | ,@OutputL1CuratedFileDelimiter 114 | ,@OutputL1CuratedFileFormat 115 | ,@OutputL1CuratedFileWriteMode 116 | ,@OutputDWStagingTable 117 | ,@LookupColumns 118 | ,@OutputDWTable 119 | ,@OutputDWTableWriteMode 120 | ,0 121 | ,1 122 | ,@IngestCount 123 | ,@IngestADFPipelineRunID 124 | ,SUSER_SNAME() 125 | ,@localdate 126 | ) 127 | END 128 | ELSE 129 | --If this is an existing Transformation 130 | BEGIN 131 | --Just update one record in case if there are duplicates 132 | UPDATE TOP (1) [ELT].[L1TransformInstance] 133 | SET 134 | 135 | [IngestCount] = null 136 | ,[L1TransformInsertCount] = null 137 | ,[L1TransformUpdateCount] = null 138 | ,[L1TransformDeleteCount] = null 139 | ,L1TransformStartTimestamp = null 140 | ,[L1TransformEndTimestamp] = null 141 | ,[L1TransformStatus] = null 142 | ,[RetryCount] = 0 143 | ,[ActiveFlag] = 1 144 | ,[ReRunL1TransformFlag] = 1 145 | ,[IngestADFPipelineRunID] = @IngestADFPipelineRunID 146 | ,[L1TransformADFPipelineRunID] = null 147 | ,[ModifiedBy] = suser_sname() 148 | ,[ModifiedTimestamp] = @localdate 149 | 150 | WHERE 151 | [IngestID] = @IngestID 152 | AND L1TransformID = @L1TransformID 153 | AND InputRawFileSystem = @InputRawFileSystem 154 | AND InputRawFileFolder = @InputRawFileFolder 155 | AND InputRawFile = @InputRawFile 156 | END 157 | END 158 | 159 | 160 | 161 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/InsertTransformInstance_L2.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[InsertTransformInstance_L2] 2 | --PK/FK 3 | @L2TransformID int = null, 4 | @IngestID int, 5 | @L1TransformID int, 6 | 7 | --Databricks Notebook 8 | @ComputeName varchar(100) = null, 9 | @ComputePath varchar(200) = null, 10 | 11 | --Custom 12 | @CustomParameters varchar(max) = null, 13 | 14 | --Input File 15 | @InputFileSystem varchar(50) = null, 16 | @InputFileFolder varchar(200) = null, 17 | @InputFile varchar(200) = null, 18 | @InputFileDelimiter char(1) = null, 19 | @InputFileHeaderFlag bit = null, 20 | @InputDWTable varchar(200) = null, 21 | 22 | --Delta 23 | @WatermarkColName varchar(50) = null, 24 | @DataFromTimestamp Datetime2 =null, 25 | @DataToTimestamp Datetime2 =null, 26 | @DataFromNumber int =null, 27 | @DataToNumber int =null, 28 | @LastDeltaDate datetime = null, 29 | @LastDeltaNumber int = null, 30 | 31 | 32 | --Curated File 33 | @OutputL2CurateFileSystem varchar(50), 34 | @OutputL2CuratedFolder varchar(200), 35 | @OutputL2CuratedFile varchar(200), 36 | @OutputL2CuratedFileDelimiter char(1) = null, 37 | @OutputL2CuratedFileFormat varchar(10) = null, 38 | @OutputL2CuratedFileWriteMode varchar(20) = null, 39 | 40 | --SQL 41 | @OutputDWStagingTable varchar(200) = null, 42 | @LookupColumns varchar(4000) = null, 43 | @OutputDWTable varchar(200) = null, 44 | @OutputDWTableWriteMode varchar(20) = null, 45 | 46 | 47 | --ADF Pipeline IDs 48 | @IngestADFPipelineRunID uniqueidentifier = null, 49 | @L1TransformADFPipelineRunID uniqueidentifier = null, 50 | 51 | --Max Retries 52 | @MaxRetries int = null 53 | 54 | AS 55 | BEGIN 56 | 57 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 58 | 59 | --Check if Transformation records already exists for the input file for same transformation e.g it's a reload 60 | IF NOT EXISTS 61 | ( 62 | SELECT 1 63 | FROM 64 | [ELT].[L2TransformInstance] 65 | WHERE 66 | [IngestID] = @IngestID 67 | AND L2TransformID = @L2TransformID 68 | --Check does curated file record already exist 69 | AND OutputL2CurateFileSystem = @OutputL2CurateFileSystem 70 | AND OutputL2CuratedFolder = @OutputL2CuratedFolder 71 | AND OutputL2CuratedFile = @OutputL2CuratedFile 72 | 73 | ) 74 | 75 | 76 | BEGIN 77 | --If this is a new transformation 78 | INSERT INTO [ELT].[L2TransformInstance] 79 | ( 80 | [L2TransformID] 81 | ,[IngestID] 82 | ,[L1TransformID] 83 | ,[ComputePath] 84 | ,[ComputeName] 85 | ,[CustomParameters] 86 | ,[InputFileSystem] 87 | ,[InputFileFolder] 88 | ,[InputFile] 89 | ,[InputFileDelimiter] 90 | ,[InputFileHeaderFlag] 91 | ,[InputDWTable] 92 | ,[WatermarkColName] 93 | ,[OutputL2CurateFileSystem] 94 | ,[OutputL2CuratedFolder] 95 | ,[OutputL2CuratedFile] 96 | ,[OutputL2CuratedFileDelimiter] 97 | ,[OutputL2CuratedFileFormat] 98 | ,[OutputL2CuratedFileWriteMode] 99 | ,[OutputDWStagingTable] 100 | ,[LookupColumns] 101 | ,[OutputDWTable] 102 | ,[OutputDWTableWriteMode] 103 | ,[RetryCount] 104 | ,[ActiveFlag] 105 | ,[IngestADFPipelineRunID] 106 | ,[L1TransformADFPipelineRunID] --1555 107 | ,[CreatedBy] 108 | ,[CreatedTimestamp] 109 | 110 | 111 | ) 112 | VALUES 113 | ( 114 | @L2TransformID, 115 | @IngestID, 116 | @L1TransformID, 117 | @ComputePath, 118 | @ComputeName, 119 | @CustomParameters, 120 | @InputFileSystem, 121 | @InputFileFolder, 122 | @InputFile, 123 | @InputFileDelimiter, 124 | @InputFileHeaderFlag, 125 | @InputDWTable, 126 | @WatermarkColName, 127 | @OutputL2CurateFileSystem, 128 | @OutputL2CuratedFolder, 129 | @OutputL2CuratedFile, 130 | @OutputL2CuratedFileDelimiter, 131 | @OutputL2CuratedFileFormat, 132 | @OutputL2CuratedFileWriteMode, 133 | @OutputDWStagingTable, 134 | @LookupColumns, 135 | @OutputDWTable, 136 | @OutputDWTableWriteMode, 137 | 0, 138 | 1, 139 | @IngestADFPipelineRunID, 140 | @L1TransformADFPipelineRunID, --1555 141 | SUSER_NAME(), 142 | @localdate 143 | ) 144 | END 145 | ELSE 146 | --If this is an existing Transformation 147 | BEGIN 148 | --Just update one record in case if there are duplicates 149 | UPDATE TOP (1) [ELT].[L2TransformInstance] 150 | SET 151 | [InputCount] = null 152 | ,[L2TransformInsertCount] = null 153 | ,[L2TransformUpdateCount] = null 154 | ,[L2TransformDeleteCount] = null 155 | ,[L2TransformStartTimestamp] = null 156 | ,[L2TransformEndTimestamp] = null 157 | ,[L2TransformStatus] = null 158 | ,[RetryCount] = 0 159 | ,[ActiveFlag] = 1 160 | ,[ReRunL2TransformFlag] = 1 161 | ,[IngestADFPipelineRunID] = @IngestADFPipelineRunID 162 | ,[L1TransformADFPipelineRunID] = null 163 | ,[L2TransformADFPipelineRunID] = null 164 | ,[ModifiedBy] = suser_sname() 165 | ,[ModifiedTimestamp] = @localdate 166 | WHERE 167 | [IngestID] = @IngestID 168 | AND L2TransformID = @L2TransformID 169 | --Check does curated file record already exist 170 | AND OutputL2CurateFileSystem = @OutputL2CurateFileSystem 171 | AND OutputL2CuratedFolder = @OutputL2CuratedFolder 172 | AND OutputL2CuratedFile = @OutputL2CuratedFile 173 | AND (ActiveFlag = 0 AND ISNULL(RetryCount,0) >= @MaxRetries) 174 | 175 | END 176 | END -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/UpdateIngestDefinition.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[UpdateIngestDefinition] 2 | @IngestID INT, 3 | @LastDeltaDate Datetime2=null, 4 | @LastDeltaNumer int=null, 5 | @IngestStatus varchar(20), 6 | @ReloadFlag bit=0 7 | AS 8 | BEGIN 9 | 10 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 11 | 12 | Update 13 | [ELT].[IngestDefinition] 14 | SET 15 | [LastDeltaDate] = 16 | CASE 17 | --When Successful and the DataToDate does not move forward since LastDeltaDate, Increase LastDeltaDate by the Interval 18 | WHEN @LastDeltaDate IS NOT NULL AND @ReloadFlag <> 1 AND @IngestStatus IN ('Success','ReRunSuccess') AND @LastDeltaDate = [LastDeltaDate] 19 | and [MaxIntervalMinutes] is NOT NULL 20 | THEN 21 | CASE 22 | WHEN 23 | DateAdd(minute,[MaxIntervalMinutes],@LastDeltaDate) > ELT.[uf_GetAestDateTime]() 24 | THEN CONVERT(VARCHAR(30),ELT.[uf_GetAestDateTime](),120) 25 | ELSE 26 | DateAdd(minute,[MaxIntervalMinutes],[LastDeltaDate]) 27 | END 28 | --Re-run delta date is later than existing delta date 29 | WHEN @LastDeltaDate IS NOT NULL AND @IngestStatus IN ('Success','ReRunSuccess') AND datediff_big(ss,[LastDeltaDate],@LastDeltaDate) >= 0 30 | THEN @LastDeltaDate 31 | --Re-run delta date is earlier than existing delta date 32 | WHEN @LastDeltaDate IS NOT NULL AND @IngestStatus IN ('Success','ReRunSuccess') AND datediff_big(ss,@LastDeltaDate,[LastDeltaDate]) >=0 33 | THEN [LastDeltaDate] 34 | ELSE [LastDeltaDate] 35 | END 36 | , [LastDeltaNumber] = 37 | CASE 38 | WHEN @LastDeltaNumer IS NOT NULL AND @IngestStatus IN ('Success','ReRunSuccess') 39 | THEN @LastDeltaNumer 40 | WHEN @LastDeltaNumer IS NOT NULL AND @IngestStatus IN ('Failure','ReRunFailure') 41 | THEN [LastDeltaNumber] 42 | WHEN @LastDeltaNumer IS NULL AND @ReloadFlag <> 1 AND @IngestStatus IN ('Success','ReRunSuccess') 43 | THEN ([LastDeltaNumber] + [MaxIntervalNumber]) 44 | ELSE [LastDeltaNumber] 45 | END 46 | ,[ModifiedBy] =suser_sname() 47 | , [ModifiedTimestamp]=@localdate 48 | WHERE [IngestID]=@IngestID 49 | END -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/UpdateIngestInstance.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[UpdateIngestInstance] 2 | @ADFIngestPipelineRunID Uniqueidentifier, 3 | @IngestStatus varchar(20) =null, 4 | @DataFromTimestamp Datetime2 =null, 5 | @DataToTimestamp Datetime2 =null, 6 | @DataFromNumber int =null, 7 | @DataToNumber int =null, 8 | @SourceCount int=null, 9 | @IngestCount int=null, 10 | @ReloadFlag bit 11 | AS 12 | BEGIN 13 | 14 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 15 | 16 | Update 17 | [ELT].[IngestInstance] 18 | SET 19 | [DataFromTimestamp] = @DataFromTimestamp 20 | ,[DataToTimestamp]=@DataToTimestamp 21 | ,[DataFromNumber] = @DataFromNumber 22 | ,[DataToNumber]=@DataToNumber 23 | ,[SourceCount] =@SourceCount 24 | ,[IngestCount]=@IngestCount 25 | ,[IngestEndTimestamp] =@localdate 26 | ,[IngestStatus] =(CASE WHEN @ReloadFlag=1 AND @IngestStatus='Success' THEN 'ReRunSuccess' 27 | WHEN @ReloadFlag=1 AND @IngestStatus='Failure' THEN 'ReRunFailure' 28 | ELSE @IngestStatus 29 | END) 30 | ,[RetryCount] = (CASE WHEN @IngestStatus IN ('Success','ReRunSuccess') THEN 0 31 | WHEN @IngestStatus IN ('Failure','ReRunFailure') THEN ISNULL([RetryCount],0) +1 32 | END) 33 | ,[ReloadFlag] =(CASE WHEN @ReloadFlag=1 AND @IngestStatus IN ('Success','ReRunSuccess') THEN 0 34 | WHEN @ReloadFlag=1 AND @IngestStatus IN ('Failure','ReRunFailure') THEN 1 35 | ELSE 0 36 | END) 37 | ,[ModifiedBy]=suser_sname() 38 | ,[ModifiedTimestamp] = @localdate 39 | WHERE 40 | ADFIngestPipelineRunID =@ADFIngestPipelineRunID 41 | END 42 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/UpdateTransformDefinition_L2.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[UpdateTransformDefinition_L2] 2 | @L2TransformID INT, 3 | @LastDeltaDate Datetime2 =null, 4 | @LastDeltaNumber int =null 5 | as 6 | BEGIN 7 | Update [ELT].[L2TransformDefinition] 8 | SET 9 | [ModifiedBy] =suser_sname(), 10 | [ModifiedTimestamp]=GETDATE(), 11 | [LastDeltaDate] = COALESCE(@LastDeltaDate,[LastDeltaDate],ELT.uf_GetAestDateTime()), 12 | [LastDeltaNumber] = COALESCE(@LastDeltaNumber,[LastDeltaNumber]) 13 | 14 | WHERE [L2TransformID] = @L2TransformID 15 | END 16 | GO -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/UpdateTransformInstance_L1.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[UpdateTransformInstance_L1] 2 | @L1TransformInstanceId INT 3 | , @L1TransformStatus VARCHAR(20) 4 | , @L1TransformADFPipelineRunID UNIQUEIDENTIFIER 5 | , @IngestCount INT = NULL 6 | , @L1TransformInsertCount INT = NULL 7 | , @L1TransformUpdateCount INT = NULL 8 | , @L1TransformDeleteCount INT = NULL 9 | , @MaxRetries int = null 10 | AS 11 | BEGIN 12 | 13 | DECLARE @localdate datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 14 | 15 | Update 16 | [ELT].[L1TransformInstance] 17 | SET 18 | [L1TransformStartTimestamp] = CASE 19 | WHEN @L1TransformStatus IN ('Running') 20 | THEN @localdate 21 | ELSE [L1TransformStartTimestamp] 22 | END 23 | 24 | , [L1TransformEndTimestamp] = CASE 25 | WHEN @L1TransformStatus IN ('Success','Failure','ReRunSuccess','ReRunFailure') 26 | THEN @localdate 27 | ELSE NULL 28 | END 29 | 30 | , [L1TransformStatus] = CASE 31 | WHEN ([ReRunL1TransformFlag] = 1 OR [RetryCount] >0) AND @L1TransformStatus='Success' 32 | THEN 'ReRunSuccess' 33 | WHEN [ReRunL1TransformFlag] = 1 AND @L1TransformStatus='Failure' 34 | THEN 'ReRunFailure' 35 | ELSE @L1TransformStatus 36 | END 37 | , [ActiveFlag] = CASE 38 | WHEN @L1TransformStatus IN ('Success','ReRunSuccess') 39 | THEN 0 40 | WHEN @L1TransformStatus = 'Failure' and ISNULL(RetryCount,0) +1 >= @MaxRetries 41 | THEN 0 42 | ELSE 1 43 | END 44 | , [ReRunL1TransformFlag] = CASE 45 | WHEN [ReRunL1TransformFlag] = 1 AND @L1TransformStatus IN ('Success','ReRunSuccess') 46 | THEN 0 47 | WHEN @L1TransformStatus in ('Failure', 'ReRunFailure') and ISNULL(RetryCount,0) +1 >= @MaxRetries 48 | THEN 0 49 | ELSE [ReRunL1TransformFlag] 50 | END 51 | , [RetryCount] = CASE 52 | WHEN 53 | @L1TransformStatus in ('Success', 'ReRunSuccess') 54 | THEN 0 55 | WHEN 56 | @L1TransformStatus in ('Failure', 'ReRunFailure') 57 | THEN ISNULL([RetryCount],0) + 1 58 | Else [RetryCount] 59 | END 60 | 61 | , [ModifiedBy] =suser_sname() 62 | , [ModifiedTimestamp]=@localdate 63 | , [L1TransformADFPipelineRunID] = @L1TransformADFPipelineRunID 64 | , [IngestCount] = ISNULL(@IngestCount,[IngestCount]) 65 | , [L1TransformInsertCount] = ISNULL(@L1TransformInsertCount,[L1TransformInsertCount]) 66 | , [L1TransformUpdateCount] = ISNULL(@L1TransformUpdateCount,[L1TransformUpdateCount]) 67 | , [L1TransformDeleteCount] = ISNULL(@L1TransformDeleteCount,[L1TransformDeleteCount]) 68 | WHERE 69 | [L1TransformInstanceID] = @L1TransformInstanceId 70 | END 71 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Stored Procedures/UpdateTransformInstance_L2.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [ELT].[UpdateTransformInstance_L2] 2 | @L2TransformInstanceId INT 3 | ,@L2TransformStatus VARCHAR(20) 4 | ,@L2TransformADFPipelineRunID UNIQUEIDENTIFIER 5 | ,@InputCount INT = NULL 6 | ,@L2TransformInsertCount INT = NULL 7 | ,@L2TransformUpdateCount INT = NULL 8 | ,@L2TransformDeleteCount INT = NULL 9 | ,@DataFromTimestamp Datetime2 = null 10 | ,@DataToTimestamp Datetime2 = null 11 | ,@DataFromNumber int = null 12 | ,@DataToNumber int = null 13 | ,@MaxRetries int = null 14 | AS 15 | BEGIN 16 | 17 | DECLARE @localdate datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time') 18 | 19 | Update 20 | [ELT].[L2TransformInstance] 21 | SET 22 | [L2TransformStartTimestamp] = CASE 23 | WHEN @L2TransformStatus IN ('Running') 24 | THEN @localdate 25 | ELSE [L2TransformStartTimestamp] 26 | END 27 | 28 | , [L2TransformEndTimestamp] = CASE 29 | WHEN @L2TransformStatus IN ('Success','Failure','ReRunSuccess','ReRunFailure') 30 | THEN @localdate 31 | ELSE NULL 32 | END 33 | , [DataFromTimestamp] = @DataFromTimestamp 34 | , [DataToTimestamp] = @DataToTimestamp 35 | , [DataFromNumber] = @DataFromNumber 36 | , [DataToNumber] = @DataToNumber 37 | , [L2TransformStatus] = CASE 38 | WHEN ([ReRunL2TransformFlag] = 1 OR [RetryCount] > 0) AND @L2TransformStatus = 'Success' 39 | THEN 'ReRunSuccess' 40 | WHEN [ReRunL2TransformFlag] = 1 AND @L2TransformStatus = 'Failure' 41 | THEN 'ReRunFailure' 42 | ELSE @L2TransformStatus 43 | END 44 | , [ActiveFlag] = CASE 45 | WHEN @L2TransformStatus IN ('Success','ReRunSuccess') 46 | THEN 0 47 | WHEN @L2TransformStatus = 'Failure' and ISNULL(RetryCount,0) +1 >= @MaxRetries 48 | THEN 0 49 | ELSE 1 50 | END 51 | , [ReRunL2TransformFlag] = CASE 52 | WHEN [ReRunL2TransformFlag] =1 AND @L2TransformStatus IN ('Success','ReRunSuccess') 53 | THEN 0 54 | WHEN @L2TransformStatus in ('Failure', 'ReRunFailure') and ISNULL(RetryCount,0) +1 >= @MaxRetries 55 | THEN 0 56 | ELSE [ReRunL2TransformFlag] 57 | END 58 | , [RetryCount] = CASE 59 | WHEN 60 | @L2TransformStatus in ('Success', 'ReRunSuccess') 61 | THEN 0 62 | WHEN 63 | @L2TransformStatus in ('Failure', 'ReRunFailure') 64 | THEN ISNULL([RetryCount],0) + 1 65 | ELSE [RetryCount] 66 | END 67 | 68 | , [ModifiedBy] =suser_sname() 69 | , [ModifiedTimestamp]=@localdate 70 | , [L2TransformADFPipelineRunID] = @L2TransformADFPipelineRunID 71 | , [InputCount] = ISNULL(@InputCount,[InputCount]) 72 | , [L2TransformInsertCount] = ISNULL(@L2TransformInsertCount,[L2TransformInsertCount]) 73 | , [L2TransformUpdateCount] = ISNULL(@L2TransformUpdateCount,[L2TransformUpdateCount]) 74 | , [L2TransformDeleteCount] = ISNULL(@L2TransformDeleteCount,[L2TransformDeleteCount]) 75 | WHERE 76 | [L2TransformInstanceID] = @L2TransformInstanceId 77 | END -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Tables/ColumnMapping.sql: -------------------------------------------------------------------------------- 1 | GO 2 | 3 | CREATE TABLE [ELT].[ColumnMapping]( 4 | [MappingID] int NOT NULL IDENTITY, 5 | [IngestID] int NULL, 6 | [L1TransformID] int NULL, 7 | [L2TransformID] int NULL, 8 | [SourceName] nvarchar(150) NOT NULL, 9 | [TargetName] nvarchar(150) NOT NULL, 10 | [Description] nvarchar(250) NULL, 11 | [TargetOrdinalPosition] int NOT NULL, 12 | [ActiveFlag] bit NULL, 13 | [CreatedBy] varchar(128) NOT NULL, 14 | [CreatedTimestamp] datetime NOT NULL, 15 | [ModifiedBy] varchar(128) NULL, 16 | [ModifiedTimestamp] datetime NULL) 17 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Tables/IngestDefinition.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [ELT].[IngestDefinition] 2 | ( 3 | [IngestID] int not null identity, 4 | [SourceSystemName] varchar(50) not null, 5 | [StreamName] varchar(100) null, 6 | [SourceSystemDescription] varchar(200) null, 7 | [Backend] varchar(30) null, 8 | [DataFormat] varchar(10) null, 9 | [EntityName] varchar(100) null, 10 | [WatermarkColName] varchar(50) null, 11 | [DeltaFormat] varchar(30) null, 12 | [LastDeltaDate] datetime2 null, 13 | [LastDeltaNumber] int null, 14 | [LastDeltaString] varchar(50) null, 15 | [MaxIntervalMinutes] int null, 16 | [MaxIntervalNumber] int null, 17 | [DataMapping] varchar(max) null, 18 | [SourceFileDropFileSystem] varchar(50) null, 19 | [SourceFileDropFolder] varchar(200) null, 20 | [SourceFileDropFile] varchar(200) null, 21 | [SourceFileDelimiter] char(1) null, 22 | [SourceFileHeaderFlag] bit null, 23 | [SourceStructure] varchar(max) null, 24 | [DestinationRawFileSystem] varchar(50) null, 25 | [DestinationRawFolder] varchar(200) null, 26 | [DestinationRawFile] varchar(200) null, 27 | [RunSequence] int null, 28 | [MaxRetries] int null, 29 | [ActiveFlag] bit not null, 30 | [L1TransformationReqdFlag] bit not null, 31 | [L2TransformationReqdFlag] bit not null, 32 | [DelayL1TransformationFlag] bit not null, 33 | [DelayL2TransformationFlag] bit not null, 34 | [CreatedBy] nvarchar(128) not null, 35 | [CreatedTimestamp] datetime not null, 36 | [ModifiedBy] nvarchar(128) null, 37 | [ModifiedTimestamp] datetime null, 38 | ) 39 | 40 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Tables/IngestInstance.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [ELT].[IngestInstance] 2 | ( 3 | [IngestInstanceID] int not null identity, 4 | [IngestID] int not null, 5 | [SourceFileDropFileSystem] varchar(50) null, 6 | [SourceFileDropFolder] varchar(200) null, 7 | [SourceFileDropFile] varchar(200) null, 8 | [DestinationRawFileSystem] varchar(50) not null, 9 | [DestinationRawFolder] varchar(200) not null, 10 | [DestinationRawFile] varchar(200) not null, 11 | [DataFromTimestamp] dateTime2 null, 12 | [DataToTimestamp] datetime2 null, 13 | [DataFromNumber] Int null, 14 | [DataToNumber] Int null, 15 | [SourceCount] int null, 16 | [IngestCount] int null, 17 | [IngestStartTimestamp] datetime null, 18 | [IngestEndTimestamp] datetime null, 19 | [IngestStatus] varchar(20) null, 20 | [RetryCount] int null, 21 | [ReloadFlag] bit null, 22 | [CreatedBy] nvarchar(128) not null, 23 | [CreatedTimestamp] datetime not null, 24 | [ModifiedBy] nvarchar(128) null, 25 | [ModifiedTimestamp] datetime null, 26 | [ADFIngestPipelineRunID] uniqueidentifier null 27 | ) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Tables/L1TransformDefinition.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [ELT].[L1TransformDefinition] 2 | ( 3 | [L1TransformID] int not null identity, 4 | [IngestID] int not null, 5 | [ComputePath] varchar(200) null, 6 | [ComputeName] varchar(100) null, 7 | [CustomParameters] varchar(max) null, 8 | [InputRawFileSystem] varchar(50) not null, 9 | [InputRawFileFolder] varchar(200) not null, 10 | [InputRawFile] varchar(200) not null, 11 | [InputRawFileDelimiter] char(1) null, 12 | [InputFileHeaderFlag] bit null, 13 | [OutputL1CurateFileSystem] varchar(50) not null, 14 | [OutputL1CuratedFolder] varchar(200) not null, 15 | [OutputL1CuratedFile] varchar(200) not null, 16 | [OutputL1CuratedFileDelimiter] char(1) null, 17 | [OutputL1CuratedFileFormat] varchar(10) null, 18 | [OutputL1CuratedFileWriteMode] varchar(20) null, 19 | --Output SQL DW Table 20 | [OutputDWStagingTable] varchar(200) null, 21 | [LookupColumns] varchar(4000) null, 22 | [OutputDWTable] varchar(200) null, 23 | [OutputDWTableWriteMode] varchar(20) null, 24 | [MaxRetries] int null, 25 | [WatermarkColName] varchar(50) null, 26 | [ActiveFlag] bit not null, 27 | [CreatedBy] nvarchar(128) not null, 28 | [CreatedTimestamp] datetime not null, 29 | [ModifiedBy] nvarchar(128) null, 30 | [ModifiedTimestamp] datetime null 31 | ) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Tables/L1TransformInstance.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [ELT].[L1TransformInstance] 2 | ( 3 | [L1TransformInstanceID] int not null identity, 4 | [L1TransformID] int not null, 5 | [IngestInstanceID] int null, 6 | [IngestID] int not null, 7 | [ComputeName] varchar(100) null, 8 | [ComputePath] varchar(200) null, 9 | [CustomParameters] varchar(max) null, 10 | [InputRawFileSystem] varchar(50) not null, 11 | [InputRawFileFolder] varchar(200) not null, 12 | [InputRawFile] varchar(200) not null, 13 | [InputRawFileDelimiter] char(1) null, 14 | [InputFileHeaderFlag] bit null, 15 | [OutputL1CurateFileSystem] varchar(50) not null, 16 | [OutputL1CuratedFolder] varchar(200) not null, 17 | [OutputL1CuratedFile] varchar(200) not null, 18 | [OutputL1CuratedFileDelimiter] char(1) null, 19 | [OutputL1CuratedFileFormat] varchar(10) null, 20 | [OutputL1CuratedFileWriteMode] varchar(20) null, 21 | [OutputDWStagingTable] varchar(200) null, 22 | [LookupColumns] varchar(4000) null, 23 | [OutputDWTable] varchar(200) null, 24 | [OutputDWTableWriteMode] varchar(20) null, 25 | [IngestCount] int null, 26 | [L1TransformInsertCount] int null, 27 | [L1TransformUpdateCount] int null, 28 | [L1TransformDeleteCount] int null, 29 | [L1TransformStartTimestamp] datetime null, 30 | [L1TransformEndTimestamp] datetime null, 31 | [L1TransformStatus] varchar(20) null, 32 | [RetryCount] int null, 33 | [ActiveFlag] bit not null, 34 | [ReRunL1TransformFlag] bit null, 35 | [IngestADFPipelineRunID] uniqueidentifier null, 36 | [L1TransformADFPipelineRunID] uniqueidentifier null, 37 | [CreatedBy] nvarchar(128) not null, 38 | [CreatedTimestamp] datetime not null, 39 | [ModifiedBy] nvarchar(128) null, 40 | [ModifiedTimestamp] datetime null 41 | ) 42 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Tables/L2TransformDefinition.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [ELT].[L2TransformDefinition] 2 | ( 3 | [L2TransformID] int not null identity, 4 | [IngestID] int null, 5 | [L1TransformID] int null, 6 | [ComputePath] varchar(200) null, 7 | [ComputeName] varchar(100) null, 8 | [CustomParameters] varchar(max) null, 9 | [InputType] varchar(15) null, 10 | [InputFileSystem] varchar(50) null, 11 | [InputFileFolder] varchar(200) null, 12 | [InputFile] varchar(200) null, 13 | [InputFileDelimiter] char(1) null, 14 | [InputFileHeaderFlag] bit null, 15 | [InputDWTable] varchar(200) null, 16 | [WatermarkColName] varchar(50) null, 17 | [LastDeltaDate] datetime2 null, 18 | [LastDeltaNumber] int null, 19 | [MaxIntervalMinutes] int null, 20 | [MaxIntervalNumber] int null, 21 | [MaxRetries] int null, 22 | [OutputL2CurateFileSystem] varchar(50) null, 23 | [OutputL2CuratedFolder] varchar(200) null, 24 | [OutputL2CuratedFile] varchar(200) null, 25 | [OutputL2CuratedFileDelimiter] char(1) null, 26 | [OutputL2CuratedFileFormat] varchar(10) null, 27 | [OutputL2CuratedFileWriteMode] varchar(20) null, 28 | --Output SQL DW Table 29 | [OutputDWStagingTable] varchar(200) null, 30 | [LookupColumns] varchar(4000) null, 31 | [OutputDWTable] varchar(200) null, 32 | [OutputDWTableWriteMode] varchar(20) null, 33 | [ActiveFlag] bit not null, 34 | [RunSequence] int not null, 35 | [CreatedBy] nvarchar(128) not null, 36 | [CreatedTimestamp] datetime not null, 37 | [ModifiedBy] nvarchar(128) null, 38 | [ModifiedTimestamp] datetime null 39 | ) -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/Tables/L2TransformInstance.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [ELT].[L2TransformInstance] 2 | ( 3 | [L2TransformInstanceID] int not null identity, 4 | [L2TransformID] int null, 5 | [IngestID] int null, 6 | [L1TransformID] int null, 7 | [ComputePath] varchar(200) null, 8 | [ComputeName] varchar(100) null, 9 | [CustomParameters] varchar(max) null, 10 | [InputFileSystem] varchar(50) null, 11 | [InputFileFolder] varchar(200) null, 12 | [InputFile] varchar(200) null, 13 | [InputFileDelimiter] char(1) null, 14 | [InputFileHeaderFlag] bit null, 15 | [InputDWTable] varchar(200) null, 16 | [WatermarkColName] varchar(50) null, 17 | [DataFromTimestamp] dateTime2 null, 18 | [DataToTimestamp] datetime2 null, 19 | [DataFromNumber] int null, 20 | [DataToNumber] int null, 21 | [OutputL2CurateFileSystem] varchar(50) not null, 22 | [OutputL2CuratedFolder] varchar(200) not null, 23 | [OutputL2CuratedFile] varchar(200) not null, 24 | [OutputL2CuratedFileDelimiter] char(1) null, 25 | [OutputL2CuratedFileFormat] varchar(10) null, 26 | [OutputL2CuratedFileWriteMode] varchar(20) null, 27 | [OutputDWStagingTable] varchar(200) null, 28 | [LookupColumns] varchar(4000) null, 29 | [OutputDWTable] varchar(200) null, 30 | [OutputDWTableWriteMode] varchar(20) null, 31 | [InputCount] int null, 32 | [L2TransformInsertCount] int null, 33 | [L2TransformUpdateCount] int null, 34 | [L2TransformDeleteCount] int null, 35 | [L2TransformStartTimestamp] datetime null, 36 | [L2TransformEndTimestamp] datetime null, 37 | [L2TransformStatus] varchar(20) null, 38 | [RetryCount] int null, 39 | [ActiveFlag] bit not null, 40 | [ReRunL2TransformFlag] bit null, 41 | [IngestADFPipelineRunID] uniqueidentifier null, 42 | [L1TransformADFPipelineRunID] uniqueidentifier null, 43 | [L2TransformADFPipelineRunID] uniqueidentifier null, 44 | [CreatedBy] nvarchar(128) not null, 45 | [CreatedTimestamp] datetime not null, 46 | [ModifiedBy] nvarchar(128) null, 47 | [ModifiedTimestamp] datetime null, 48 | [L2SnapshotGroupID] INT NULL, 49 | [L2SnapshotInstanceID] INT NULL 50 | ) 51 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/ELT/elt.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA [ELT] -------------------------------------------------------------------------------- /elt-framework/ControlDB/Post Deployment Script/Script.PostDeployment.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Post-Deployment Script Template 3 | -------------------------------------------------------------------------------------- 4 | This file contains SQL statements that will be appended to the build script. 5 | Use SQLCMD syntax to include a file in the post-deployment script. 6 | Example: :r .\myfile.sql 7 | Use SQLCMD syntax to reference a variable in the post-deployment script. 8 | Example: :setvar TableName MyTable 9 | SELECT * FROM [$(TableName)] 10 | -------------------------------------------------------------------------------------- 11 | */ 12 | 13 | --IngestDefinition 14 | :r .\datasources\RestAPI\IngestDefinition\PurviewRestAPI.sql 15 | :r .\datasources\RestAPI\IngestDefinition\AzureRestAPI.sql 16 | :r .\datasources\UploadFiles\IngestDefinition\CustomFormRecognizerModel.sql 17 | 18 | --L1TransformDefinition 19 | :r .\datasources\RestAPI\L1TransformDefinition\L1T_PurviewRestAPI.sql 20 | :r .\datasources\RestAPI\L1TransformDefinition\L1T_AzureRestAPI.sql 21 | :r .\datasources\UploadFiles\L1TransformDefinition\L1T_CustomFormRecognizerModel.sql 22 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/Post Deployment Script/datasources/RestAPI/IngestDefinition/AzureRestAPI.sql: -------------------------------------------------------------------------------- 1 | /* 2 | ELT Configuration for REST API Data Sources 3 | */ 4 | 5 | IF OBJECT_ID('tempdb..#AzureRestAPI_Ingest') IS NOT NULL DROP TABLE #AzureRestAPI_Ingest; 6 | 7 | --Create Temp table with same structure as IngestDefinition 8 | CREATE TABLE #AzureRestAPI_Ingest 9 | ( 10 | [SourceSystemName] [varchar](50) NOT NULL, 11 | [StreamName] [varchar](100) NULL, 12 | [SourceSystemDescription] [varchar](200) NULL, 13 | [Backend] [varchar](30) NULL, 14 | [EntityName] [varchar](100) NULL, 15 | [WatermarkColName] [varchar](50) NULL, 16 | [LastDeltaDate] [datetime2](7) NULL, 17 | [LastDeltaNumber] [int] NULL, 18 | [LastDeltaString] [varchar](50) NULL, 19 | [MaxIntervalMinutes] [int] NULL, 20 | [MaxIntervalNumber] [int] NULL, 21 | [DataMapping] [varchar](max) NULL, 22 | [DestinationRawFileSystem] [varchar](50) NULL, 23 | [DestinationRawFolder] [varchar](200) NULL, 24 | [DestinationRawFile] [varchar](200) NULL, 25 | [RunSequence] [int] NULL, 26 | [MaxRetries] [int] NULL, 27 | [ActiveFlag] [bit] NOT NULL, 28 | [L1TransformationReqdFlag] [bit] NOT NULL, 29 | [L2TransformationReqdFlag] [bit] NOT NULL, 30 | [DelayL1TransformationFlag] [bit] NOT NULL, 31 | [DelayL2TransformationFlag] [bit] NOT NULL 32 | ); 33 | 34 | 35 | --Insert Into Temp Table 36 | INSERT INTO #AzureRestAPI_Ingest 37 | --Operations 38 | SELECT 'Azure' AS [SourceSystemName], 39 | 'operations' AS [StreamName], 40 | 'Azure APIs' AS [SourceSystemDescription], 41 | 'AZURE REST API' AS [Backend], 42 | '/providers/Microsoft.Purview/operations?api-version=2020-12-01-preview' AS [EntityName], 43 | NULL AS [WatermarkColName], 44 | NULL AS [LastDeltaDate], 45 | NULL AS [LastDeltaNumber], 46 | NULL AS [LastDeltaString], 47 | NULL AS [MaxIntervalMinutes], 48 | NULL AS [MaxIntervalNumber], 49 | '{ 50 | "type": "TabularTranslator", 51 | "mappings": [ 52 | { 53 | "source": { 54 | "path": "[''name'']" 55 | }, 56 | "sink": { 57 | "name": "name", 58 | "type": "String" 59 | } 60 | }, 61 | { 62 | "source": { 63 | "path": "[''display''][''provider'']" 64 | }, 65 | "sink": { 66 | "name": "provider", 67 | "type": "String" 68 | } 69 | }, 70 | { 71 | "source": { 72 | "path": "[''display''][''resource'']" 73 | }, 74 | "sink": { 75 | "name": "resource", 76 | "type": "String" 77 | } 78 | }, 79 | { 80 | "source": { 81 | "path": "[''display''][''operation'']" 82 | }, 83 | "sink": { 84 | "name": "operation", 85 | "type": "String" 86 | } 87 | }, 88 | { 89 | "source": { 90 | "path": "[''display''][''description'']" 91 | }, 92 | "sink": { 93 | "name": "description", 94 | "type": "String" 95 | } 96 | } 97 | ], 98 | "collectionReference": "$[''value'']", 99 | "mapComplexValuesToString": true 100 | }'AS [DataMapping], 101 | 'raw-bronze' AS [DestinationRawFileSystem], 102 | 'purview/operations/YYYY-MM' AS [DestinationRawFolder] , 103 | 'operations_YYYYMMDD.parquet' AS [DestinationRawFile], 104 | 1 AS [RunSequence], 105 | 3 AS [MaxRetries], 106 | 1 AS [ActiveFlag], 107 | 1 AS [L1TransformationReqdFlag], 108 | 1 AS [L2TransformationReqdFlag], 109 | 1 AS [DelayL1TransformationFlag], 110 | 1 AS [DelayL2TransformationFlag] ; 111 | 112 | --Merge with Temp table for re-runnability 113 | 114 | MERGE INTO [ELT].[IngestDefinition] AS tgt 115 | USING #AzureRestAPI_Ingest AS src 116 | ON src.[SourceSystemName]=tgt.[SourceSystemName] AND src.[StreamName] =tgt.[StreamName] 117 | WHEN MATCHED THEN 118 | UPDATE SET tgt.[SourceSystemDescription] = src.[SourceSystemDescription], 119 | tgt.[Backend] = src.[Backend], 120 | tgt.[EntityName] = src.[EntityName], 121 | tgt.[WatermarkColName] = src.[WatermarkColName], 122 | tgt.[LastDeltaDate] = src.[LastDeltaDate], 123 | tgt.[LastDeltaNumber] = src.[LastDeltaNumber], 124 | tgt.[LastDeltaString] = src.[LastDeltaString], 125 | tgt.[MaxIntervalMinutes] = src.[MaxIntervalMinutes], 126 | tgt.[MaxIntervalNumber] = src.[MaxIntervalNumber], 127 | tgt.[DataMapping] = src.[DataMapping], 128 | tgt.[DestinationRawFileSystem] = src.[DestinationRawFileSystem], 129 | tgt.[DestinationRawFolder] = src.[DestinationRawFolder], 130 | tgt.[DestinationRawFile] = src.[DestinationRawFile], 131 | tgt.[RunSequence] = src.[RunSequence], 132 | tgt.[MaxRetries] = src.[MaxRetries], 133 | tgt.[ActiveFlag] = src.[ActiveFlag], 134 | tgt.[L1TransformationReqdFlag] = src.[L1TransformationReqdFlag], 135 | tgt.[L2TransformationReqdFlag] = src.[L2TransformationReqdFlag], 136 | tgt.[DelayL1TransformationFlag] = src.[DelayL1TransformationFlag], 137 | tgt.[DelayL2TransformationFlag] = src.[DelayL2TransformationFlag], 138 | tgt.[ModifiedBy] = USER_NAME(), 139 | tgt.[ModifiedTimestamp] = GetDate() 140 | WHEN NOT MATCHED BY TARGET THEN 141 | INSERT([SourceSystemName], 142 | [StreamName], 143 | [SourceSystemDescription], 144 | [Backend], 145 | [EntityName], 146 | [WatermarkColName], 147 | [LastDeltaDate], 148 | [LastDeltaNumber], 149 | [LastDeltaString], 150 | [MaxIntervalMinutes], 151 | [MaxIntervalNumber], 152 | [DataMapping], 153 | [DestinationRawFileSystem], 154 | [DestinationRawFolder], 155 | [DestinationRawFile], 156 | [RunSequence], 157 | [MaxRetries], 158 | [ActiveFlag], 159 | [L1TransformationReqdFlag], 160 | [L2TransformationReqdFlag], 161 | [DelayL1TransformationFlag], 162 | [DelayL2TransformationFlag], 163 | [CreatedBy], 164 | [CreatedTimestamp] ) 165 | VALUES (src.[SourceSystemName], 166 | src.[StreamName], 167 | src.[SourceSystemDescription], 168 | src.[Backend], 169 | src.[EntityName], 170 | src.[WatermarkColName], 171 | src.[LastDeltaDate], 172 | src.[LastDeltaNumber], 173 | src.[LastDeltaString], 174 | src.[MaxIntervalMinutes], 175 | src.[MaxIntervalNumber], 176 | src.[DataMapping], 177 | src.[DestinationRawFileSystem], 178 | src.[DestinationRawFolder], 179 | src.[DestinationRawFile], 180 | src.[RunSequence], 181 | src.[MaxRetries], 182 | src.[ActiveFlag], 183 | src.[L1TransformationReqdFlag], 184 | src.[L2TransformationReqdFlag], 185 | src.[DelayL1TransformationFlag], 186 | src.[DelayL2TransformationFlag], 187 | USER_NAME(), 188 | GETDATE()); -------------------------------------------------------------------------------- /elt-framework/ControlDB/Post Deployment Script/datasources/RestAPI/IngestDefinition/PurviewRestAPI.sql: -------------------------------------------------------------------------------- 1 | /* 2 | ELT Configuration for REST API Data Sources 3 | */ 4 | 5 | IF OBJECT_ID('tempdb..#PurviewRestAPI_Ingest') IS NOT NULL DROP TABLE #PurviewRestAPI_Ingest; 6 | 7 | --Create Temp table with same structure as IngestDefinition 8 | CREATE TABLE #PurviewRestAPI_Ingest 9 | ( 10 | [SourceSystemName] [varchar](50) NOT NULL, 11 | [StreamName] [varchar](100) NULL, 12 | [SourceSystemDescription] [varchar](200) NULL, 13 | [Backend] [varchar](30) NULL, 14 | [EntityName] [varchar](100) NULL, 15 | [WatermarkColName] [varchar](50) NULL, 16 | [LastDeltaDate] [datetime2](7) NULL, 17 | [LastDeltaNumber] [int] NULL, 18 | [LastDeltaString] [varchar](50) NULL, 19 | [MaxIntervalMinutes] [int] NULL, 20 | [MaxIntervalNumber] [int] NULL, 21 | [DataMapping] [varchar](max) NULL, 22 | [DestinationRawFileSystem] [varchar](50) NULL, 23 | [DestinationRawFolder] [varchar](200) NULL, 24 | [DestinationRawFile] [varchar](200) NULL, 25 | [RunSequence] [int] NULL, 26 | [MaxRetries] [int] NULL, 27 | [ActiveFlag] [bit] NOT NULL, 28 | [L1TransformationReqdFlag] [bit] NOT NULL, 29 | [L2TransformationReqdFlag] [bit] NOT NULL, 30 | [DelayL1TransformationFlag] [bit] NOT NULL, 31 | [DelayL2TransformationFlag] [bit] NOT NULL 32 | ); 33 | 34 | --Insert Into Temp Table 35 | INSERT INTO #PurviewRestAPI_Ingest 36 | --datasources 37 | SELECT 'Purview' AS [SourceSystemName], 38 | 'datasources' AS [StreamName], 39 | 'Microsoft Purview APIs' AS [SourceSystemDescription], 40 | 'ATLAS REST API' AS [Backend], 41 | '/scan/datasources?api-version=2022-02-01-preview' AS [EntityName], 42 | NULL AS [WatermarkColName], 43 | NULL AS [LastDeltaDate], 44 | NULL AS [LastDeltaNumber], 45 | NULL AS [LastDeltaString], 46 | NULL AS [MaxIntervalMinutes], 47 | NULL AS [MaxIntervalNumber], 48 | '{ 49 | "type": "TabularTranslator", 50 | "mappings": [ 51 | { 52 | "source": { 53 | "path": "[''properties''][''endpoint'']" 54 | }, 55 | "sink": { 56 | "name": "endpoint", 57 | "type": "String" 58 | } 59 | }, 60 | { 61 | "source": { 62 | "path": "[''properties''][''resourceGroup'']" 63 | }, 64 | "sink": { 65 | "name": "resourceGroup", 66 | "type": "String" 67 | } 68 | }, 69 | { 70 | "source": { 71 | "path": "[''properties''][''subscriptionId'']" 72 | }, 73 | "sink": { 74 | "name": "subscriptionId", 75 | "type": "String" 76 | } 77 | }, 78 | { 79 | "source": { 80 | "path": "[''properties''][''location'']" 81 | }, 82 | "sink": { 83 | "name": "location", 84 | "type": "String" 85 | } 86 | }, 87 | { 88 | "source": { 89 | "path": "[''properties''][''resourceName'']" 90 | }, 91 | "sink": { 92 | "name": "resourceName", 93 | "type": "String" 94 | } 95 | }, 96 | { 97 | "source": { 98 | "path": "[''properties''][''resourceId'']" 99 | }, 100 | "sink": { 101 | "name": "resourceId", 102 | "type": "String" 103 | } 104 | }, 105 | { 106 | "source": { 107 | "path": "[''properties''][''dataUseGovernance'']" 108 | }, 109 | "sink": { 110 | "name": "dataUseGovernance", 111 | "type": "String" 112 | } 113 | }, 114 | { 115 | "source": { 116 | "path": "[''properties''][''createdAt'']" 117 | }, 118 | "sink": { 119 | "name": "createdAt", 120 | "type": "String" 121 | } 122 | }, 123 | { 124 | "source": { 125 | "path": "[''properties''][''lastModifiedAt'']" 126 | }, 127 | "sink": { 128 | "name": "lastModifiedAt", 129 | "type": "String" 130 | } 131 | }, 132 | { 133 | "source": { 134 | "path": "[''properties''][''parentCollection'']" 135 | }, 136 | "sink": { 137 | "name": "parentCollection", 138 | "type": "String" 139 | } 140 | }, 141 | { 142 | "source": { 143 | "path": "[''properties''][''collection''][''lastModifiedAt'']" 144 | }, 145 | "sink": { 146 | "name": "Collection_lastModifiedAt", 147 | "type": "String" 148 | } 149 | }, 150 | { 151 | "source": { 152 | "path": "[''properties''][''collection''][''referenceName'']" 153 | }, 154 | "sink": { 155 | "name": "referenceName", 156 | "type": "String" 157 | } 158 | }, 159 | { 160 | "source": { 161 | "path": "[''properties''][''collection''][''type'']" 162 | }, 163 | "sink": { 164 | "name": "type", 165 | "type": "String" 166 | } 167 | }, 168 | { 169 | "source": { 170 | "path": "[''properties''][''dataSourceCollectionMovingState'']" 171 | }, 172 | "sink": { 173 | "name": "dataSourceCollectionMovingState", 174 | "type": "String" 175 | } 176 | }, 177 | { 178 | "source": { 179 | "path": "[''kind'']" 180 | }, 181 | "sink": { 182 | "name": "kind", 183 | "type": "String" 184 | } 185 | }, 186 | { 187 | "source": { 188 | "path": "[''id'']" 189 | }, 190 | "sink": { 191 | "name": "id", 192 | "type": "String" 193 | } 194 | }, 195 | { 196 | "source": { 197 | "path": "[''name'']" 198 | }, 199 | "sink": { 200 | "name": "name", 201 | "type": "String" 202 | } 203 | }, 204 | { 205 | "source": { 206 | "path": "$[''count'']" 207 | }, 208 | "sink": { 209 | "name": "count", 210 | "type": "String" 211 | } 212 | } 213 | ], 214 | "collectionReference": "$[''value'']", 215 | "mapComplexValuesToString": true 216 | }'AS [DataMapping], 217 | 'raw-bronze' AS [DestinationRawFileSystem], 218 | 'purview/datasources/YYYY-MM' AS [DestinationRawFolder] , 219 | 'datasources_YYYYMMDD.parquet' AS [DestinationRawFile], 220 | 1 AS [RunSequence], 221 | 3 AS [MaxRetries], 222 | 1 AS [ActiveFlag], 223 | 1 AS [L1TransformationReqdFlag], 224 | 1 AS [L2TransformationReqdFlag], 225 | 1 AS [DelayL1TransformationFlag], 226 | 1 AS [DelayL2TransformationFlag] 227 | 228 | UNION 229 | --Collections 230 | SELECT 'Purview' AS [SourceSystemName], 231 | 'collections' AS [StreamName], 232 | 'Microsoft Purview APIs' AS [SourceSystemDescription], 233 | 'ATLAS REST API' AS [Backend], 234 | '/collections?api-version=2019-11-01-preview' AS [EntityName], 235 | NULL AS [WatermarkColName], 236 | NULL AS [LastDeltaDate], 237 | NULL AS [LastDeltaNumber], 238 | NULL AS [LastDeltaString], 239 | NULL AS [MaxIntervalMinutes], 240 | NULL AS [MaxIntervalNumber], 241 | '{ 242 | "type": "TabularTranslator", 243 | "mappings": [ 244 | { 245 | "source": { 246 | "path": "[''name'']" 247 | }, 248 | "sink": { 249 | "name": "name", 250 | "type": "String" 251 | } 252 | }, 253 | { 254 | "source": { 255 | "path": "[''friendlyName'']" 256 | }, 257 | "sink": { 258 | "name": "friendlyName", 259 | "type": "String" 260 | } 261 | }, 262 | { 263 | "source": { 264 | "path": "[''description'']" 265 | }, 266 | "sink": { 267 | "name": "description", 268 | "type": "String" 269 | } 270 | }, 271 | { 272 | "source": { 273 | "path": "[''systemData''][''createdBy'']" 274 | }, 275 | "sink": { 276 | "name": "createdBy", 277 | "type": "String" 278 | } 279 | }, 280 | { 281 | "source": { 282 | "path": "[''systemData''][''createdByType'']" 283 | }, 284 | "sink": { 285 | "name": "createdByType", 286 | "type": "String" 287 | } 288 | }, 289 | { 290 | "source": { 291 | "path": "[''systemData''][''createdAt'']" 292 | }, 293 | "sink": { 294 | "name": "createdAt", 295 | "type": "DateTime" 296 | } 297 | }, 298 | { 299 | "source": { 300 | "path": "[''systemData''][''lastModifiedByType'']" 301 | }, 302 | "sink": { 303 | "name": "lastModifiedByType", 304 | "type": "String" 305 | } 306 | }, 307 | { 308 | "source": { 309 | "path": "[''systemData''][''lastModifiedAt'']" 310 | }, 311 | "sink": { 312 | "name": "lastModifiedAt", 313 | "type": "DateTime" 314 | } 315 | }, 316 | { 317 | "source": { 318 | "path": "[''collectionProvisioningState'']" 319 | }, 320 | "sink": { 321 | "name": "collectionProvisioningState", 322 | "type": "String" 323 | } 324 | }, 325 | { 326 | "source": { 327 | "path": "[''parentCollection''][''type'']" 328 | }, 329 | "sink": { 330 | "name": "parentCollectionType", 331 | "type": "String" 332 | } 333 | }, 334 | { 335 | "source": { 336 | "path": "[''parentCollection''][''referenceName'']" 337 | }, 338 | "sink": { 339 | "name": "parentCollectionReferenceName", 340 | "type": "String" 341 | } 342 | } 343 | ], 344 | "collectionReference": "$[''value'']", 345 | "mapComplexValuesToString": true 346 | }' AS [DataMapping], 347 | 'raw-bronze' AS [DestinationRawFileSystem], 348 | 'purview/collections/YYYY-MM' AS [DestinationRawFolder] , 349 | 'collections_YYYYMMDD.parquet' AS [DestinationRawFile], 350 | 2 AS [RunSequence], 351 | 3 AS [MaxRetries], 352 | 1 AS [ActiveFlag], 353 | 1 AS [L1TransformationReqdFlag], 354 | 1 AS [L2TransformationReqdFlag], 355 | 1 AS [DelayL1TransformationFlag], 356 | 1 AS [DelayL2TransformationFlag] 357 | ; 358 | 359 | 360 | --Merge with Temp table for re-runnability 361 | 362 | MERGE INTO [ELT].[IngestDefinition] AS tgt 363 | USING #PurviewRestAPI_Ingest AS src 364 | ON src.[SourceSystemName]=tgt.[SourceSystemName] AND src.[StreamName] =tgt.[StreamName] 365 | WHEN MATCHED THEN 366 | UPDATE SET tgt.[SourceSystemDescription] = src.[SourceSystemDescription], 367 | tgt.[Backend] = src.[Backend], 368 | tgt.[EntityName] = src.[EntityName], 369 | tgt.[WatermarkColName] = src.[WatermarkColName], 370 | tgt.[LastDeltaDate] = src.[LastDeltaDate], 371 | tgt.[LastDeltaNumber] = src.[LastDeltaNumber], 372 | tgt.[LastDeltaString] = src.[LastDeltaString], 373 | tgt.[MaxIntervalMinutes] = src.[MaxIntervalMinutes], 374 | tgt.[MaxIntervalNumber] = src.[MaxIntervalNumber], 375 | tgt.[DataMapping] = src.[DataMapping], 376 | tgt.[DestinationRawFileSystem] = src.[DestinationRawFileSystem], 377 | tgt.[DestinationRawFolder] = src.[DestinationRawFolder], 378 | tgt.[DestinationRawFile] = src.[DestinationRawFile], 379 | tgt.[RunSequence] = src.[RunSequence], 380 | tgt.[MaxRetries] = src.[MaxRetries], 381 | tgt.[ActiveFlag] = src.[ActiveFlag], 382 | tgt.[L1TransformationReqdFlag] = src.[L1TransformationReqdFlag], 383 | tgt.[L2TransformationReqdFlag] = src.[L2TransformationReqdFlag], 384 | tgt.[DelayL1TransformationFlag] = src.[DelayL1TransformationFlag], 385 | tgt.[DelayL2TransformationFlag] = src.[DelayL2TransformationFlag], 386 | tgt.[ModifiedBy] = USER_NAME(), 387 | tgt.[ModifiedTimestamp] = GetDate() 388 | WHEN NOT MATCHED BY TARGET THEN 389 | INSERT([SourceSystemName], 390 | [StreamName], 391 | [SourceSystemDescription], 392 | [Backend], 393 | [EntityName], 394 | [WatermarkColName], 395 | [LastDeltaDate], 396 | [LastDeltaNumber], 397 | [LastDeltaString], 398 | [MaxIntervalMinutes], 399 | [MaxIntervalNumber], 400 | [DataMapping], 401 | [DestinationRawFileSystem], 402 | [DestinationRawFolder], 403 | [DestinationRawFile], 404 | [RunSequence], 405 | [MaxRetries], 406 | [ActiveFlag], 407 | [L1TransformationReqdFlag], 408 | [L2TransformationReqdFlag], 409 | [DelayL1TransformationFlag], 410 | [DelayL2TransformationFlag], 411 | [CreatedBy], 412 | [CreatedTimestamp] ) 413 | VALUES (src.[SourceSystemName], 414 | src.[StreamName], 415 | src.[SourceSystemDescription], 416 | src.[Backend], 417 | src.[EntityName], 418 | src.[WatermarkColName], 419 | src.[LastDeltaDate], 420 | src.[LastDeltaNumber], 421 | src.[LastDeltaString], 422 | src.[MaxIntervalMinutes], 423 | src.[MaxIntervalNumber], 424 | src.[DataMapping], 425 | src.[DestinationRawFileSystem], 426 | src.[DestinationRawFolder], 427 | src.[DestinationRawFile], 428 | src.[RunSequence], 429 | src.[MaxRetries], 430 | src.[ActiveFlag], 431 | src.[L1TransformationReqdFlag], 432 | src.[L2TransformationReqdFlag], 433 | src.[DelayL1TransformationFlag], 434 | src.[DelayL2TransformationFlag], 435 | USER_NAME(), 436 | GETDATE()); 437 | 438 | GO -------------------------------------------------------------------------------- /elt-framework/ControlDB/Post Deployment Script/datasources/RestAPI/L1TransformDefinition/L1T_AzureRestAPI.sql: -------------------------------------------------------------------------------- 1 | /* 2 | ELT Configuration for REST API Data Sources 3 | */ 4 | Declare @SourceSystem VARCHAR(50), @StreamName VARCHAR(100) 5 | SET @SourceSystem='Azure' 6 | SET @StreamName ='%' 7 | 8 | IF OBJECT_ID('tempdb..#AzureRestAPI_L1') IS NOT NULL DROP TABLE #AzureRestAPI_L1; 9 | --Create Temp table with same structure as L1TransformDefinition 10 | CREATE TABLE #AzureRestAPI_L1 11 | ( 12 | [IngestID] int not null, 13 | [ComputePath] varchar(200) null, 14 | [ComputeName] varchar(100) null, 15 | [CustomParameters] varchar(max) null, 16 | [InputRawFileSystem] varchar(50) not null, 17 | [InputRawFileFolder] varchar(200) not null, 18 | [InputRawFile] varchar(200) not null, 19 | [InputRawFileDelimiter] char(1) null, 20 | [InputFileHeaderFlag] bit null, 21 | [OutputL1CurateFileSystem] varchar(50) not null, 22 | [OutputL1CuratedFolder] varchar(200) not null, 23 | [OutputL1CuratedFile] varchar(200) not null, 24 | [OutputL1CuratedFileDelimiter] char(1) null, 25 | [OutputL1CuratedFileFormat] varchar(10) null, 26 | [OutputL1CuratedFileWriteMode] varchar(20) null, 27 | [OutputDWStagingTable] varchar(200) null, 28 | [LookupColumns] varchar(4000) null, 29 | [OutputDWTable] varchar(200) null, 30 | [OutputDWTableWriteMode] varchar(20) null, 31 | [MaxRetries] int null, 32 | [WatermarkColName] varchar(50) null, 33 | [ActiveFlag] bit not null 34 | ); 35 | 36 | --Insert Into Temp Table 37 | INSERT INTO #AzureRestAPI_L1 38 | SELECT [IngestID] 39 | ,'L1Transform' AS [ComputePath] 40 | ,'L1Transform-Generic-Synapse' AS [ComputeName] 41 | , NULL AS [CustomParameters] 42 | ,[DestinationRawFileSystem] AS [InputRawFileSystem] 43 | ,[DestinationRawFolder] AS [InputRawFileFolder] 44 | ,[DestinationRawFile] AS [InputRawFile] 45 | , NULL AS [InputRawFileDelimiter] 46 | , 1 AS [InputFileHeaderFlag] 47 | , 'curated-silver' AS [OutputL1CurateFileSystem] 48 | , [DestinationRawFolder] AS [OutputL1CuratedFolder] 49 | , 'standardized_'+ [DestinationRawFile] AS [OutputL1CuratedFile] 50 | , NULL AS [OutputL1CuratedFileDelimiter] 51 | , 'parquet' AS [OutputL1CuratedFileFormat] 52 | , 'overwrite' AS [OutputL1CuratedFileWriteMode] 53 | , 'stg.merge_' + [SourceSystemName] +'_'+ StreamName AS [OutputDWStagingTable] 54 | , CASE WHEN StreamName = 'operations' THEN '[''name'']' 55 | ELSE NULL 56 | END AS [LookupColumns] 57 | , SourceSystemName + '.' + StreamName AS [OutputDWTable] 58 | , 'append' AS [OutputDWTableWriteMode] 59 | ,3 AS [MaxRetries] 60 | , NULL AS [WatermarkColName] 61 | , 1 AS [ActiveFlag] 62 | FROM [ELT].[IngestDefinition] 63 | WHERE [SourceSystemName]=@SourceSystem 64 | AND [StreamName] like @StreamName; 65 | 66 | 67 | --Merge with Temp table for re-runnability 68 | 69 | MERGE INTO [ELT].[L1TransformDefinition] AS tgt 70 | USING #AzureRestAPI_L1 AS src 71 | ON src.[InputRawFileSystem] = tgt.[InputRawFileSystem] 72 | AND src.[InputRawFileFolder] = tgt.[InputRawFileFolder] 73 | AND src.[InputRawFile] = tgt.[InputRawFile] 74 | AND src.[OutputL1CurateFileSystem] = tgt.[OutputL1CurateFileSystem] 75 | AND src.[OutputL1CuratedFolder] = tgt.[OutputL1CuratedFolder] 76 | AND src.[OutputL1CuratedFile] = tgt.[OutputL1CuratedFile] 77 | WHEN MATCHED THEN 78 | UPDATE SET tgt.[IngestID] =src.[IngestID], 79 | tgt.[ComputePath] =src.[ComputePath], 80 | tgt.[ComputeName] =src.[ComputeName], 81 | tgt.[CustomParameters] =src.[CustomParameters], 82 | tgt.[InputRawFileSystem] =src.[InputRawFileSystem], 83 | tgt.[InputRawFileFolder] =src.[InputRawFileFolder], 84 | tgt.[InputRawFile] =src.[InputRawFile], 85 | tgt.[InputRawFileDelimiter] =src.[InputRawFileDelimiter], 86 | tgt.[InputFileHeaderFlag] =src.[InputFileHeaderFlag], 87 | tgt.[OutputL1CurateFileSystem] =src.[OutputL1CurateFileSystem], 88 | tgt.[OutputL1CuratedFolder] =src.[OutputL1CuratedFolder], 89 | tgt.[OutputL1CuratedFile] =src.[OutputL1CuratedFile], 90 | tgt.[OutputL1CuratedFileDelimiter] =src.[OutputL1CuratedFileDelimiter], 91 | tgt.[OutputL1CuratedFileFormat] =src.[OutputL1CuratedFileFormat], 92 | tgt.[OutputL1CuratedFileWriteMode] =src.[OutputL1CuratedFileWriteMode], 93 | tgt.[OutputDWStagingTable] =src.[OutputDWStagingTable], 94 | tgt.[LookupColumns] =src.[LookupColumns], 95 | tgt.[OutputDWTable] =src.[OutputDWTable], 96 | tgt.[OutputDWTableWriteMode] =src.[OutputDWTableWriteMode], 97 | tgt.[MaxRetries] =src.[MaxRetries], 98 | tgt.[WatermarkColName] =src.[WatermarkColName], 99 | tgt.[ActiveFlag] =src.[ActiveFlag], 100 | tgt.[ModifiedBy] = USER_NAME(), 101 | tgt.[ModifiedTimestamp] = GetDate() 102 | WHEN NOT MATCHED BY TARGET THEN 103 | INSERT([IngestID], 104 | [ComputePath], 105 | [ComputeName], 106 | [CustomParameters], 107 | [InputRawFileSystem], 108 | [InputRawFileFolder], 109 | [InputRawFile], 110 | [InputRawFileDelimiter], 111 | [InputFileHeaderFlag], 112 | [OutputL1CurateFileSystem], 113 | [OutputL1CuratedFolder], 114 | [OutputL1CuratedFile], 115 | [OutputL1CuratedFileDelimiter], 116 | [OutputL1CuratedFileFormat], 117 | [OutputL1CuratedFileWriteMode], 118 | [OutputDWStagingTable], 119 | [LookupColumns], 120 | [OutputDWTable], 121 | [OutputDWTableWriteMode], 122 | [MaxRetries], 123 | [WatermarkColName], 124 | [ActiveFlag], 125 | [CreatedBy], 126 | [CreatedTimestamp] ) 127 | VALUES (src.[IngestID], 128 | src.[ComputePath], 129 | src.[ComputeName], 130 | src.[CustomParameters], 131 | src.[InputRawFileSystem], 132 | src.[InputRawFileFolder], 133 | src.[InputRawFile], 134 | src.[InputRawFileDelimiter], 135 | src.[InputFileHeaderFlag], 136 | src.[OutputL1CurateFileSystem], 137 | src.[OutputL1CuratedFolder], 138 | src.[OutputL1CuratedFile], 139 | src.[OutputL1CuratedFileDelimiter], 140 | src.[OutputL1CuratedFileFormat], 141 | src.[OutputL1CuratedFileWriteMode], 142 | src.[OutputDWStagingTable], 143 | src.[LookupColumns], 144 | src.[OutputDWTable], 145 | src.[OutputDWTableWriteMode], 146 | src.[MaxRetries], 147 | src.[WatermarkColName], 148 | src.[ActiveFlag], 149 | USER_NAME(), 150 | GETDATE()); 151 | 152 | GO -------------------------------------------------------------------------------- /elt-framework/ControlDB/Post Deployment Script/datasources/RestAPI/L1TransformDefinition/L1T_PurviewRestAPI.sql: -------------------------------------------------------------------------------- 1 | /* 2 | ELT Configuration for REST API Data Sources 3 | */ 4 | Declare @SourceSystem VARCHAR(50), @StreamName VARCHAR(100) 5 | SET @SourceSystem='Purview' 6 | SET @StreamName ='%' 7 | 8 | IF OBJECT_ID('tempdb..#PurviewRestAPI_L1') IS NOT NULL DROP TABLE #PurviewRestAPI_L1; 9 | --Create Temp table with same structure as L1TransformDefinition 10 | CREATE TABLE #PurviewRestAPI_L1 11 | ( 12 | [IngestID] int not null, 13 | [ComputePath] varchar(200) null, 14 | [ComputeName] varchar(100) null, 15 | [CustomParameters] varchar(max) null, 16 | [InputRawFileSystem] varchar(50) not null, 17 | [InputRawFileFolder] varchar(200) not null, 18 | [InputRawFile] varchar(200) not null, 19 | [InputRawFileDelimiter] char(1) null, 20 | [InputFileHeaderFlag] bit null, 21 | [OutputL1CurateFileSystem] varchar(50) not null, 22 | [OutputL1CuratedFolder] varchar(200) not null, 23 | [OutputL1CuratedFile] varchar(200) not null, 24 | [OutputL1CuratedFileDelimiter] char(1) null, 25 | [OutputL1CuratedFileFormat] varchar(10) null, 26 | [OutputL1CuratedFileWriteMode] varchar(20) null, 27 | [OutputDWStagingTable] varchar(200) null, 28 | [LookupColumns] varchar(4000) null, 29 | [OutputDWTable] varchar(200) null, 30 | [OutputDWTableWriteMode] varchar(20) null, 31 | [MaxRetries] int null, 32 | [WatermarkColName] varchar(50) null, 33 | [ActiveFlag] bit not null 34 | ); 35 | 36 | --Insert Into Temp Table 37 | INSERT INTO #PurviewRestAPI_L1 38 | SELECT [IngestID] 39 | ,'L1Transform' AS [ComputePath] 40 | ,'L1Transform-Generic-Synapse' AS [ComputeName] 41 | , NULL AS [CustomParameters] 42 | ,[DestinationRawFileSystem] AS [InputRawFileSystem] 43 | ,[DestinationRawFolder] AS [InputRawFileFolder] 44 | ,[DestinationRawFile] AS [InputRawFile] 45 | , NULL AS [InputRawFileDelimiter] 46 | , 1 AS [InputFileHeaderFlag] 47 | , 'curated-silver' AS [OutputL1CurateFileSystem] 48 | , [DestinationRawFolder] AS [OutputL1CuratedFolder] 49 | , 'standardized_'+ [DestinationRawFile] AS [OutputL1CuratedFile] 50 | , NULL AS [OutputL1CuratedFileDelimiter] 51 | , 'parquet' AS [OutputL1CuratedFileFormat] 52 | , 'overwrite' AS [OutputL1CuratedFileWriteMode] 53 | , 'stg.merge_' + [SourceSystemName] +'_'+ StreamName AS [OutputDWStagingTable] 54 | , CASE WHEN StreamName = 'datasources' THEN '[''resourceName'']' 55 | WHEN StreamName = 'collections' THEN '[''name'']' 56 | END AS [LookupColumns] 57 | , SourceSystemName + '.' + StreamName AS [OutputDWTable] 58 | , 'append' AS [OutputDWTableWriteMode] 59 | ,3 AS [MaxRetries] 60 | , CASE WHEN StreamName = 'datasources' THEN 'lastModifiedAt' 61 | WHEN StreamName = 'collections' THEN 'lastModifiedAt' 62 | END AS [WatermarkColName] 63 | , 1 AS [ActiveFlag] 64 | FROM [ELT].[IngestDefinition] 65 | WHERE [SourceSystemName]=@SourceSystem 66 | AND [StreamName] like @StreamName; 67 | 68 | 69 | --Merge with Temp table for re-runnability 70 | 71 | MERGE INTO [ELT].[L1TransformDefinition] AS tgt 72 | USING #PurviewRestAPI_L1 AS src 73 | ON src.[InputRawFileSystem] = tgt.[InputRawFileSystem] 74 | AND src.[InputRawFileFolder] = tgt.[InputRawFileFolder] 75 | AND src.[InputRawFile] = tgt.[InputRawFile] 76 | AND src.[OutputL1CurateFileSystem] = tgt.[OutputL1CurateFileSystem] 77 | AND src.[OutputL1CuratedFolder] = tgt.[OutputL1CuratedFolder] 78 | AND src.[OutputL1CuratedFile] = tgt.[OutputL1CuratedFile] 79 | WHEN MATCHED THEN 80 | UPDATE SET tgt.[IngestID] =src.[IngestID], 81 | tgt.[ComputePath] =src.[ComputePath], 82 | tgt.[ComputeName] =src.[ComputeName], 83 | tgt.[CustomParameters] =src.[CustomParameters], 84 | tgt.[InputRawFileSystem] =src.[InputRawFileSystem], 85 | tgt.[InputRawFileFolder] =src.[InputRawFileFolder], 86 | tgt.[InputRawFile] =src.[InputRawFile], 87 | tgt.[InputRawFileDelimiter] =src.[InputRawFileDelimiter], 88 | tgt.[InputFileHeaderFlag] =src.[InputFileHeaderFlag], 89 | tgt.[OutputL1CurateFileSystem] =src.[OutputL1CurateFileSystem], 90 | tgt.[OutputL1CuratedFolder] =src.[OutputL1CuratedFolder], 91 | tgt.[OutputL1CuratedFile] =src.[OutputL1CuratedFile], 92 | tgt.[OutputL1CuratedFileDelimiter] =src.[OutputL1CuratedFileDelimiter], 93 | tgt.[OutputL1CuratedFileFormat] =src.[OutputL1CuratedFileFormat], 94 | tgt.[OutputL1CuratedFileWriteMode] =src.[OutputL1CuratedFileWriteMode], 95 | tgt.[OutputDWStagingTable] =src.[OutputDWStagingTable], 96 | tgt.[LookupColumns] =src.[LookupColumns], 97 | tgt.[OutputDWTable] =src.[OutputDWTable], 98 | tgt.[OutputDWTableWriteMode] =src.[OutputDWTableWriteMode], 99 | tgt.[MaxRetries] =src.[MaxRetries], 100 | tgt.[WatermarkColName] =src.[WatermarkColName], 101 | tgt.[ActiveFlag] =src.[ActiveFlag], 102 | tgt.[ModifiedBy] = USER_NAME(), 103 | tgt.[ModifiedTimestamp] = GetDate() 104 | WHEN NOT MATCHED BY TARGET THEN 105 | INSERT([IngestID], 106 | [ComputePath], 107 | [ComputeName], 108 | [CustomParameters], 109 | [InputRawFileSystem], 110 | [InputRawFileFolder], 111 | [InputRawFile], 112 | [InputRawFileDelimiter], 113 | [InputFileHeaderFlag], 114 | [OutputL1CurateFileSystem], 115 | [OutputL1CuratedFolder], 116 | [OutputL1CuratedFile], 117 | [OutputL1CuratedFileDelimiter], 118 | [OutputL1CuratedFileFormat], 119 | [OutputL1CuratedFileWriteMode], 120 | [OutputDWStagingTable], 121 | [LookupColumns], 122 | [OutputDWTable], 123 | [OutputDWTableWriteMode], 124 | [MaxRetries], 125 | [WatermarkColName], 126 | [ActiveFlag], 127 | [CreatedBy], 128 | [CreatedTimestamp] ) 129 | VALUES (src.[IngestID], 130 | src.[ComputePath], 131 | src.[ComputeName], 132 | src.[CustomParameters], 133 | src.[InputRawFileSystem], 134 | src.[InputRawFileFolder], 135 | src.[InputRawFile], 136 | src.[InputRawFileDelimiter], 137 | src.[InputFileHeaderFlag], 138 | src.[OutputL1CurateFileSystem], 139 | src.[OutputL1CuratedFolder], 140 | src.[OutputL1CuratedFile], 141 | src.[OutputL1CuratedFileDelimiter], 142 | src.[OutputL1CuratedFileFormat], 143 | src.[OutputL1CuratedFileWriteMode], 144 | src.[OutputDWStagingTable], 145 | src.[LookupColumns], 146 | src.[OutputDWTable], 147 | src.[OutputDWTableWriteMode], 148 | src.[MaxRetries], 149 | src.[WatermarkColName], 150 | src.[ActiveFlag], 151 | USER_NAME(), 152 | GETDATE()); 153 | 154 | GO -------------------------------------------------------------------------------- /elt-framework/ControlDB/Post Deployment Script/datasources/UploadFiles/IngestDefinition/CustomFormRecognizerModel.sql: -------------------------------------------------------------------------------- 1 | /* 2 | ELT Configuration for CustomFormRecognizerModel JSON Data Source 3 | */ 4 | 5 | IF OBJECT_ID('tempdb..#CFRM_Ingest') IS NOT NULL DROP TABLE #CFRM_Ingest; 6 | 7 | --Create Temp table with similar schema as IngestDefinition 8 | CREATE TABLE #CFRM_Ingest 9 | ( 10 | [SourceSystemName] [varchar](50) NOT NULL, 11 | [StreamName] [varchar](100) NULL, 12 | [SourceSystemDescription] [varchar](200) NULL, 13 | [Backend] [varchar](30) NULL, 14 | [DataFormat] [varchar](10) NULL, 15 | [SourceFileDropFileSystem] [varchar](50) NULL, 16 | [SourceFileDropFolder] [varchar](200) NULL, 17 | [SourceFileDropFile] [varchar](200) NULL, 18 | [DestinationRawFileSystem] [varchar](50) NULL, 19 | [DestinationRawFolder] [varchar](200) NULL, 20 | [DestinationRawFile] [varchar](200) NULL, 21 | [ActiveFlag] [bit] NOT NULL, 22 | [L1TransformationReqdFlag] [bit] NOT NULL, 23 | [L2TransformationReqdFlag] [bit] NOT NULL, 24 | [DelayL1TransformationFlag] [bit] NOT NULL, 25 | [DelayL2TransformationFlag] [bit] NOT NULL 26 | ); 27 | 28 | 29 | --Insert Into Temp Table 30 | INSERT INTO #CFRM_Ingest 31 | --Operations 32 | SELECT 'AIML-OCR' AS [SourceSystemName], 33 | 'restatements' AS [StreamName], 34 | 'Custom Document Intelligence Model output for Real Estate Statements' AS [SourceSystemDescription], 35 | 'Azure AI DocI REST API' AS [Backend], 36 | 'JSON' AS [DataFormat], 37 | 'doci' AS [SourceFileDropFileSystem], 38 | 'inference/re-statements' AS [SourceFileDropFolder], 39 | '*.json' AS [SourceFileDropFile], 40 | 'raw-bronze' AS [DestinationRawFileSystem], 41 | 're-statements/YYYY/MM' AS [DestinationRawFolder], 42 | 'YYYY-MM-DD_HHMISS_re-statement.json' AS [DestinationRawFile], 43 | 1 AS [ActiveFlag], 44 | 1 AS [L1TransformationReqdFlag], 45 | 0 AS [L2TransformationReqdFlag], 46 | 0 AS [DelayL1TransformationFlag], 47 | 1 AS [DelayL2TransformationFlag] 48 | UNION 49 | SELECT 'AIML-OCR' AS [SourceSystemName], 50 | 'form10q' AS [StreamName], 51 | 'Custom Document Intelligence Model output for Form 10-Q' AS [SourceSystemDescription], 52 | 'Azure AI DocI REST API' AS [Backend], 53 | 'JSON' AS [DataFormat], 54 | 'doci' AS [SourceFileDropFileSystem], 55 | 'inference/form10q' AS [SourceFileDropFolder], 56 | '*.json' AS [SourceFileDropFile], 57 | 'raw-bronze' AS [DestinationRawFileSystem], 58 | 'form10q/YYYY/MM' AS [DestinationRawFolder], 59 | 'YYYY-MM-DD_HHMISS_form10q.json' AS [DestinationRawFile], 60 | 1 AS [ActiveFlag], 61 | 1 AS [L1TransformationReqdFlag], 62 | 0 AS [L2TransformationReqdFlag], 63 | 0 AS [DelayL1TransformationFlag], 64 | 1 AS [DelayL2TransformationFlag] 65 | 66 | --Merge with Temp table for re-runnability 67 | 68 | MERGE INTO [ELT].[IngestDefinition] AS tgt 69 | USING #CFRM_Ingest AS src 70 | ON src.[SourceSystemName]=tgt.[SourceSystemName] AND src.[StreamName]=tgt.[StreamName] 71 | WHEN MATCHED THEN 72 | UPDATE SET tgt.[SourceSystemDescription] = src.[SourceSystemDescription], 73 | tgt.[Backend] = src.[Backend], 74 | tgt.[DataFormat] = src.[DataFormat], 75 | tgt.[SourceFileDropFileSystem] = src.[SourceFileDropFileSystem], 76 | tgt.[SourceFileDropFolder] =src.[SourceFileDropFolder], 77 | tgt.[SourceFileDropFile] = src.[SourceFileDropFile], 78 | tgt.[DestinationRawFileSystem] = src.[DestinationRawFileSystem], 79 | tgt.[DestinationRawFolder] = src.[DestinationRawFolder], 80 | tgt.[DestinationRawFile] = src.[DestinationRawFile], 81 | tgt.[ActiveFlag] = src.[ActiveFlag], 82 | tgt.[L1TransformationReqdFlag] = src.[L1TransformationReqdFlag], 83 | tgt.[L2TransformationReqdFlag] = src.[L2TransformationReqdFlag], 84 | tgt.[DelayL1TransformationFlag] = src.[DelayL1TransformationFlag], 85 | tgt.[DelayL2TransformationFlag] = src.[DelayL2TransformationFlag], 86 | tgt.[ModifiedBy] = USER_NAME(), 87 | tgt.[ModifiedTimestamp] = GetDate() 88 | WHEN NOT MATCHED BY TARGET THEN 89 | INSERT( [SourceSystemName], 90 | [StreamName], 91 | [SourceSystemDescription], 92 | [Backend], 93 | [DataFormat], 94 | [SourceFileDropFileSystem], 95 | [SourceFileDropFolder], 96 | [SourceFileDropFile], 97 | [DestinationRawFileSystem], 98 | [DestinationRawFolder], 99 | [DestinationRawFile], 100 | [ActiveFlag], 101 | [L1TransformationReqdFlag], 102 | [L2TransformationReqdFlag], 103 | [DelayL1TransformationFlag], 104 | [DelayL2TransformationFlag], 105 | [ModifiedBy], 106 | [ModifiedTimestamp]) 107 | VALUES (src.[SourceSystemName], 108 | src.[StreamName], 109 | src.[SourceSystemDescription], 110 | src.[Backend], 111 | src.[DataFormat], 112 | src.[SourceFileDropFileSystem], 113 | src.[SourceFileDropFolder], 114 | src.[SourceFileDropFile], 115 | src.[DestinationRawFileSystem], 116 | src.[DestinationRawFolder], 117 | src.[DestinationRawFile], 118 | src.[ActiveFlag], 119 | src.[L1TransformationReqdFlag], 120 | src.[L2TransformationReqdFlag], 121 | src.[DelayL1TransformationFlag], 122 | src.[DelayL2TransformationFlag], 123 | USER_NAME(), 124 | GETDATE()); -------------------------------------------------------------------------------- /elt-framework/ControlDB/Post Deployment Script/datasources/UploadFiles/L1TransformDefinition/L1T_CustomFormRecognizerModel.sql: -------------------------------------------------------------------------------- 1 | /* 2 | ELT Configuration for CustomFormRecognizerModel JSON Data Source 3 | */ 4 | Declare @SourceSystem VARCHAR(50), @StreamName VARCHAR(100) 5 | SET @SourceSystem='AIML-OCR' 6 | SET @StreamName ='%' 7 | 8 | IF OBJECT_ID('tempdb..#CFRM_L1') IS NOT NULL DROP TABLE #CFRM_L1; 9 | --Create Temp table with same structure as L1TransformDefinition 10 | CREATE TABLE #CFRM_L1 11 | ( 12 | [IngestID] int not null, 13 | [ComputePath] varchar(200) null, 14 | [ComputeName] varchar(100) null, 15 | [InputRawFileSystem] varchar(50) not null, 16 | [InputRawFileFolder] varchar(200) not null, 17 | [InputRawFile] varchar(200) not null, 18 | [OutputL1CurateFileSystem] varchar(50) not null, 19 | [OutputL1CuratedFolder] varchar(200) not null, 20 | [OutputL1CuratedFile] varchar(200) not null, 21 | [OutputL1CuratedFileFormat] varchar(10) null, 22 | [OutputL1CuratedFileWriteMode] varchar(20) null, 23 | [OutputDWStagingTable] varchar(200) null, 24 | [LookupColumns] varchar(4000) null, 25 | [OutputDWTable] varchar(200) null, 26 | [OutputDWTableWriteMode] varchar(20) null, 27 | [ActiveFlag] bit not null 28 | ); 29 | 30 | --Insert Into Temp Table 31 | INSERT INTO #CFRM_L1 32 | SELECT [IngestID] 33 | ,'L1Transform' AS [ComputePath] 34 | ,'L1Transform-ReStatement' AS [ComputeName] 35 | ,[DestinationRawFileSystem] AS [InputRawFileSystem] 36 | ,[DestinationRawFolder] AS [InputRawFileFolder] 37 | ,[DestinationRawFile] AS [InputRawFile] 38 | , 'curated-silver' AS [OutputL1CurateFileSystem] 39 | , [DestinationRawFolder] AS [OutputL1CuratedFolder] 40 | , 'standardized_'+ [DestinationRawFile] AS [OutputL1CuratedFile] 41 | , 'json' AS [OutputL1CuratedFileFormat] 42 | , 'overwrite' AS [OutputL1CuratedFileWriteMode] 43 | , null as [OutputDWStagingTable] 44 | , null as [LookupColumns] 45 | , null as [OutputDWTable] 46 | , null as [OutputDWTableWriteMode] 47 | , 1 AS [ActiveFlag] 48 | FROM [ELT].[IngestDefinition] 49 | WHERE [SourceSystemName]=@SourceSystem 50 | AND [StreamName] ='restatements' 51 | 52 | UNION 53 | SELECT [IngestID] 54 | ,'L1Transform' AS [ComputePath] 55 | ,'L1Transform-SEC-Form10Q' AS [ComputeName] 56 | ,[DestinationRawFileSystem] AS [InputRawFileSystem] 57 | ,[DestinationRawFolder] AS [InputRawFileFolder] 58 | ,[DestinationRawFile] AS [InputRawFile] 59 | , 'curated-silver' AS [OutputL1CurateFileSystem] 60 | , [DestinationRawFolder] AS [OutputL1CuratedFolder] 61 | , 'standardized_'+ [DestinationRawFile] AS [OutputL1CuratedFile] 62 | , 'json' AS [OutputL1CuratedFileFormat] 63 | , 'overwrite' AS [OutputL1CuratedFileWriteMode] 64 | , '[stg].[merge_sec_form10q]' as [OutputDWStagingTable] 65 | , '[''org_name'',''reporting_quarter'']' as [LookupColumns] 66 | , '[sec].[form10q]' as [OutputDWTable] 67 | , 'append' as [OutputDWTableWriteMode] 68 | , 1 AS [ActiveFlag] 69 | FROM [ELT].[IngestDefinition] 70 | WHERE [SourceSystemName]=@SourceSystem 71 | AND [StreamName] ='form10q' 72 | --Merge with Temp table for re-runnability 73 | 74 | MERGE INTO [ELT].[L1TransformDefinition] AS tgt 75 | USING #CFRM_L1 AS src 76 | ON src.[InputRawFileSystem] = tgt.[InputRawFileSystem] 77 | AND src.[InputRawFileFolder] = tgt.[InputRawFileFolder] 78 | AND src.[InputRawFile] = tgt.[InputRawFile] 79 | AND src.[OutputL1CurateFileSystem] = tgt.[OutputL1CurateFileSystem] 80 | AND src.[OutputL1CuratedFolder] = tgt.[OutputL1CuratedFolder] 81 | AND src.[OutputL1CuratedFile] = tgt.[OutputL1CuratedFile] 82 | WHEN MATCHED THEN 83 | UPDATE SET tgt.[IngestID] =src.[IngestID], 84 | tgt.[ComputePath] =src.[ComputePath], 85 | tgt.[ComputeName] =src.[ComputeName], 86 | tgt.[InputRawFileSystem] =src.[InputRawFileSystem], 87 | tgt.[InputRawFileFolder] =src.[InputRawFileFolder], 88 | tgt.[InputRawFile] =src.[InputRawFile], 89 | tgt.[OutputL1CurateFileSystem] =src.[OutputL1CurateFileSystem], 90 | tgt.[OutputL1CuratedFolder] =src.[OutputL1CuratedFolder], 91 | tgt.[OutputL1CuratedFile] =src.[OutputL1CuratedFile], 92 | tgt.[OutputL1CuratedFileFormat] =src.[OutputL1CuratedFileFormat], 93 | tgt.[OutputL1CuratedFileWriteMode] =src.[OutputL1CuratedFileWriteMode], 94 | tgt.[OutputDWStagingTable] = src.[OutputDWStagingTable], 95 | tgt.[LookupColumns] = src.[LookupColumns], 96 | tgt.[OutputDWTable] = src.[OutputDWTable], 97 | tgt.[OutputDWTableWriteMode]=src.[OutputDWTableWriteMode], 98 | tgt.[ActiveFlag] =src.[ActiveFlag], 99 | tgt.[ModifiedBy] = USER_NAME(), 100 | tgt.[ModifiedTimestamp] = GetDate() 101 | WHEN NOT MATCHED BY TARGET THEN 102 | INSERT([IngestID], 103 | [ComputePath], 104 | [ComputeName], 105 | [InputRawFileSystem], 106 | [InputRawFileFolder], 107 | [InputRawFile], 108 | [OutputL1CurateFileSystem], 109 | [OutputL1CuratedFolder], 110 | [OutputL1CuratedFile], 111 | [OutputL1CuratedFileFormat], 112 | [OutputL1CuratedFileWriteMode], 113 | [ActiveFlag], 114 | [CreatedBy], 115 | [CreatedTimestamp] ) 116 | VALUES (src.[IngestID], 117 | src.[ComputePath], 118 | src.[ComputeName], 119 | src.[InputRawFileSystem], 120 | src.[InputRawFileFolder], 121 | src.[InputRawFile], 122 | src.[OutputL1CurateFileSystem], 123 | src.[OutputL1CuratedFolder], 124 | src.[OutputL1CuratedFile], 125 | src.[OutputL1CuratedFileFormat], 126 | src.[OutputL1CuratedFileWriteMode], 127 | src.[ActiveFlag], 128 | USER_NAME(), 129 | GETDATE()); 130 | 131 | GO -------------------------------------------------------------------------------- /elt-framework/ControlDB/Pre Deployment Script/DBRoles.PreDeployment.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Pre-Deployment Script Template 3 | -------------------------------------------------------------------------------------- 4 | This file contains SQL statements that will be executed before the build script. 5 | Use SQLCMD syntax to include a file in the pre-deployment script. 6 | Example: :r .\myfile.sql 7 | Use SQLCMD syntax to reference a variable in the pre-deployment script. 8 | Example: :setvar TableName MyTable 9 | SELECT * FROM [$(TableName)] 10 | -------------------------------------------------------------------------------------- 11 | */ 12 | ----Create database user for Azure Datafactory Managed Identity 13 | --CREATE USER [$(adf_name)] FROM EXTERNAL PROVIDER; 14 | 15 | ----Assign Roles to ADF Managed Identity 16 | --EXEC sp_addrolemember db_datareader, [$(adf_name)]; --Reader 17 | --EXEC sp_addrolemember db_datawriter, [$(adf_name)]; --Writer 18 | --GRANT EXECUTE TO [$(adf_name)]; --Execute Stored Procedures 19 | -------------------------------------------------------------------------------- /elt-framework/ControlDB/Pre Deployment Script/Script.PreDeployment.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Pre-Deployment Script Template 3 | -------------------------------------------------------------------------------------- 4 | This file contains SQL statements that will be executed before the build script. 5 | Use SQLCMD syntax to include a file in the pre-deployment script. 6 | Example: :r .\myfile.sql 7 | Use SQLCMD syntax to reference a variable in the pre-deployment script. 8 | Example: :setvar TableName MyTable 9 | SELECT * FROM [$(TableName)] 10 | -------------------------------------------------------------------------------------- 11 | */ 12 | 13 | --:r .\DBRoles.PreDeployment.sql -------------------------------------------------------------------------------- /pictures/Azure Modern Data Platform - Ingest Definition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/Azure Modern Data Platform - Ingest Definition.png -------------------------------------------------------------------------------- /pictures/Azure Modern Data Platform - L1 Transform Definition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/Azure Modern Data Platform - L1 Transform Definition.png -------------------------------------------------------------------------------- /pictures/Azure Modern Data Platform - L2 Transform Definition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/Azure Modern Data Platform - L2 Transform Definition.png -------------------------------------------------------------------------------- /pictures/Azure Modern Data Platform.drawio: -------------------------------------------------------------------------------- 1 | 7L1Zk5xG1jD8axzxfhfjYF8u2QuKnSoKuHGwFVDs+/LrP7IleSS3ZuyJsWxNPOpQq4skCzLPlmfLkz+hXL1JQ9jlWpuk1U8IlGw/ofxPCALjOHT+AS37hxaKxj40ZEORfGiC/tngFEf68ZufWuciSccvOk5tW01F97ER/tAYt02TxtMXbeEwtOuX3Z5tlXzR0IVZ+q7BicPqfeujSKb84yw+TQu0X9Iiyz+9GYY+3qnDT50/Nox5mLTrZ02o8BPKDW07ffhUb1xaAeB9CRfxX9z9dWBD2kx/5AurH/WbqtQVjRLKyhLWrhL/wMkPj1nCav4444+jnfZPIFjzYkqdLozB9Xqi+SeUzae6Oq/g8+M4DW2Zcm3VDm/9Uejt57zzfoQfB72kw5RunzV9HLGUtnU6DfvZ5eNdhPoIvY/kQ6Afr9d/IgP71Cf/DBH0x7bwI/6zXx/9TxCdHz5C6T+AGIa9A1CanCTz8bIdprzN2iashH+2skM7N0kKHgvg8s8+att2H8H4Sqdp/0j/4Ty1XwL5hOCwe+D7P0PIp2v/7RqnP13z28cXfLjaP78y06E4AZAOHxv/JW7Gdh7i9N8A4BNnTuGQpdO/64h+6AjA829xPaRVOBXLlxz3pyPu07g/I3VnGtKwfofPk1M78LGo34QD+/aXGbsP8gVAL/x08Sw2gFYWUHRxSg01jNLKbMdiKtrmvB+109TWn3VgqiIDNyaAePbjG/gknMKfUObDJSJ2TfYTwhUua9grdJWyljl/dOeeC/fs/KSB/3iZY7TzLxfmFTGCDrnIaq7gASi9/aNAL+r8YB74Twj7LA0+wxTOKkXW8hB6iQpslimTP3ZqkWe1wJKLjc7kjLVX7mUcI3Y/O0I6byvOvbbLQDofsgwYqh/J7Q4blxx73mHd9h9iMQ0hKm2j7wi5GqP6XrS92hFptUQILdr3xz4OSn52hM5XUM/L7lbBo+yuzrVouzW+MJG0zdg5hGLouec57It/rwTLtbHGQOLQe/YGs0tcUrbnEGAiq8j7RCb8rzM9W2vV4egxg/VivMYG9DRt/CmPDjyMtB4eN0Zbjz/yy0OhSXoqwXOR0gm7t9dKSG7anBkuep/i2A0CXfCIO654ZjuRvfSSCQybzwFomFmmO8zyaML66KwaxGsy20PceLS9CWZqhVTb7g7xehHy2Z1bmPAEeYCi3HPQGyveoli9XA3qSNGdyxVzBLPSxbTOV6lYZFWwJPoyjZgqnO35+XuveagruIOpt+LymlxCPNQoXJHApEV5M47u7KNcCFfeyp5ToXzHnraSLE6tvrAIQfewKPhG26x7xd+ejNhJV/oqDHNinN/rPd3s02q+Gm7f7otvNf2oCy5O6+lmeGXb2YZeybOZNDabRmG2W7LpnoJRjGDoHtfK4/DJIWMk8KzzH1ZmKZ2QoU0WlouQyyTg0xbB16tO6HFAOFwHK5fn85Upd04VyPyhvPiGgy4hqWlGU9DU4uiYfM38Ec4v2Xxdm8zL8u0+9A6hXEcRfezny1lbcIeaLZTrIRVK9/JM81plFgmAVYbtQZRspCkCYDCO6NWKKMaSEy5WTrI64b9OesyH/vnceoeMHs1tvQIgs4344g8zxXEkoVoiX1mOfKEnk7EnLYpWFV5Q6yIYfShCVpuuzUsFaEvuHB4L3f2+WgaSli3ZyZgk6/1dYJOFutdgDAzdDg+HVDFtWbQrenUTK/RbHNOzKy8mHpFAYig53ZK/XrdTRrEuD+jmURsc0l/KO1Fmds9nWs8KnsKhCbFfMUXK79eswIyJaYyDv78qqnzYjLNyRJDng7vq9Stk4SlwlpFbzNQf3JDWlHCi/HvOrJ56MWoPkkQpZwdx6a2SSTpyiBvzXqM9zfrQ1bmB+dVMVDTuzkjyqJUt/uJyJohTE+t293nOLOGPZmfOD3tS5XZcRC171NJjmxOJlw8JLbSj9O3zfojTCz/hvFwwElq3JitfqRfnP3RpycznhWW9sThf5500JMK++Lw7Jd+2jyK8i7nP6EbPSvGu5TTG6C+/w4To7HxhYkUY1tTzhmymR+OlzxbitNShO0jK0s8i9W8JtbaFwADCWIkJC9Hs5SeL7JU8R9AvEqGzRJ8Ro/K3bFtu1liSz7u+Kod98R/uWMLqSV6Vmi/s4Dkbxb/ql8eo1ClZ84i9HkWPbjbi0goKsFxTOvNIRj24od5FUPn6ZVfMc1sviwoY5lylEXHKSpzvMizP75VeHYEawdorfIJ5nxIXQgQtgbOxaVm6v59EIFaBr/chakQjfdvaUz0+CfzVGpcLTumWjfkNQ2Y81XjrcNNPfIkYkRvmc7UXm55Ntotangnv1eGzbjxith2QObY6nKA9bBpIskzt544pBb62JZ/GUGqKT1CJz+BI+LyM0vvYinOZzpRxsogoURLVpAPcQYixulH1gJGeeZzoZa17RATDk9surzJ1ACZj6NQTIS9xdCutGXNLl0ValmnyxThpi4slzkNXTu3gSFYiXO89uSRG0F8E+f6gi8bqypscQmJLIpY+hwXsrISXQeEuPZjJvRI84zLmObf0Xo/oS0nhpBVPkhBb2T/f/YJWw4DWSyUncXa+HyDHucIXz7YCTyimK9szKBmPcXuruZJ/7fcK3s7vjTR2SgTtXACvLEGKA0XGbJ1Nx4GVyskm7HbVoMhE5NsdE15YnRv9K9q9S3ZzleUiZoaf7d6NDfdKpG6pQ0z6sk1a52XhA72tgGI597aOjBzSzjkkL5daIb3KwZVpHjYlXy9nY/AmRkU5QeoXNNXCi47Xh68kWJmq4zZxywE/udIYm1V1b8KWPfIEIaR90HuMKbnozml8QcVXyjnQRw/X2MM8TNGXQ9MSkbbtn1jS3gZoj/zk9fai+33gs40AyxrEC61I8kMFYTalakDoHXZN5qPN+cILeqbDcZI565CsN73wto66iodM2zLNmMS0hPDFqDwVltLaDId4VDdXeOKZttTscj5pFGQeukLl8xKXSGdZyk5pscKpR1+f0LGATI/2Ld3l+/kphVz+1DhPhYvpyRKSMPMwBgavbs+Koyf6EdKNgWWPJWzJES3oVxCX5ytol0bd152gLlesVe7J44FkzzQ21NLdqwdEcMJV9eCHnKw5GQP1orgDiLNBHc2Sm1J6GM0sCtbi6nVJiSjM4bt2qzWJzGgjoWev2fzTGAFsE5//XVNOqrlqDllxw8YwFe71OlcSgt31D0vhsGjCqLMLB4RPEMh0Lw7kyzlGed/tSKD3hS4ZGuUSkiLtg/aHRJgvD+K+YJoBd+NUAwzIjyrTIkeNydC/YRF5r3r6OW1Gswj96wUwiPo94PscvPVmPgU3IUg4wlqMUxamzh8HItvTo/SkC1rSbi7acTS5QLI3XJ2U0dWDRzd3/Z7K/H2uDsD9pX6w6P26Luks+zYUK7iQhluE2lH8uvG4p6ajknUR8rzQLIsai2fIdVCEdUdwhshwigrfskggAgtQ1a2UltfWYoG3sffOpSSssvxh8vsKl2ucJOlrrz8e+GqnOMk5XL4AWTrHbkN3q6cYU9nfqc3jbjPQc8nypA/xMrO0gTdDlYVJV2aw5HpQGuSF5Nt/5BdIOAM87VxeFRtzIfdWYkCtZpy7a9hXnPNlGdgHH7X204pL0m9g8WJfGrw4Qr8zeMmv2Lvkt7J3YeSd2aS2WXESO8R03fjOeHpnGn1mxXZt0UxvA8TZn3AAyvCjORSfAAPQ/GQRsc+2mT5awzDymaVU1BlYAAsg1+sxDoEYfxvPL2A0P49L9iehAf2N4wEnT0v7s5/3boivYAWFf0agz3/Qb+WVwP8Or8R/4URA/6AT4dMTvxcnAvqOG+T2djZc5uh7YAXmmIf0l3NIv5wD+vO4AabRL6USiv2Mv+MA/CscgH8zPxz6P0bx2B+kePQ7c5th7yieB94q5DRP41NV378Hsgcj+uXjeL4d1f+6yP5tNA+9A/b3TfPEH6V58vuieeK9qzitw2Z6U3veglzfheLDnGjcx2L8xUmHpYjTP1H/IX4TeEFI/A9KfJT+RlhB3jvwtTDOi+YEBaSm4dAUwG/+96Pl46h++TSmT+j55dEO5QgCaX8qopAvEAV/LUJG/JVC6j3zcCDqikAmCIn+vcEzFCa/jJ7R1F8cPvvU8X9ND8DIvxl1+OeIg38HaX8Ffr6vNQuj3rHdpZ3+Hde9xSq/BPc78ffbsGVdJMkHzKZjcYTR2/Ogr4rS9yj4t3T1TuL9mtPx8S0/fZ428TVJ+A/oZwyByS+k4T8+MtEfxsHHp5tgOp8J1S+/0D6f40kbv0XZr0P6L9Y4+B0WPwSpv5Ol7cNgfnlTPE7C+BPXMZQkvsAcjv0xZfvbqRvvDR9xbmIQ2P8ulL9fB/Pn4QCH3ju9iPfaxNewQPxMfSs84O/wIKjA5SIOYZ2up0oFdPJ0Ct+SKL4DxDiW+gswSKNwTH/J0uZUHeI/Ud+D4d/gCPkKjtCvcsrPn4Ttn44k9L2X+P+KlwDFiZ9+yzT/3lOMYl9BDvatMPNejL3DRXbqad2/TFL64zD6j5ds+Et58zXbBf6quEG/lfXyKdzyXyRE/kHF51fE/D6l/X3geE88TloX/zjVgDme5uFkXIQIawCAJhr/SSyfASsJx/yfFsA8VadlzP2arAv9jv5Z/Saf7p1i+h8m3H1K7au3DOQr/7ymURGDNfQ1gq+zZ2tSnO/4LJ2VF3mBF/58pP4qPr5kAfg9CxB/ZbzvPf3fm/EzdP8Gvf+juZKJ+z5XkkUTNPUqPFa5dTT5FYTOS9yIanzmM0ybVAdt3hIpbxCtIforaAIkbToirhTnoRbYvAwErTrQtEQIFV/YQ92p6XaHzYX8EPsnZ3L0JTZ7S8k8r9GJTKIHPPEWZML00+NXygQJlzGqb3DiueYNojbtJvZh3YKcTIK51f05UAmB/pkdSXhRb2SbxEHIOdkn5xp+ui8jiP5X6KXMoXHDKj4KjghkN1BmddvmvW0b40EvL3RYFnEnCXhkbItRrE3mFSZXRIWxlIvIMB0rMItoZgWrgKxDucLumdUZM+p2c409sYa6M1QJWQJ1Hx2BsgbHdiyTYc8n8Ap0o5WLwt2euIzJ0oc0Tce5AkhwqcI+FVlbFDu5WRa63zaL3282c9ucTb5t3KS4LPc8x8CaiiKwi3gR8kWQoAoSRKqk7gJVRhazMbQiQRxVqha3MYliQOdolCxbQtJiNMitdUhFMm1NIZA3mhccOvQSSDyR41IZM52AjON1XromdCc29aDYPgkOhYMEIt2MZXXKEREsti6PXbUaWEgOVgrMaBqgNTicotsg0r0PtFU696PmNSHzNavgWf26MG6d5AHWIQVhXbCcRQpfCoPz1awR8B0AQWsifNJVBzbohwqDrA5Vngr6uNz720k6eL9F/CW0GvGAjlG/FyEA3yMlCueKJgL7Ur2Zt3G5M73JPqdYcKF0tQWek+GLrzKLdXn5mRA4HRRmzg7dW5HpNJ62EohwrsMqYaJM3+dDtwz8EEVfcjvxsdplZjeQbsusEU/XHX0qU1ecShybORshrXwdJLWyc8aeN4JnHc5JHlLsCk/6CjMYWy8WQ/jt89olsag9Pfi+3AeUZu7ZxWk5efE0Q1GU5KQxVj+80uW3gtZZu7pGSsF6S2KrlgiyBmtksgUu3jhZowpBvMKsXT4zJpClZbX2w3CjGl1nPO9jg3S7ulfsZ83dzGRXn4d6nxbTl0dYTU+xyxIzVGKdmYDMquV6uequGO9LnF6osXMY9ULVmKHJxDB1aTQTBQ91yJ2Vb9TE7Lcy07KM12TArutLBjmYYsQYZWm9/Bt5u90sfxuxuwRy4GxSb+4wc00RlWTlsidILq0ds/N8QHnbhQLpK9nkR4ECRzXcATHyWO+14D/2R5dizfEkBec28xpTnFKVZYRgKLn+ZjH4EDq95cqclrMVXWltSphBSl0T/6le8VuwYIgmXmKHsO6JaClttwsXM8q6sPLVUfPuuLjRva2BtEyHCbtsJe2b+iJhFroH1xC/WoVNTJ0RE3R72+GbyAS5wIQKp8iGpTLwHV5BmiufagmXQvBLhhlv2gSQYiTFDvwAeVGy1T8eeL0T46JlzS6V/H16joxkRywqP7rEhx4AF9hMW9NkP5V1HxGiURuiTY7hZk7WCDsu1sa0yNHPSZVl5uDk62sUCKvGhnINSEvtPcbjsJ0cZUS/FKk1P9ZjO6id39l+FV6SVUYRxgfETXaq7eBierqEZMRCr5ZQHQZz8MUeFzawU59bbOt8PnPsCMQaTtdNmbp7VYtxWjFPnoQ8rFu8g4nVIIWKiIed6T2n1VLmWfPPfjVeBtmwYnAKhPN+ccfh9aLMrCXBfcz296rGJKv3wmuLCt1wjR/6g114Oy0gId04xgQPTtgjr+Qs5oSnfJtXG+RXZYrBXOG+71Zu38NbkYWFWNkWJbQoLcgvUn9GHhPe/FafIXa3D6asYXkbr1p3bWyPUYO7brx2nIgLzlpsQ47iBaqDZQAUzLOyyOq+vBUk/BBubG/EPTu5lKXaWmlPDJ5nUAzTDLVATLJUwQKS06zkLhdlzcTXKLyE4zXWL6lTXiQBz1pZ7u+hDNawyX7N+j17gygn6zgQba+mdwQWWfsxHeVyLZBc5bAXkH1Z+zJE7VqXqxOpSw9NZyN4CtwNd1NMsh4xbge2MtQRpUjRoeYgvapuxbmrl3jauAnYawfZrqIzxbqRAVoNcPleGDCEXGCHfcqT3j8xYRjgse0U0omJXcyEXYjBeugI11q6h/7DBfliL+tEOSWFCFHUDG889OixvhjKJBofF+NBSlmFoatLi6Y70LnZWEb6AoPTlhTsqnlNcYsSIocUB2v4OSP5NpBdMptQyFNP2K3uJ4jl9WY1p5s3BvvuFKjeSDK0XyWQzkiQ7Fwdhr+fAugw1WHrhJWJD6zoc12/516g9vMwK8ckK9w23FWtYObJVXrD9nNhmuChvrNQtGxlnopg1TnRfHMu4jhvrMNhGIkwC712LRvnTlCxMxHH4b53SNYiLRV7WEese+UdPhJMmIKLV8RctNRC+hQ35EMJ3Vq7rpLniCHOvphIvCQpLJwvBr8SlSW2SB0z2zMTSGDNhr5zWEmmwSK2wHJzCWj1LroMjyJRSVvyvuMzxZdg1RvNZt/nUTa4uQ5lC462ZOrengy4o4obHN/sULytJzdztANHNppNDUjOG0W5msLnqbiXr7GpIqLEdhqkxPt5N0T75OOmjyhEs0dNoWv9PUtx3q6CaRXJJ3+FpPkhzFvxtAlBvN043DXKBKybzNsvtVB2Rto6r8uIdn9FikaJZf/ijvqwSRNilCtWckpnEIglcU4p+QHLpOdITNylkWELr8EaviULCpV4K53Zqjnum5iCn6wG7AurAfmK4Uzg762GT21/fsT8q37rP2ojfjIigGo/Tu0ADIm/3Mj+BNnf5Hx8xQtNfgWw38ocQ99HU/+F++bbeWl+3Uz5K0zeZwKgX/PNw9Q3A8v7PLxv5aYh/gUY/62b5i8GB/x+G68dnpob9P/YoW2O9P97B5y/y/2qhmX6iwN4/F96YP9bNHyi2t8Q7Xs/LI78jH8lMwL7ZmlF70O0nwtJ6P85RXXO7/8ismDs97H1Vwb/4PduMG4ewukDnqS2Sv4vYulXr+N3gqVPQv/frQLfu4PSfHNQqh8clCyEhRgBOszVbx2UQ2Ih+ss//2pMT5tPlKZQ/RZ9uL1QYKM2Z93lUuGsVeOZRtnj4brPtxLXVhVsUJqGELkW7cCt4/UYSYNa1HW4Oi3TJ6a8ax/ckvTFhsC27htAmvhcKZN9+bVa43NAPz10/bATbQsebtMDX+jxoJa07voASy8vkTTdKXrA5mZwA9jdyF0Ro9/xRYGhNCropcv2VIumpIhqfAylvDNvwLSyEbAT81h0ZMJN7s0iDIHtvdzMc2ETY2JNkBT02YJ1oh97KrahFNd9WR4AgtSv7lA44SCjd3Lmkax3p0839lGHbXAj8rIlH8bz+QFaBEKqG8mp0vXRmqdpcz4Zd9MYvRwaaRwMETu9KxS2XDmd1d5D5ePnG5bbuXZ+v1yl1JptBo/ktcoCfEelphddPCfywclnE6FtuoUCd+A48bQ7pp0nczHsBparblE2AG9URMMIK5DByj2BrT3t53fdBKfRtiteYHuxvdXQopjEcHbWx2JdZrDPjk91wPZ65pUQ43upO8E8ubYbTjMs5pql3lTQS3OXO9Q3O4pSbLSq5qPSsPvANuc4by9Mb+nGTCClcRSSZnBP865QQ2UvewoyiVKdZBk9LL5iDJJ4PL8wHhn6a2bt6Q3YWmkyIRbwe6DPAkqN+fVo40dA4yeBieGLuMxZ55r9tiH7DeFEDQHgwjuWtq7x7ivM2T3g7G3fX8QmkIZyE/cLqfmyQEQye5Qxbk6h03/w1kaY6PHTIniUVKC30FGFOvUXTEZYNs07NNfRF0eXsNuQfn41oYgy9YV1H8GNQlZBJDzDTdEa+K2NA0PTiwMA7SiLFVHBM+KpkmEd9NKkbYMtqynznSmA512uislJT+PFTXVyqKtL4EU9XWItQO6MIMKieQqMwRbnbZHuCusHZjAt5Pgs0DUKuuFqGXzvlDrKahcBuCpsNMeIYld3d+Z94DYDzMyufDFh9fC0Zuw6F0JJgR3eXiB6o6DF8dVoA3cTzp4Bc01C1tWQhLHyGYpUfstjq+H9oMiVDjLBw4TR0aAmsG3nhkPM6KBpc6c7dtj2xT/Bz0qAs9XQmtUA8iVPEYKMlOXxAAxqKKKQ2O3zXNZYIeUSKaeg7Uoo5YZztnkBJQRYT3C83pYm0U6YFuKkCyl2ldw117tq6Yx2uND9bLvoSu1Aj7IchiOLye6J3DS5DWKXcpANHisP6vrb0x6lbXrlJB86M8K/9LLC709shmapn3EF8kX5RVoGBJzybBvXCkvouROCkgcsq2DdpijBC+Gz4BCJKxdz6gnxIyNeBiC/FrLM/ajLsiU8kWBJWmH1BdMx281fij5gopM3+HW/8ARWmhBPmpKtF93ttuYPn1z5nVal1hzk5UOwxmHwcF+nLa8cqwwxq1WcYPOV+xPh2uIi34tGV1W9Yhl8PheB6R7sTD+EjvVo1ctrRaoR7l9BVgVDF7STZBPW9hKel+yZY2GQAU4SWxWF8JjrOytHtZEOEWWtGMDej3I9l1iJpa9gWxdbPtsMhTPau2ulSwU8fa4T7otttiPFA7eTsZfoCqM5s+4Uo7c4H2lLRPg2scqWjSpRQFL8IZB61j/HfH5SzNxYUVcTcSmtw0YE9gMkE3nC6Nvo1rlPTbIaOYBdWuES25EsUixvIQ0WaDkAu6bZzWLGReavhFsyz1UIZolUxHB3mgHpEL/qbn2FcAJZkBBDdbWXHVZI756xsqFhA5I2nQHjC6FvhNZd4MhEj2rkT6wt4VjMkZu9DmV77jYgTzXFeiyz+cl3pnyP72cHaoxqfw9LCBPsSM1cPRBQmrXQIHjMgm08Z+KityOO7MR1XfpI9nTJuTy2iaodtDyCYCylMDM7n+75++RzpcQPGl9eXsY6YeNjAxM0YyWTxZtctg5UxpRXd55LPfIxOnbXrmIbC7T7tbabrYgVPTDHMUjElImRpNqpkj9VOszmklntzRnWO9zUBK2G4WZjo5sZxchdECePSJEHX+MycCHn9ylg6bwx6lzYLrmgwVOt6NMNVxsxHEstIXvJul0vvaZbKKTQ65BZw1tcQSMz5KGQgtG/rLDHL5gCFqfD28He4hugHZmyLXtK7WO2mivOnMDsbvSY5byw3ZAoDqmyvsJ8XgfwjmTNqA+sU0uxvYbuRMyQD/bNW1dZe1znkheEKlD6sr4w6n59XDnr6Xd2xcDNK7iAVSqYtY5a2FaibweTDrj32g/IhYDAQ3n5KBfbQ9adcsTrvcLaCSuVoi+CJAluJ+wWM6GA67gudCjUUmOvb9qRjUd3vQek+3BzGXNoB28buTmfJ4TBimIRRzWXqXbDGzQ9NqBuDGvlcJXktIJAtBCfGamBHx4J477SanTB53Gsmc3dFUNiH5EQb5RqwOoyitQmDfTCjKZMBhRQYPC5Ns1LVr6txS33hii+cd1dHMvGdUZu8/qTTk3OfEA0jYOxlyKDamaF0hmlPPGOGYb6ZiAg8hO7rT+OKETJk6GU3qpSelVcAWAU0Z7wm4ZHYK96T5g48FMvN6Aysk1zmA0Iv8xZLIXPnfY8vz6UxxZY5V1B3goDZKapx4FYYqc2eS5T4njt0K6TWdUgcwboUa0sLUF/LmGCtltDTz6uRqPXDi9nN5au9oZCaOTun+tcdpVNyJZ8Zsfhyb0/bHOWt2sWbMVcyqsFPPWyZ6QVWkc9Qtgj1aUGxw+5jg27Cj3T44DtUzFGqDB8TJuSzHGZ3ugH2c8yegUbq1OiJm5zr95jCoKhMXXugRiaa/TSlekiXgk8DEW1ujDTY4ozDeh/dhqgxt2r3GJFH4iv0wv8wsFAIFOh7XoyKOMxBWaogImWR3JVjEuF+2XXlAQ9Px4qWShcTA0ku5QVuR8GcFZbzZN0mQuIslUKQhDE2td4ZgbWs8HpD0FqyCRML3jdrgmfEwESLxx8S10sB/UQYH+4XBEGle7hs/Lp9ro2om9HgUAOj5ISL4Jq7fwxF44TPPy1sgl0tBcKIcQNbZvOoElCMRqcIbIBrVfiBSPcWpFGwCWMk2NmnUzWMG+ROftwoO1926mXTo/NgHFVGJ91l1AKJ2GePE9foqq/l5pxFSYd9ZSypJBbNxFXzM9qEcU4zR3p3aU8QLiv9sgGMDkzeD6IRDNOIIrk1DwzoKE4gXQogWV2W8aFIDIiMXuD0Y2Mg5VBv+1PvYMec+oSlG9I55p2ahtLdMczOh+HkHl5xW1DXjvfUwp/wxLwwvjiYfnS3RIgF4i36htDNs4F27eJWNZB2wjApV892oOMctEUTYDW3kmGdLnoF+6ixvXzRhuODyoIgdBRcrX9wpJ2wDRh1D7doFiHA9Osm0FIfRCLHmVO2VRObUMxIFfheFWYyqEQLbjRcaKBlC2tAmHIZ2cyauEvc7E9UgoFATllehTRgwTWVkdWKbrZB2skFQ/M03Zcr7u/cRH/JEEoPPVfXTHSyexl5IKE3u1ZQcU06NFAw+skTWlACPSw6/rjIlI6Qr/lO0RXK5npRs8u+uaNZNSoY0qHofAMzTY0QkaNvEUoC2AuLoDK3gJOThSK8ql2TlHxZgiGiQ5w+Hq6qPty8fVIA0Q1UIbXK4Fuz34EwB3INxRBrHpUeTgvUYEjr9pUz5TwaJ2Vghe0Nvw7TDlKE55mL4kmhW4txWzb1ZUmPLk/AbsRx+YGAVR20cs8TQcBCDHkYJ+Gak/RCyZXcqcwy9HdON4Ww1jzzbiI3cr5enIEtR6A+GG+rSlpsrTZpeoFawTsQpE8mBczbY3uT3ZovegmXmEMlKgZTPrels5parnXSmHZHvKsxZ/muR+r9RE91InCEVXgSTYfSc/qjEpmN9GbA2UJsASBgE2SuXMFTVsfPtI5jGC0ZYBYdgDSTmlGw4qJtdEeDYYZUVKqEgDzIPsDJ7O6n+CTd6pDbU7FMXVuQFUfNT3S6ctzqMjhdY9o+n5fU8MgCfVZjnB9mqtXZ9GYZhm6R7+K0EocqLps+hHp/cXwK9y7FhCKRkyQ1nWS289Nv4avyD8W9yJ5QNLEhhajTXcMruHTsER7ySnMjCEAocTJCbeqZOEH/SyupMdmihPiZhDbL2PragrCjYbra3jEjUU3wSSfaXooEe4+bsIFszS7ulUVBow1VodoPOwe/CMo5o1pSJaAoh2UBiFralcpXo12PjVtEwn5TS4ATByOKHqvh92keMzuiNbJBjgPUFjtSGbylJ94wWbGxuYg/wE3roMxgGpFyGZ6xz18+d51oCz4xFgcyqdljPU0VMOVC9YzdIiQY7BvCCVGz0j4oG9ThH/FwjKgTsubUsrQAXk6D05D2PGpiLciXabtzUNB4c/mOtAvNp9tj3PoZCOLMdPsgRQnFd8xKgbAoG4XXJQ6ptbFx8ZR/B1W4RI9uVdzhtzMExLaqwsFNZEPY8QhFPEWEyP0JrqgK0rDm6MXV9FhTpEhNhxvtAkCovQxIgfAVBvS/rWJxcZyi/Qxu4t7U9UFj8yFU/82NcpIDCs5MLDShKHVHRp03+7UKXixJlGweuZRzG4m9lZMtGCfFsZRF3y68Oq5hEBACHhesandPajyMDR6zz1thWmblR6+FGjgp6UL4MFghn9acUUrF7ec10KFAGQ9egXHAkdNnp9GOFh8LUoQS6R2HDmBOwi+5tJpos7zXL/Wu2sqIS4BvJ+CKp85hZL5LJGC5lJ4dzowXQOjmikF94+4JJZQcFda1EtEuUGQBYoT6Vc3pucsrbRzKONm0ISxVMbNCB8FklHWcYeMp+SxYDXot5uGmgmMCu0Vw0zozvmIA+G1XXmvytzBEo1YsLCokEtZE8IRSx2mHusL/WBBilx4KTbTIolkLiqGgEu1+0MhSKSV1NEZoF6OCHd7WZsdzD28tyirMG6jc812hQgv4i43I5opcneKTUZgULIHZxq9bxCe5hoRllEfx272C2M2kEJ3HaZturecNiuFOulmIQOjvGimtXbsdHCxurgu8al8XhZ9AT429MntGmrxqaGtPEsiG9vcb5Yyh7kK9Eon79a6LHi1OGWH8ewVT+Rs4tFdMa9FS5XED2xVbq4skrytNg/fS/nefzPUy5rhaUWf5UKFzbfUprTpNsw0nw80kLEalAXDMkO9KVj8gR5BoglRbJc7vQabGaADKeds0C2swGimGIS6rVQygrU0rN1fkIbIh1V6dfDcgCpaeYWEgAqCnMBBweoER2UI84U0tYILa8vTppUGOipbw/URvxbVjTvpxlR+hen6quXbbrVQNZPQuGOEeYEhtbAHMxKHUyKRSWO5+4NzQsWFl1M2mbccx1iFayZoYxY/UJzhhot6c3ttwOyk9hbrPQ74OsrxsE+pn8pFSvZLYZgMqBHVrl6APGB1N8ZDRIx4fAq5lNrAkhsJqiILPyqj232C1luKDYeqy4+yp5uZZo95aOIUrD+N7/VuaSTNgQqs/wLqCauUMEKHD42xgo17zkEthmktckBwat4ruS5sdw0mCxU59ZR4mVgIWW50kOIJqZIDnfQYowmITTFsH8/nyU+i1+3BPhm2ywtqrCKczw7UE1TREdunDGOxrJ/PiDQwqwutv3TPN22fcEriig/Dxmgn7TyBjSDcj4uMH8/uxMZp4CY2SQ+eOgBZvNmXCwmrykpd/JsNTforo6bWLKc+qtQidpKC4Kii43FdvUHspleu5AEZXG2SeRkxqELdmpXLWHVrFAKb4Fgv9xX6NnC58erUvQrceBNuCYU0s9D3uI1ZQKhr1pTq+VLGs/KWO3UP2J1c9btLnguYqytA33tFbAKWEEOD0btkzlNiIAhAMfzQ8Mcs95u0DW4AJ+rcXmv6STkAMtBluDraDlxUNGUFBAMXPZvU95elUMnycHz+LY6gvzYGkL798pVRA4PAD2Jr754fAoVwEsie5aXrdH0G/vXUHRui3SybGIpAA3UeRP75mARs0YMXjuz26h2siVUsSByKM7UPnu6DjLhE6scW8IVOquFYE+LKKo6OinIg1R2AIukUIfSywaqrskB/Ztxb64bmvTGvKbK8XF8nW94ONJcW5vtVlApbPtWee3+/cmClYGnYCyzVQVFt/y8SSv7zAByK/Yx9vm2F+iLSA0Pwu1APAf8MUe+jPd9sYznyB2L+P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9/0ejPQhO/UzDn0V7iN9Ee9C/PdqDficnWvxbiP9+BfSPGzN//7DMj7D9TqrJIn9Ltd8/HfrUH4U+/V1BH39f8vJjNcUqLNN3mDmZfvoSkL+tzgtAPX7cU/gflfl9v2/xWVTVp+poTduATl+eAPyx8Uti+PMl2ZeVeX/ndCTkm0mp9xuvzXY9YYpArPw/F502BBCdzj5Ep3lOe14m0KH6EJ2eFvvt/rayl2V7vcZ55WGkPVA8CqGSgQg1u8Hrpi7LdtE7hh5cKza4VYCu1g3Z7GufrxvsWa9wyFnPtvxIzNRuzi9Sk+GJkz2Z2sJIdbV53Xr4qbXkE8uduj1MB0ydPrPl1jNUIa2GFa9dfbcSzWQel4hpBzk7h2ed78l0y2L85rLKgcgopJ/VwmmKZTfGt3jmHDP3IildN+BTHTonIk7/LKOWhOhcGDnH+BQ5t3YTtXU2XotiAuUe5hmd36Lu53/JvlpMLsATI8RNA5RnPfdqJZUeOwVWdN83awRNpHzMcuCl2C3ZI2WBp8uCpfaNoQKsjTXqboDTdDmRB15hJn+OmWUijnw5ThN6a4p8PTYLizD5blIlB3wHOwu8BIGcPbXRMqX0euFNkc+X377lAbqtS8sYvHwhNYagn1nGq3l31V93QY7NhAwzFB8XXboUal5tM/CwL7cdLZX2LpxaqFlikbIswW1YVSyst4JLjTy63cRnwfqa5R6o6vrFVD0g/bCEBhigBE6UcSalXQj0XzmxjSTtFFZdb1G1X6+bgccTH5Uc6VWnUQaU/Bvw+FJky9Cv9JpHva8SriuooSQDR3cp4TiFDQ0nx3yly28x+LBPkHzYndis0Q1j6n2/pw/2GK7ooAFFmIuAycVdsP5gO7cNw5exDnN6G5le0A+/wepHXqs3whxcfR5s4KW8GBDID6jc56vClJWxPNJ4sNtqQIZM9giKMDCK+KiKlQ6ks1uzIRsle568SVp4UH0gW9RhoX6XN3u0NeYAIEl7DqQtGrnL4PjYEsRUUzRY0ksFc0IQjtmK7kyQZFBQFTz/lPVEoq7BpTnGvOAZhMAa7kPWQusf7G5tJHBJ2M5SIgYvqhmwRlIF7vQsQRy9tw4srW5FUeiVBTTmroIvsebf/NFY3L3pr8TzISO0lLgrLnoJJIoR83JOiyGBefd2yJGC8hx16QZgAMm+BmG8fBfeaNOKFIityp6RC2uFAjwfy5U2AHV52cUuzUy+emV1GmUcg+Cogc9vlFA3A7tvUtwW+TPsKLvGM4uwDmHa7C6EORmxSGlmlRko2ke5W1jRr26n4Xe23jUoAPZHPAjBklf2BVw8qnBndlFIt9Zpg5O6bSHvupS3BGMrhUtgZYUzK4qk8ntLERavVtRuhCWKPao0P7kCMAfE9IaOzvjKcxfiEfmtnCOW792twc4uF3/P64HrZ/FCXzRt00/wbs9STJxCEpTm1co0r62HVtbjrn8OC3SQS2Zdjie7ZlJl5QUn3NSiXQbB3FleMC4oeLcJ+VWqBa9CWE2LvXlzr0Mq+2YuJKAe0sNdhmhKNLZOvQvVokWbE1adq3ZPeA2NxRnUva5jPvitC89QYFQ0nzZyMoqS2AUXi86OI4n55cg2c5g95Ag3PkJA3bLBrvssruGR9OK+emjhFL0ycj2pte9GQoMt4Dim02W9GfY904x4mhBSLolDDu6Y6IvlMbgNiI6oUD6y0e7PeSUwz6A4WpTVcSPqDOCof5SEnlnzW0GtMqPvsWKOsQXdn2Tj3iVbHWCWwa7btR+So5b0diCGcoPgon4EM6ELG7rriyaQD3OiFfMZTVTTBC2nAwYAYU0XeBnBwjXHKDqRkCK5wOZNOiySGniRYSqCnhGYyko2+kWtTImRTXdLckSMUluJDHFyX9fmpKyuIbUOTjEVuP0C96KlL56avTVCKHCUmNit+f2oWuzIDOOwy3hQn+0W189pxENVW6nXwXl2v5OLDEQUmwPFASFtjpcpfqsd2TpJ8Frl8YXe8hJY47NSb8D8LaayxmsK2Rc2M584SOHCgN+rsS83m4cPytCqGdEqbILaEXj+pWFM2dYe7nd1faBOTOn91RULT3qqbDMfT/wc48qKkW5XSu/2aohfbhdufPVW5iXhGo6PttlEYrH2Zzylhkfo+5gkCgzs4lZ3wwdzKRtVAX7TEhWUmwnFsMcmMLX6xoWG+7WsYTFVh9ImHjT1ZFHyfsFrcQa4QOR0wQkVGiHUaYpMe420xOirPBakOUdDhl+T60s09Ad3my7e89AC3dvtPjYtV8xZdjEJmE5t1pEaJREVDyueXUTnOWenioilNomowRiZy2Fsz1640TsFN870yo3WMy0gCZ3V+tYr6JtnlMYCegHpbz755ud7S5eDnoXBhYZdSjTHfP1o0Pd67n9cT5z8bZGNT7WlvtBxf0bh92ruNysK/KmCx99tDv7RQ1w+lYf6fcOP+K4Mv0+lh/5X4PypAMzvwxn7ruCMv4fzf39U0VZM3mefP5wvhX+8+uc5ReDi17OlPkNuWkXt+jle3xrOG58svL+GIz6Zyt8Lpv5A0vc3r39HfXnkBvzJO/Z5wbevFWnHyJ9p8v1REH++2Hhfreo2hM34bIc6fHMYfBdS5d8i+PedefifTZhfP+sJQX+m6c+ObP7NAz4w0Mfv/AZvf8IBUPDXXX/RUMTld3Hs0D9H859VG/uVkf/4lpSvKD5fKy8GuOxbHT6Ev5c23x/rfDOO+FQSlfiNRor+jP3nHPLusb89/eV3HvtBQHwzxsPfH3XChlOcn0381w6W+p90uv/HxshvdoF9uv673O3Ee+loDHGensD4+jr3fwNLyG8O6yW+Ug6a/ivxRL7H09+utZH0e6BgJP0eKt+sSu8nVvkvihZ/SU8/IegHUfnH14xfMfMflTP+a+FEfOUEYRAEACkTkFl0KThD6rtQhphjHtJfPhvSOXIUHCj8UTf6K9H163lSvzFWvkL1X1Oivhky30dIpQJoAXbatd8REt+G8/ehDf7EYf8abejXalV/M7R9pVZ1N7zB/7tBGduGQ/J34gz+zljtK26AUzX6fhAGRvOLWYXN34g0hMJ/D2l/KaOR70/O4NPF6N7z2f+kLvsnYe3L407en3f7Ne/bt9NukR+y8XfZDP2+ZCP5Xqd8A9TZpLVJCnI3Ptr50CmgJuA5PT/+4/xlTgPz1MnfTmn4X+TJz2mH+pPsTeJLGfopV/f3DkD5M46pFcKykuZ2mkSFJbVwiUUr+Af6DjF/gZfuhN+wg4gP9DNGU58aQNgH/vnX+/+M+7xd7Z9fmelQnPMHZPCZ9PyNN+93wz1fBcjHfr8b7fmQv/gXBHe+Okrs70Db3wDlD1HmvwvKXzmaa2/Cbnwvzb73/FRV+/ww37u68zzogIi/qZ7Eprpt4cv5N/GoD220h3/4ADYagsJA+MfP2NvmKZHcQTv9Ip500s8e2Ln6RF8f7oFkHchklJT2XNB+/t4pkwryaTqID/dLCjwrB/WLenebUYYG1yDNZ6HiTFsf2SlaaZPGpvkFqrSAfcgs+aT7Yfa7bCwUUIdk4uuJmkodjUmzUcE+sGAZ01AlFyorEpD+Ms/NSD8PHiQsPZwtRVPyWRPIol/RZeLeNjSSu6mRVYLs8KJrDY2F9SzAq8nMU7qWvZE6i74er0RPmpjo9lTHhiQxKpqZCl1SpynfBh3sF98Nqh3m+oE+1RsoRZQzle7g0OzYqA6BI5SvhfAxe1Z3k/BuFg5rCXHDS7vMcbKqxEXBaz1fXI7+eJEjOX7Ew2UnJL8F28bh2nemY+Oux7AQiZfAahdr22gat/vVvGXF4ezzcyIitAt3hbvqEKxVi29BzcPT7O2y31HKXGDyWssYd+u7u7kafRyMiJzTNdwA8DFSgU6DVLKVAS5ttloeIPFUety0lxYRx0Ox0ACc48nWrFjCfexHWAD22mFHVRBEwmkTWUQBBW1ZzVIylRBJIMOHxRruyIxZHdDNzbvp9M14K/ajX4yhgxwBww7c5khwOGF2AZuD5kDPF/5hhYNvs5qPCh3KqrwpTV5RPMfnWnflLdleSYDfDMSviBxDluZ56xbklgTBxdggM1TN40Dj+ckbgBLw5/P2KphD0ClOcS9dKPRgx2rH9qA6Dgt++SBkX3Q4gx35YjhBU+9SK985LYbp6D17LIGSG0ZLa7Ymg61U2WrQ+SHEJgVS76LcA3lR91OTMbKHk6ZHF6/LJLIJ6hvHpfcD5UHIXFyAtE8beYUmI+n2W87eIxEMM1DuOclUYEe/ktYED+qTxOmrZLf+/qrkWcyqaMqXWoYecM6HSWUqqOl7skU7qtTjYNeVOuwF9DKvKRNqEBAPqn2dQHaWUCcqdmfU1/Z6XS60UjW8+IFVnygOqgnRDy1nr0mga6oPDjMV7YxYHU+aeS85jsuBuX6fQ7zLpYORbfh4R7lp7M9lhYhvjqHbjbcKN6HvW4/DTTc4ohcfEtBhtndF6pN23CfjuFXrjPIvTyY87Arp28OIUOk+Y6+tW58wY9c8kAtIIxUzcm35DSQDsqR0dZ0uiASlUJ1RpqLCvDxJkupNB7/cEeYhdXpNru2qNqsBRFTYHazM1s10ElEvkXyYE5Q79IrYrlM7V+xl1dNrlBljV/eCjuETMVLQI36hEkiH9prBcWUYQOYeQBefDo766nSg4hLLvUSCfUve3odpNzLiqODqMHJGgBpUyoPrSZ8P8JCJURBmy+1AGcGUarQv1FL1ACPdkCzSIdDqu8eFga2neZOQx7GK9QJlg7sjtHttjBZQoHFCZ7jfc8MWLsUsEsvqOzQs8fYASy/7cT/OOZ/ta7wTFN8ed6mA23zxe8xjiNSbF1SThE1A80HhRjYYsEuN8Obd2VAdhkphuz0rW4MP8064052jYVMDOcaOeqcP3jfTt1RM5eTKp+30R9Pr3u0OkvrZCb+04jQv4iMwu0WgV7qq9YAX17dtklQNFVVwC5rXM3/khbvLxqz46laMj2B3Z7pwkDtdpumW9qUGSDRc3FSLwF7tB3Gd4dAeusi+VcMBnpYTmxjcVm5LTPySq7osk5VPDzuC4xHfzXUX3CD7dvXtY5bkrI+FkCymnuofjah3uj593HvuYOggpia5XretOqZm9PxH7+JLqIvkVHsA2JfH4xo/fK8y+iqHidEPuXJ4oQZ+6zhuvasBbZ9rmndBBP5+zXM/EvmK3CVZcSSfhvDVvV409WgtvCLuekvL8Ij3r0MvaxQINUFKnlELIMrMTqI8HjS7Ma+m7dYt3CmqTvm4rMURMl3IEEaslUyHvlkYIKardkR9ys7pfJXM29A/WSkmCgbJCScpRKFOlV6UOhpXfbJQTeTuuU99Y9RhqNW7E45kpi79xdbm/Xr1+HQ2jM1JJiN/6sioIUEyqshF4KqQ6oSyXaP7Ldc1Yq+MZOxLokPhwh9tnmh2NjL9oDgVLAu/INOmV1K0N+ZDM3KPUAKDG9z2jidHG8ooAz+zgT2mrDQmDBt7CpLYzAswtrrZo9S6R3adTllmEzDJ0OcMRahK0de9FY9hRI5XP+oXepqspPP6xbfx2+pi8R5trwiJ9HVOuVjfbrIb7wEKkvAlF5OYcaQnD5di9+X2dkzgYongwgrgHXX84YShpxheolFhdji1Syg4CnZA1NGYIa/GV2p8L81J4DjPx5cduxsJhUSk29kXhtjdOz8ZuNAMsU5I+Q2eOli9aHtBboffhImSH3XqNHoi7DP8mOhqTJqRI62t4MOw29yGY3GXIdXinggWjeDcZSdvPTyH4hyQzVyoB4XeLqmFRHBKH/OyVKg8zxt5rtR808QvOJ38a3GDwjhG0PnGPE2p1zd5RQ0jSJ7HVSWutmnMNzAhOT0scBA3sUaOUye7y9X9/qT4rOlo0RWlgr4TNxOoU/JoTW2+z/O4VVXrVlV+kNguIWm+hTci88RTa+tsvjEiH8+PsBFeCpnXCXVpGkPGKsoLn/1xL/OqWWNIGfSW0CiCi9JDYjzh4Tn4KhnCfKXwCkr9XibocnMrbPcJe70asb1Xm1iRuc/nudvtIn1YSsyQ0TYDwbt69Z0/31UVsmznVF6iF0TRfV9KDJfnXy8+gW1NUVBiCFn+wyom3SXpXJ3U+rwJaSgvpkJBNJqbkCsyGqkIF92xaY5fP66hLdcxDqOjOCJGKJ7vecllHMx+qeeoAfXEqYqKqtPeD1NdVYtbme7/r+06dlhFguAfrUyGI5hok0w2N4bosYkmmPn6ZSyttE8btPGINCAETXV1T1dhyWlUEGl+0k6F5HaT47Ex8nuhEnuGVVM0bUGjSUACpNLtTUfV3ZmGY6CQvLTHqepkvJU3d/aWQzLgoHd9j0zWb8iEsIJIV4N3MauvoLYauXXVa+y9A9ZelKAxQSLjCqa7vTOivvHMSA80H9K3zxOmAIsWwGRXE+VEucOCq1IeSU0PLCfU+pzJzkayKvQZI2u09nSbO1zG7FukZffOwVitcufTy2KwVSIQelibV2Z+LnKThGP8+sbIaCo1l0yP+CKNVhB08C5SH3ZjymjShfgux70McbzRxzXvX7Mk0JOYTNH64HXl14HjpuqDY9LUngbTPew+w/5NpoRXbpclOVDua8LogmqQKSBg0iN6AvRNr66893wgcRMfYcWx+IaFz2KAMH4azeXkpmtsvhj9/WUrEUfzIsM7fnfZhmt0mt8KjANctgR0asR1awBAHgQejcMSsdBdXGrVeexsMJu+tk0Vylc++ri3mW/lsV6rKrOLQf/gXdSTsHKTOI0FgZ5ivQ4afeCHWK4PQ9PeBdVHuBRx85Way+GxzZrs6LB21kE3FqwgWPHPuCUF66fqUb3aYrhvw7hEvPkJgB926KOv61O17BS52UPCFRE/WIWG6lN0DSyFk2TSk30PtAkfWv0qbgLnO5ErkYw80RUksZSElVYEBkqZbkIlF5ovjO+WQ++9bRzAwEkvcrmg3a0XVlUZerYLuBdPExbCriwuLgQT3lN39eBBBrV35tk3mBCrRI4vrOzfp5EmmFfjkmM7iunrSoQRsmxDB6yRyTUVKTf4dj8wg1WqKlZE2c1X258OjXMFAkhtyqaICR5kAYD2HDrM97keXBrEUZsQwvMS8uoR7GL+QRUskDTPdXk7z2xRhzCvOVMmFE/B3zpWs5wUbEyimot9TxDXFn5xb9GM1GIrx66vmmOdes27qR8vUbY8EpNsZU83X8Ro7/2BOasx+ksDDWkyN2L1OL6pniiSg0faOWGCpuN0BXboWeWFMWZCChd15DujboF3kxChWgYbIRlzob0MuQNszPsZv+hnbqcGJzUOobmNfTmxkPFjs6VvCQ54+KhXnU32cFcKTZaTZYgKr/dLwd+b5iaFugxR7J/48DwkFkmTg5Y2isiO4X7X+kdfl7TVpNK3KGgoIxbEA+slnnpcwGq2pS2qn2B5RhFv0NpInCl/tIVn7QuWQIdgL9vz9LrOw775s9c4FbQG/ZTyBtJc7IUVnoRqsNVZGQkcC9cju5tG0jT+kZRaFGiAiEarJw8skzIwmwzte6NV3LRViqqi2nQWKi6rsPZdUP5M5fE7Iyq/aSL9YQuV5X/c76V+Z9jxl5ntH1qo7E889z81in47jfLve6T/aMD97zbn/vLE+x/3x/775txxOPX9/OspxSkbGrzbgFf8DA== -------------------------------------------------------------------------------- /pictures/Azure Modern Data Platform.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/Azure Modern Data Platform.png -------------------------------------------------------------------------------- /pictures/ELT_Framework_Dataflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/ELT_Framework_Dataflow.png -------------------------------------------------------------------------------- /pictures/Ingestion_Pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/Ingestion_Pipeline.png -------------------------------------------------------------------------------- /pictures/L1Transformation_Pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/L1Transformation_Pipeline.png -------------------------------------------------------------------------------- /pictures/L2Transformation_Pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/L2Transformation_Pipeline.png --------------------------------------------------------------------------------