├── .github
└── workflows
│ └── ControlDB-deployment.yml
├── .gitignore
├── LICENSE
├── README.md
├── devops
├── azure-pipeline-cicd-controldb.yml
├── azure-pipeline-cicd-db.yml
└── templates
│ ├── build-sqldb.yml
│ └── deploy-sqldb.yml
├── elt-framework
└── ControlDB
│ ├── ControlDB.sln
│ ├── ControlDB.sqlproj
│ ├── ELT
│ ├── Constraints
│ │ ├── CC_IngestDefinition_DataMapping.sql
│ │ ├── CC_IngestDefinition_MaxIntervalMinutes.sql
│ │ ├── CC_IngestDefinition_MaxIntervalNumber.sql
│ │ ├── CC_IngestDefinition_SourceStructure.sql
│ │ ├── CC_IngestInstance_IngestStatus.sql
│ │ ├── CC_L1TransformDefinition_CustomParameters.sql
│ │ ├── CC_L1TransformDefinition_OutputDWTableWriteMode.sql
│ │ ├── CC_L1TransformDefinition_OutputL1CuratedFileWriteMode.sql
│ │ ├── CC_L1TransformInstance_L1TransformStatus.sql
│ │ ├── CC_L1TransformInstance_OutputDWTableWriteMode.sql
│ │ ├── CC_L1TransformInstance_OutputL1CuratedFileWriteMode.sql
│ │ ├── CC_L2TransformDefinition_CustomParameters.sql
│ │ ├── CC_L2TransformDefinition_InputType.sql
│ │ ├── CC_L2TransformDefinition_MaxIntervalMinutes.sql
│ │ ├── CC_L2TransformDefinition_MaxIntervalNumber.sql
│ │ ├── CC_L2TransformDefinition_OutputDWTableWriteMode.sql
│ │ ├── CC_L2TransformDefinition_OutputL2CuratedFileWriteMode.sql
│ │ ├── CC_L2TransformInstance_L2TransformStatus.sql
│ │ ├── CC_L2TransformInstance_OutputDWTableWriteMode.sql
│ │ ├── CC_L2TransformInstance_OutputL2CuratedFileWriteMode.sql
│ │ ├── DC_Attribute_CreatedBy.sql
│ │ ├── DC_Attribute_CreatedTimestamp.sql
│ │ ├── DC_Attribute_ModifiedBy.sql
│ │ ├── DC_Attribute_ModifiedTimestamp.sql
│ │ ├── DC_IngestDefinition_CreatedBy.sql
│ │ ├── DC_IngestDefinition_CreatedTimestamp.sql
│ │ ├── DC_IngestDefinition_DelayL1TransformationFlag.sql
│ │ ├── DC_IngestDefinition_DelayL2TransformationFlag.sql
│ │ ├── DC_IngestDefinition_MaxRetries.sql
│ │ ├── DC_IngestDefinition_ModifiedBy.sql
│ │ ├── DC_IngestDefinition_ModifiedTimestamp.sql
│ │ ├── DC_IngestDefinition_RunSequence.sql
│ │ ├── DC_L1TransformDefinition_CreatedBy.sql
│ │ ├── DC_L1TransformDefinition_CreatedTimestamp.sql
│ │ ├── DC_L1TransformDefinition_MaxRetries.sql
│ │ ├── DC_L1TransformDefinition_ModifiedBy.sql
│ │ ├── DC_L1TransformDefinition_ModifiedTimestamp.sql
│ │ ├── DC_L2TransformDefinition.RunSequence.sql
│ │ ├── DC_L2TransformDefinition_CreatedBy.sql
│ │ ├── DC_L2TransformDefinition_CreatedTimestamp.sql
│ │ ├── DC_L2TransformDefinition_MaxRetries.sql
│ │ ├── DC_L2TransformDefinition_ModifiedBy.sql
│ │ ├── DC_L2TransformDefinition_ModifiedTimestamp.sql
│ │ └── PK_L1TransformInstance.sql
│ ├── Database.sqlproj
│ ├── Functions
│ │ ├── uf_GetAestDateTime.sql
│ │ ├── uf_GetColumnMapping.sql
│ │ ├── uf_GetIngestionColumnList.sql
│ │ ├── uf_GetSalesforceQuery.sql
│ │ └── uf_GetTabularTranslatorMappingJson.sql
│ ├── Indexes
│ │ ├── UI_IngestDefinition.sql
│ │ ├── UI_IngestInstance.sql
│ │ ├── UI_L1TransformDefinition.sql
│ │ ├── UI_L1TransformInstance.sql
│ │ ├── UI_L2TransformDefinition.sql
│ │ └── UI_L2TransformInstance.sql
│ ├── Keys
│ │ ├── FK_Attribute_IngestID.sql
│ │ ├── FK_Attribute_L1TransformID.sql
│ │ ├── FK_Attribute_L2TransformID.sql
│ │ ├── FK_IngestInstance_IngestID.sql
│ │ ├── FK_L1TransformDefinition_IngestID.sql
│ │ ├── FK_L1TransformInstance_IngestID.sql
│ │ ├── FK_L1TransformInstance_L1TransformID.sql
│ │ ├── FK_L2TransformDefinition_IngestID.sql
│ │ ├── FK_L2TransformInstance_IngestID.sql
│ │ ├── FK_L2TransformInstance_L1TransformID.sql
│ │ ├── FK_L2TransformInstance_L2TransformID.sql
│ │ ├── PK_Attribute.sql
│ │ ├── PK_IngestDefinition.sql
│ │ ├── PK_IngestInstance.sql
│ │ ├── PK_L1TransformDefinition.sql
│ │ ├── PK_L2TransformDefinition.sql
│ │ └── PK_L2TransformInstance.sql
│ ├── Stored Procedures
│ │ ├── GetIngestDefinition.sql
│ │ ├── GetIngestDefinition_FileDrop.sql
│ │ ├── GetRunningTransformInstances.sql
│ │ ├── GetTransformDefinition_L1.sql
│ │ ├── GetTransformDefinition_L2.sql
│ │ ├── GetTransformInstance_L1.sql
│ │ ├── GetTransformInstance_L2.sql
│ │ ├── InsertIngestInstance.sql
│ │ ├── InsertTransformInstance_L1.sql
│ │ ├── InsertTransformInstance_L2.sql
│ │ ├── UpdateIngestDefinition.sql
│ │ ├── UpdateIngestInstance.sql
│ │ ├── UpdateTransformDefinition_L2.sql
│ │ ├── UpdateTransformInstance_L1.sql
│ │ └── UpdateTransformInstance_L2.sql
│ ├── Tables
│ │ ├── ColumnMapping.sql
│ │ ├── IngestDefinition.sql
│ │ ├── IngestInstance.sql
│ │ ├── L1TransformDefinition.sql
│ │ ├── L1TransformInstance.sql
│ │ ├── L2TransformDefinition.sql
│ │ └── L2TransformInstance.sql
│ └── elt.sql
│ ├── Post Deployment Script
│ ├── Script.PostDeployment.sql
│ └── datasources
│ │ ├── RestAPI
│ │ ├── IngestDefinition
│ │ │ ├── AzureRestAPI.sql
│ │ │ └── PurviewRestAPI.sql
│ │ └── L1TransformDefinition
│ │ │ ├── L1T_AzureRestAPI.sql
│ │ │ └── L1T_PurviewRestAPI.sql
│ │ └── UploadFiles
│ │ ├── IngestDefinition
│ │ └── CustomFormRecognizerModel.sql
│ │ └── L1TransformDefinition
│ │ └── L1T_CustomFormRecognizerModel.sql
│ └── Pre Deployment Script
│ ├── DBRoles.PreDeployment.sql
│ └── Script.PreDeployment.sql
└── pictures
├── Azure Modern Data Platform - Ingest Definition.png
├── Azure Modern Data Platform - L1 Transform Definition.png
├── Azure Modern Data Platform - L2 Transform Definition.png
├── Azure Modern Data Platform.drawio
├── Azure Modern Data Platform.png
├── ELT_Framework_Dataflow.png
├── Ingestion_Pipeline.png
├── L1Transformation_Pipeline.png
└── L2Transformation_Pipeline.png
/.github/workflows/ControlDB-deployment.yml:
--------------------------------------------------------------------------------
1 | name: ControlDB Deployment
2 | on: workflow_dispatch
3 | jobs:
4 | release:
5 | runs-on: windows-latest
6 | steps:
7 | - name: Checkout code
8 | uses: actions/checkout@v3
9 | - name: Display checkout files and folders
10 | run: Get-ChildItem ${{github.workspace}} -Recurse
11 | - name: Azure Login
12 | uses: Azure/login@v1
13 | with:
14 | creds: '{"clientId":"${{ secrets.CLIENT_ID }}","clientSecret":"${{ secrets.CLIENT_SECRET }}","subscriptionId":"${{ secrets.SUBSCRIPTION_ID }}","tenantId":"${{ secrets.TENANT_ID }}"}'
15 | - name: Build and Deploy SQL Database
16 | uses: Azure/sql-action@v2
17 | with:
18 | connection-string: ${{secrets.CONTROLDB_CONNECTIONSTRING}}
19 | path: ${{github.workspace}}\elt-framework\ControlDB\ELT\Database.sqlproj
20 | action: 'Publish'
21 | arguments: '/p:DropPermissionsNotInSource=false /p:BlockOnPossibleDataLoss=false /p:DropRoleMembersNotInSource=false /p:DropObjectsNotInSource=false /p:IgnoreColumnOrder=true /p:IgnorePermissions=true'
22 | - name: Azure Logout
23 | run: |
24 | az logout
25 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | ## Ignore Visual Studio temporary files, build results, and
2 | ## files generated by popular Visual Studio add-ons.
3 | ##
4 | ## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore
5 |
6 | # User-specific files
7 | *.rsuser
8 | *.suo
9 | *.user
10 | *.userosscache
11 | *.sln.docstates
12 |
13 | # User-specific files (MonoDevelop/Xamarin Studio)
14 | *.userprefs
15 |
16 | # Mono auto generated files
17 | mono_crash.*
18 |
19 | # Build results
20 | [Dd]ebug/
21 | [Dd]ebugPublic/
22 | [Rr]elease/
23 | [Rr]eleases/
24 | x64/
25 | x86/
26 | [Ww][Ii][Nn]32/
27 | [Aa][Rr][Mm]/
28 | [Aa][Rr][Mm]64/
29 | bld/
30 | [Bb]in/
31 | [Oo]bj/
32 | [Ll]og/
33 | [Ll]ogs/
34 |
35 | # Visual Studio 2015/2017 cache/options directory
36 | .vs/
37 | # Uncomment if you have tasks that create the project's static files in wwwroot
38 | #wwwroot/
39 |
40 | # Visual Studio 2017 auto generated files
41 | Generated\ Files/
42 |
43 | # MSTest test Results
44 | [Tt]est[Rr]esult*/
45 | [Bb]uild[Ll]og.*
46 |
47 | # NUnit
48 | *.VisualState.xml
49 | TestResult.xml
50 | nunit-*.xml
51 |
52 | # Build Results of an ATL Project
53 | [Dd]ebugPS/
54 | [Rr]eleasePS/
55 | dlldata.c
56 |
57 | # Benchmark Results
58 | BenchmarkDotNet.Artifacts/
59 |
60 | # .NET Core
61 | project.lock.json
62 | project.fragment.lock.json
63 | artifacts/
64 |
65 | # ASP.NET Scaffolding
66 | ScaffoldingReadMe.txt
67 |
68 | # StyleCop
69 | StyleCopReport.xml
70 |
71 | # Files built by Visual Studio
72 | *_i.c
73 | *_p.c
74 | *_h.h
75 | *.ilk
76 | *.meta
77 | *.obj
78 | *.iobj
79 | *.pch
80 | *.pdb
81 | *.ipdb
82 | *.pgc
83 | *.pgd
84 | *.rsp
85 | *.sbr
86 | *.tlb
87 | *.tli
88 | *.tlh
89 | *.tmp
90 | *.tmp_proj
91 | *_wpftmp.csproj
92 | *.log
93 | *.tlog
94 | *.vspscc
95 | *.vssscc
96 | .builds
97 | *.pidb
98 | *.svclog
99 | *.scc
100 |
101 | # Chutzpah Test files
102 | _Chutzpah*
103 |
104 | # Visual C++ cache files
105 | ipch/
106 | *.aps
107 | *.ncb
108 | *.opendb
109 | *.opensdf
110 | *.sdf
111 | *.cachefile
112 | *.VC.db
113 | *.VC.VC.opendb
114 |
115 | # Visual Studio profiler
116 | *.psess
117 | *.vsp
118 | *.vspx
119 | *.sap
120 |
121 | # Visual Studio Trace Files
122 | *.e2e
123 |
124 | # TFS 2012 Local Workspace
125 | $tf/
126 |
127 | # Guidance Automation Toolkit
128 | *.gpState
129 |
130 | # ReSharper is a .NET coding add-in
131 | _ReSharper*/
132 | *.[Rr]e[Ss]harper
133 | *.DotSettings.user
134 |
135 | # TeamCity is a build add-in
136 | _TeamCity*
137 |
138 | # DotCover is a Code Coverage Tool
139 | *.dotCover
140 |
141 | # AxoCover is a Code Coverage Tool
142 | .axoCover/*
143 | !.axoCover/settings.json
144 |
145 | # Coverlet is a free, cross platform Code Coverage Tool
146 | coverage*.json
147 | coverage*.xml
148 | coverage*.info
149 |
150 | # Visual Studio code coverage results
151 | *.coverage
152 | *.coveragexml
153 |
154 | # NCrunch
155 | _NCrunch_*
156 | .*crunch*.local.xml
157 | nCrunchTemp_*
158 |
159 | # MightyMoose
160 | *.mm.*
161 | AutoTest.Net/
162 |
163 | # Web workbench (sass)
164 | .sass-cache/
165 |
166 | # Installshield output folder
167 | [Ee]xpress/
168 |
169 | # DocProject is a documentation generator add-in
170 | DocProject/buildhelp/
171 | DocProject/Help/*.HxT
172 | DocProject/Help/*.HxC
173 | DocProject/Help/*.hhc
174 | DocProject/Help/*.hhk
175 | DocProject/Help/*.hhp
176 | DocProject/Help/Html2
177 | DocProject/Help/html
178 |
179 | # Click-Once directory
180 | publish/
181 |
182 | # Publish Web Output
183 | *.[Pp]ublish.xml
184 | *.azurePubxml
185 | # Note: Comment the next line if you want to checkin your web deploy settings,
186 | # but database connection strings (with potential passwords) will be unencrypted
187 | *.pubxml
188 | *.publishproj
189 |
190 | # Microsoft Azure Web App publish settings. Comment the next line if you want to
191 | # checkin your Azure Web App publish settings, but sensitive information contained
192 | # in these scripts will be unencrypted
193 | PublishScripts/
194 |
195 | # NuGet Packages
196 | *.nupkg
197 | # NuGet Symbol Packages
198 | *.snupkg
199 | # The packages folder can be ignored because of Package Restore
200 | **/[Pp]ackages/*
201 | # except build/, which is used as an MSBuild target.
202 | !**/[Pp]ackages/build/
203 | # Uncomment if necessary however generally it will be regenerated when needed
204 | #!**/[Pp]ackages/repositories.config
205 | # NuGet v3's project.json files produces more ignorable files
206 | *.nuget.props
207 | *.nuget.targets
208 |
209 | # Microsoft Azure Build Output
210 | csx/
211 | *.build.csdef
212 |
213 | # Microsoft Azure Emulator
214 | ecf/
215 | rcf/
216 |
217 | # Windows Store app package directories and files
218 | AppPackages/
219 | BundleArtifacts/
220 | Package.StoreAssociation.xml
221 | _pkginfo.txt
222 | *.appx
223 | *.appxbundle
224 | *.appxupload
225 |
226 | # Visual Studio cache files
227 | # files ending in .cache can be ignored
228 | *.[Cc]ache
229 | # but keep track of directories ending in .cache
230 | !?*.[Cc]ache/
231 |
232 | # Others
233 | ClientBin/
234 | ~$*
235 | *~
236 | *.dbmdl
237 | *.dbproj.schemaview
238 | *.jfm
239 | *.pfx
240 | *.publishsettings
241 | orleans.codegen.cs
242 |
243 | # Including strong name files can present a security risk
244 | # (https://github.com/github/gitignore/pull/2483#issue-259490424)
245 | #*.snk
246 |
247 | # Since there are multiple workflows, uncomment next line to ignore bower_components
248 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
249 | #bower_components/
250 |
251 | # RIA/Silverlight projects
252 | Generated_Code/
253 |
254 | # Backup & report files from converting an old project file
255 | # to a newer Visual Studio version. Backup files are not needed,
256 | # because we have git ;-)
257 | _UpgradeReport_Files/
258 | Backup*/
259 | UpgradeLog*.XML
260 | UpgradeLog*.htm
261 | ServiceFabricBackup/
262 | *.rptproj.bak
263 |
264 | # SQL Server files
265 | *.mdf
266 | *.ldf
267 | *.ndf
268 |
269 | # Business Intelligence projects
270 | *.rdl.data
271 | *.bim.layout
272 | *.bim_*.settings
273 | *.rptproj.rsuser
274 | *- [Bb]ackup.rdl
275 | *- [Bb]ackup ([0-9]).rdl
276 | *- [Bb]ackup ([0-9][0-9]).rdl
277 |
278 | # Microsoft Fakes
279 | FakesAssemblies/
280 |
281 | # GhostDoc plugin setting file
282 | *.GhostDoc.xml
283 |
284 | # Node.js Tools for Visual Studio
285 | .ntvs_analysis.dat
286 | node_modules/
287 |
288 | # Visual Studio 6 build log
289 | *.plg
290 |
291 | # Visual Studio 6 workspace options file
292 | *.opt
293 |
294 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
295 | *.vbw
296 |
297 | # Visual Studio 6 auto-generated project file (contains which files were open etc.)
298 | *.vbp
299 |
300 | # Visual Studio 6 workspace and project file (working project files containing files to include in project)
301 | *.dsw
302 | *.dsp
303 |
304 | # Visual Studio 6 technical files
305 | *.ncb
306 | *.aps
307 |
308 | # Visual Studio LightSwitch build output
309 | **/*.HTMLClient/GeneratedArtifacts
310 | **/*.DesktopClient/GeneratedArtifacts
311 | **/*.DesktopClient/ModelManifest.xml
312 | **/*.Server/GeneratedArtifacts
313 | **/*.Server/ModelManifest.xml
314 | _Pvt_Extensions
315 |
316 | # Paket dependency manager
317 | .paket/paket.exe
318 | paket-files/
319 |
320 | # FAKE - F# Make
321 | .fake/
322 |
323 | # CodeRush personal settings
324 | .cr/personal
325 |
326 | # Python Tools for Visual Studio (PTVS)
327 | __pycache__/
328 | *.pyc
329 |
330 | # Cake - Uncomment if you are using it
331 | # tools/**
332 | # !tools/packages.config
333 |
334 | # Tabs Studio
335 | *.tss
336 |
337 | # Telerik's JustMock configuration file
338 | *.jmconfig
339 |
340 | # BizTalk build output
341 | *.btp.cs
342 | *.btm.cs
343 | *.odx.cs
344 | *.xsd.cs
345 |
346 | # OpenCover UI analysis results
347 | OpenCover/
348 |
349 | # Azure Stream Analytics local run output
350 | ASALocalRun/
351 |
352 | # MSBuild Binary and Structured Log
353 | *.binlog
354 |
355 | # NVidia Nsight GPU debugger configuration file
356 | *.nvuser
357 |
358 | # MFractors (Xamarin productivity tool) working folder
359 | .mfractor/
360 |
361 | # Local History for Visual Studio
362 | .localhistory/
363 |
364 | # Visual Studio History (VSHistory) files
365 | .vshistory/
366 |
367 | # BeatPulse healthcheck temp database
368 | healthchecksdb
369 |
370 | # Backup folder for Package Reference Convert tool in Visual Studio 2017
371 | MigrationBackup/
372 |
373 | # Ionide (cross platform F# VS Code tools) working folder
374 | .ionide/
375 |
376 | # Fody - auto-generated XML schema
377 | FodyWeavers.xsd
378 |
379 | # VS Code files for those working on multiple tools
380 | .vscode/*
381 | !.vscode/settings.json
382 | !.vscode/tasks.json
383 | !.vscode/launch.json
384 | !.vscode/extensions.json
385 | *.code-workspace
386 |
387 | # Local History for Visual Studio Code
388 | .history/
389 |
390 | # Windows Installer files from build outputs
391 | *.cab
392 | *.msi
393 | *.msix
394 | *.msm
395 | *.msp
396 |
397 | # JetBrains Rider
398 | *.sln.iml
399 | /elt-framework/.azuredatastudio
400 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # elt-framework
2 | The Extract Load Transform (ELT) framework is a metadata-driven orchestration framework designed for modern cloud data platforms. It simplifies ingestion and transformation pipelines, ensuring a consistent development experience and ease of maintenance. The framework supports batch ingestion and has been extensively tested with **[Microsoft Fabric](https://github.com/bennyaustin/fabric-dataplatform)** and Azure managed services like **[Azure Databricks](https://github.com/rorymcmanus87/databricks-dataplatform)** and **[Azure Synapse](https://github.com/bennyaustin/synapse-dataplatform)**. It utilizes an ANSI-compatible control database as the metadata repository.
3 |
4 | ## Key Features:
5 | * **Configurable and Extendable:** Easily adapt the framework to meet specific needs.
6 | * **Data Source Agnostic:** Ingest data from various sources such as databases, Delta Lake, REST API, flat files, JSON, XML, without storing connection strings as metadata.
7 | * **Delta and Full Loads:** Support for both incremental and full data loads.
8 | * **Re-run and Retry Capability:** Automatically handle failures without manual intervention.
9 | * **In-built Audit Tracking:** Track data processing activities with built-in audit capabilities.
10 | * **Extended Audit Capability:** Enhance audit tracking with Azure PaaS services like Diagnostic Logging.
11 | * **Eliminates Manual Data Patching:** Streamline data processing by removing the need for manual interventions.
12 | * **Data Lineage Support:** Maintain data lineage throughout the data lifecycle.
13 | * **Level1 and Level2 Transformations:** Support for one-to-many and many-to-many transformations.
14 | * **On-demand Pipeline and Transformation Management:** Enable or disable pipelines and transformations as needed.
15 |
16 | ## Documentation
17 | Key concepts and configuration metadata explained in detail at **[Wiki](https://github.com/bennyaustin/elt-framework/wiki)**
18 |
19 | ## Getting Started
20 | To get started follow these steps:
21 | 1. **Clone or Fork the Repository**: Start by cloning or forking the repository from [github.com/bennyaustin/elt-framework](https://github.com/bennyaustin/elt-framework).
22 |
23 | 2. **Deploy ControlDB**: The GitHub Action [workflows/ControlDB-deployment.yml](https://github.com/bennyaustin/elt-framework/blob/main/.github/workflows/ControlDB-deployment.yml) executes the workflow to deploy controlDB objects.
24 |
25 | **Pre-Requisites**
26 | * controlDB is already provisioned by IaC process like [07-IaC-Bicep](https://github.com/bennyaustin/fabric-accelerator/wiki/07-IaC-Bicep) or [iac-synapse-dataplatform](https://github.com/bennyaustin/iac-synapse-dataplatform)
27 | * Create [1](https://learn.microsoft.com/en-us/entra/identity-platform/howto-create-service-principal-portal) or reuse an existing Service Principal. Take note of the Application (client) ID and the secret, they will be required later.
28 | * Grant _db_owner_ permission to the Service Principal
29 | ```sql
30 | CREATE USER [] FROM EXTERNAL PROVIDER
31 | GO
32 | EXEC sp_addrolemember 'db_owner', []
33 | GO
34 | ```
35 |
36 | This GitHub Action requires the following repository secrets:
37 |
38 | 1. **CLIENT_ID**: Client/Application ID of the Service Principal.
39 | 1. **CLIENT_SECRET**: Service Principal Secret.
40 | 1. **SUBSCRIPTION_ID**: Azure Subscription ID of controlDB
41 | 1. **TENANT_ID**: Entra Tenant ID of controlDB
42 | 1. **CONTROLDB_CONNECTIONSTRING**: controlDB connection string in service principal authentication format.
43 | ```
44 | Server=;Authentication=Active Directory Service Principal; Encrypt=True;Database=controlDB;User Id=;Password=
45 | ```
46 | Now, hit the Run Workflow on GitHub action [ControlDB-deployment.yml](https://github.com/bennyaustin/elt-framework/actions/workflows/ControlDB-deployment.yml) to deploy database objects.
47 |
48 | ## Implementation References
49 | * **[Microsoft Fabric data platform using ELT Framework](https://github.com/bennyaustin/fabric-dataplatform)**
50 | * **[Azure Databricks data platform using ELT Framework](https://github.com/bennyaustin/synapse-dataplatform)**
51 | * **[Azure Synapse data platform using ELT Framework](https://github.com/bennyaustin/synapse-dataplatform)**
52 |
53 | ## Collaborate
54 | You can collaborate in various ways, including:
55 | - Pull Requests
56 | - Update/Enrich [Wiki documentation](https://github.com/bennyaustin/elt-framework/wiki)
57 | - Raise [issues](https://github.com/bennyaustin/elt-framework/issues) when you spot one
58 | - Answer questions in the [discussion](https://github.com/bennyaustin/elt-framework/discussions) forum
59 |
60 | Please contact me to be added as a contributor.
61 |
62 | ## Contact
63 | If you have any questions or need support, please contact the maintainer:
64 | - [Benny Austin](https://github.com/bennyaustin)
65 |
--------------------------------------------------------------------------------
/devops/azure-pipeline-cicd-controldb.yml:
--------------------------------------------------------------------------------
1 | # Pipeline to build and deploy controlDB for ELT Framework https://github.com/bennyaustin/elt-framework
2 | # Add steps that build, run tests, deploy, and more:
3 | # https://aka.ms/yaml
4 |
5 | trigger:
6 | - main
7 |
8 | pool:
9 | name: Azure Pipelines
10 | demands:
11 | - msbuild
12 | - visualstudio
13 | vmImage: windows-latest
14 |
15 | steps:
16 | - task: VSBuild@1
17 | displayName: Build database project
18 | inputs:
19 | solution: '**\ControlDB.sln'
20 |
21 | - publish: $(System.DefaultWorkingDirectory)
22 | artifact: dacpac
23 | displayName: Publish Dacpac
24 |
25 | - task: AzureKeyVault@2
26 | displayName: Get keyvault secrets
27 | inputs:
28 | connectedServiceName: ba-devops-spn
29 | KeyVaultName: ba-keyvault1
30 | SecretsFilter: ba-sqlserver1-sqlusername,ba-sqlserver1-password
31 | RunAsPreJob: true
32 | - task: SqlAzureDacpacDeployment@1
33 | displayName: Deploy database
34 | inputs:
35 | ConnectedServiceNameARM: ba-devops-spn
36 | AuthenticationType: server
37 | ServerName: ba-sqlserver1.database.windows.net,1433
38 | DatabaseName: ControlDB
39 | SqlUsername: $(ba-sqlserver1-sqlusername)
40 | SqlPassword: $(ba-sqlserver1-password)
41 | DacpacFile: elt-framework\ControlDB\bin\Release\ControlDB.dacpac
42 | IpDetectionMethod: AutoDetect
43 | DeleteFirewallRule: true
44 | AdditionalArguments: /p:DropPermissionsNotInSource=false /p:DropRoleMembersNotInSource=false /p:DropObjectsNotInSource=true /p:IgnoreColumnOrder=true /p:IgnorePermissions=true
--------------------------------------------------------------------------------
/devops/azure-pipeline-cicd-db.yml:
--------------------------------------------------------------------------------
1 | # Pipeline to build and deploy controlDB for ELT Framework https://github.com/bennyaustin/elt-framework
2 | # Add steps that build, run tests, deploy, and more:
3 | # https://aka.ms/yaml
4 |
5 | trigger:
6 | branches:
7 | include:
8 | - main
9 | paths:
10 | include:
11 | - elt-framework/ControlDB
12 |
13 | pool:
14 | name: Azure Pipelines
15 | demands:
16 | - msbuild
17 | - visualstudio
18 | vmImage: windows-latest
19 |
20 | stages:
21 | - stage: ci
22 | displayName: Build Stage
23 | jobs:
24 | - template: templates/build-sqldb.yml
25 | parameters:
26 | solutionName: ControlDB
27 | buildPath: elt-framework\ControlDB\bin\Release
28 |
29 | - stage: cd_nonprod
30 | displayName: Deploy Stage - Non Prod
31 | dependsOn: ci
32 | condition: succeeded()
33 | jobs:
34 | - template: templates/deploy-sqldb.yml
35 | parameters:
36 | serviceConnectionName: devops-contoso-spn
37 | keyvaultName: ba-kv01-kn3acb6lw3vr4
38 | buildArtifactName: dacpac
39 | targetDatabaseServer: ba-sql01-kn3acb6lw3vr4.database.windows.net,1433
40 | targetDatabase: controlDB
41 |
42 | - stage: cd_prod
43 | displayName: Deploy Stage - Prod
44 | dependsOn: ci
45 | condition: succeeded()
46 | jobs:
47 | - template: templates/deploy-sqldb.yml
48 | parameters:
49 | serviceConnectionName: devops-contoso-spn
50 | keyvaultName: ba-kv01-o5rium4bfcf6a
51 | buildArtifactName: dacpac
52 | targetDatabaseServer: ba-sql01-o5rium4bfcf6a.database.windows.net,1433
53 | targetDatabase: controlDB
--------------------------------------------------------------------------------
/devops/templates/build-sqldb.yml:
--------------------------------------------------------------------------------
1 | parameters:
2 | solutionName:
3 | buildPath:
4 |
5 |
6 | jobs:
7 | - job:
8 | displayName: Build SQL DB Job
9 | steps:
10 | - task: VSBuild@1
11 | displayName: Build db solution ${{parameters.solutionName}}
12 | inputs:
13 | solution: '**\${{parameters.solutionName}}.sln'
14 |
15 | - publish: $(System.DefaultWorkingDirectory)/${{parameters.buildPath}}
16 | artifact: dacpac
17 | displayName: Publish ${{parameters.solutionName}} dacpac
--------------------------------------------------------------------------------
/devops/templates/deploy-sqldb.yml:
--------------------------------------------------------------------------------
1 | parameters:
2 | serviceConnectionName:
3 | keyvaultName:
4 | buildArtifactName:
5 | targetDatabaseServer:
6 | targetDatabase:
7 |
8 |
9 | jobs:
10 | - job: Deploy
11 | displayName: Deploy SQL DB Job
12 | steps:
13 | - task: AzureKeyVault@2
14 | displayName: Get secrets
15 | inputs:
16 | connectedServiceName: ${{parameters.serviceConnectionName}}
17 | KeyVaultName: ${{parameters.keyvaultName}}
18 | SecretsFilter: sqlserver-admin-username,sqlserver-admin-password
19 | RunAsPreJob: true
20 |
21 | - task: DownloadPipelineArtifact@2
22 | displayName: Download dacpac
23 | inputs:
24 | source: current
25 | artifactName: ${{parameters.buildArtifactName}}
26 | path: $(System.DefaultWorkingDirectory)
27 |
28 | - task: SqlAzureDacpacDeployment@1
29 | displayName: Deploy database - ${{parameters.targetDatabase}}
30 | inputs:
31 | ConnectedServiceNameARM: ${{parameters.serviceConnectionName}}
32 | AuthenticationType: server
33 | ServerName: ${{parameters.targetDatabaseServer}}
34 | DatabaseName: ${{parameters.targetDatabase}}
35 | SqlUsername: $(sqlserver-admin-username)
36 | SqlPassword: $(sqlserver-admin-password)
37 | DacpacFile: ${{parameters.targetDatabase}}.dacpac
38 | IpDetectionMethod: AutoDetect
39 | DeleteFirewallRule: true
40 | AdditionalArguments: /p:DropPermissionsNotInSource=false /p:DropRoleMembersNotInSource=false /p:DropObjectsNotInSource=true /p:IgnoreColumnOrder=true /p:IgnorePermissions=true /p:DoNotDropObjectTypes=Users;Credentials;Permissions;RoleMembership;SecurityPolicies;Logins;ServerRoleMembership;ServerRoles
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ControlDB.sln:
--------------------------------------------------------------------------------
1 |
2 | Microsoft Visual Studio Solution File, Format Version 12.00
3 | # Visual Studio Version 17
4 | VisualStudioVersion = 17.2.32602.215
5 | MinimumVisualStudioVersion = 10.0.40219.1
6 | Project("{00D1A9C2-B5F0-4AF3-8072-F6C62B433612}") = "ControlDB", "ControlDB.sqlproj", "{B4521477-9A6C-4EBC-A067-02EE9F0B9978}"
7 | EndProject
8 | Global
9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution
10 | Debug|Any CPU = Debug|Any CPU
11 | Release|Any CPU = Release|Any CPU
12 | EndGlobalSection
13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution
14 | {B4521477-9A6C-4EBC-A067-02EE9F0B9978}.Debug|Any CPU.ActiveCfg = Release|Any CPU
15 | {B4521477-9A6C-4EBC-A067-02EE9F0B9978}.Debug|Any CPU.Build.0 = Release|Any CPU
16 | {B4521477-9A6C-4EBC-A067-02EE9F0B9978}.Debug|Any CPU.Deploy.0 = Release|Any CPU
17 | {B4521477-9A6C-4EBC-A067-02EE9F0B9978}.Release|Any CPU.ActiveCfg = Release|Any CPU
18 | {B4521477-9A6C-4EBC-A067-02EE9F0B9978}.Release|Any CPU.Build.0 = Release|Any CPU
19 | {B4521477-9A6C-4EBC-A067-02EE9F0B9978}.Release|Any CPU.Deploy.0 = Release|Any CPU
20 | EndGlobalSection
21 | GlobalSection(SolutionProperties) = preSolution
22 | HideSolutionNode = FALSE
23 | EndGlobalSection
24 | GlobalSection(ExtensibilityGlobals) = postSolution
25 | SolutionGuid = {A4971EE5-99CC-4196-B086-305A02EF4317}
26 | EndGlobalSection
27 | EndGlobal
28 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ControlDB.sqlproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Debug
5 | AnyCPU
6 | ControlDB
7 | 2.0
8 | 4.1
9 | {b4521477-9a6c-4ebc-a067-02ee9f0b9978}
10 | Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider
11 | Database
12 |
13 |
14 | ControlDB
15 | ControlDB
16 | 1033,CI
17 | BySchemaAndSchemaType
18 | True
19 | v4.7.2
20 | CS
21 | Properties
22 | False
23 | True
24 | True
25 | dbo
26 |
27 | SQL_Latin1_General_CP1_CI_AS
28 | False
29 |
30 |
31 | bin\Release\
32 | $(MSBuildProjectName).sql
33 | False
34 | pdbonly
35 | true
36 | false
37 | true
38 | prompt
39 | 4
40 |
41 |
42 | bin\Debug\
43 | $(MSBuildProjectName).sql
44 | false
45 | true
46 | full
47 | false
48 | true
49 | true
50 | prompt
51 | 4
52 |
53 |
54 | False
55 |
56 |
57 | 11.0
58 |
59 | True
60 | 11.0
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 | Always
170 |
171 |
172 | Always
173 |
174 |
175 | Always
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 | Always
195 |
196 |
197 | Always
198 |
199 |
200 |
201 |
202 |
203 | Always
204 |
205 |
206 |
207 |
208 | 1.0.0
209 | All
210 |
211 |
212 |
213 |
214 |
215 |
216 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_IngestDefinition_DataMapping.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [CC_IngestDefinition_DataMapping]
3 | CHECK (ISJSON([DataMapping]) = 1)
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_IngestDefinition_MaxIntervalMinutes.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [CC_IngestDefinition_MaxIntervalMinutes]
3 | CHECK ([MaxIntervalMinutes] IS NULL OR [MaxIntervalMinutes] > 0)
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_IngestDefinition_MaxIntervalNumber.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [CC_IngestDefinition_MaxIntervalNumber]
3 | CHECK ([MaxIntervalNumber] IS NULL OR [MaxIntervalNumber] > 0)
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_IngestDefinition_SourceStructure.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [CC_IngestDefinition_SourceStructure]
3 | CHECK (ISJSON([SourceStructure]) = 1)
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_IngestInstance_IngestStatus.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestInstance]
2 | ADD CONSTRAINT [CC_IngestInstance_IngestStatus]
3 | CHECK ([IngestStatus]='ReRunFailure' OR [IngestStatus]='ReRunSuccess' OR [IngestStatus]='Running' OR [IngestStatus]='Failure' OR [IngestStatus]='Success')
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L1TransformDefinition_CustomParameters.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformDefinition]
2 | ADD CONSTRAINT [CC_L1TransformDefinition_CustomParameters]
3 | CHECK (ISJSON([CustomParameters]) = 1)
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L1TransformDefinition_OutputDWTableWriteMode.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformDefinition]
2 | ADD CONSTRAINT [CC_L1TransformDefinition_OutputDWTableWriteMode]
3 | CHECK ([OutputDWTableWriteMode]='append' OR [OutputDWTableWriteMode]='overwrite' OR [OutputDWTableWriteMode]='error' OR [OutputDWTableWriteMode]='errorifexists' OR [OutputDWTableWriteMode]='ignore')
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L1TransformDefinition_OutputL1CuratedFileWriteMode.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformDefinition]
2 | ADD CONSTRAINT [CC_L1TransformDefinition_OutputL1CuratedFileWriteMode]
3 | CHECK ([OutputL1CuratedFileWriteMode] ='append' OR [OutputL1CuratedFileWriteMode] ='overwrite' OR [OutputL1CuratedFileWriteMode] ='ignore' OR [OutputL1CuratedFileWriteMode] = 'error' OR [OutputL1CuratedFileWriteMode]='errorifexists')
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L1TransformInstance_L1TransformStatus.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformInstance]
2 | ADD CONSTRAINT [CC_L1TransformInstance_L1TransformStatus]
3 | CHECK ([L1TransformStatus] ='ReRunFailure' OR [L1TransformStatus] ='ReRunSuccess' OR [L1TransformStatus] ='Running' OR [L1TransformStatus] ='DWUpload' OR [L1TransformStatus]='Failure' OR [L1TransformStatus]='Success')
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L1TransformInstance_OutputDWTableWriteMode.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformInstance]
2 | ADD CONSTRAINT [CC_L1TransformInstance_OutputDWTableWriteMode]
3 | CHECK ([OutputDWTableWriteMode]='append' OR [OutputDWTableWriteMode]='overwrite' OR [OutputDWTableWriteMode]='error' OR [OutputDWTableWriteMode]='errorifexists' OR [OutputDWTableWriteMode]='ignore')
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L1TransformInstance_OutputL1CuratedFileWriteMode.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformInstance]
2 | ADD CONSTRAINT [CC_L1TransformInstance_OutputL1CuratedFileWriteMode]
3 | CHECK ([OutputL1CuratedFileWriteMode] ='append' OR [OutputL1CuratedFileWriteMode] ='overwrite' OR [OutputL1CuratedFileWriteMode] ='ignore' OR [OutputL1CuratedFileWriteMode] = 'error' OR [OutputL1CuratedFileWriteMode]='errorifexists')
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L2TransformDefinition_CustomParameters.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [CC_L2TransformDefinition_CustomParameters]
3 | CHECK (ISJSON([CustomParameters]) = 1)
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L2TransformDefinition_InputType.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [CC_L2TransformDefinition_InputType]
3 | CHECK ([InputType] ='Raw' OR [InputType] ='Curated' OR [InputType] ='Datawarehouse')
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L2TransformDefinition_MaxIntervalMinutes.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [CC_L2TransformDefinition_MaxIntervalMinutes]
3 | CHECK ([MaxIntervalMinutes] IS NULL OR [MaxIntervalMinutes] > 0)
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L2TransformDefinition_MaxIntervalNumber.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [CC_L2TransformDefinition_MaxIntervalNumber]
3 | CHECK ([MaxIntervalNumber] IS NULL OR [MaxIntervalNumber] > 0)
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L2TransformDefinition_OutputDWTableWriteMode.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [CC_L2TransformDefinition_OutputDWTableWriteMode]
3 | CHECK ([OutputDWTableWriteMode]='append' OR [OutputDWTableWriteMode]='overwrite' OR [OutputDWTableWriteMode]='ignore' OR [OutputDWTableWriteMode]='error' OR [OutputDWTableWriteMode]='errorifexists')
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L2TransformDefinition_OutputL2CuratedFileWriteMode.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [CC_L2TransformDefinition_OutputL2CuratedFileWriteMode]
3 | CHECK ([OutputL2CuratedFileWriteMode]='append' OR [OutputL2CuratedFileWriteMode]='overwrite' OR [OutputL2CuratedFileWriteMode]='ignore' OR [OutputL2CuratedFileWriteMode]='error' OR [OutputL2CuratedFileWriteMode]='errorifexists')
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L2TransformInstance_L2TransformStatus.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformInstance]
2 | ADD CONSTRAINT [CC_L2TransformInstance_L2TransformStatus]
3 | CHECK ([L2TransformStatus] ='ReRunFailure' OR [L2TransformStatus] ='ReRunSuccess' OR [L2TransformStatus] ='Running' OR [L2TransformStatus] ='DWUpload' OR [L2TransformStatus]='Failure' OR [L2TransformStatus]='Success')
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L2TransformInstance_OutputDWTableWriteMode.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformInstance]
2 | ADD CONSTRAINT [CC_L2TransformInstance_OutputDWTableWriteMode]
3 | CHECK ([OutputDWTableWriteMode]='append' OR [OutputDWTableWriteMode]= 'overwrite' OR [OutputDWTableWriteMode]= 'ignore' OR [OutputDWTableWriteMode]= 'error' OR [OutputDWTableWriteMode]= 'errorifexists')
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/CC_L2TransformInstance_OutputL2CuratedFileWriteMode.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformInstance]
2 | ADD CONSTRAINT [CC_L2TransformInstance_OutputL2CuratedFileWriteMode]
3 | CHECK ([OutputL2CuratedFileWriteMode]='append' OR [OutputL2CuratedFileWriteMode]= 'overwrite' OR [OutputL2CuratedFileWriteMode]= 'ignore' OR [OutputL2CuratedFileWriteMode]= 'error' OR [OutputL2CuratedFileWriteMode]= 'errorifexists')
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_Attribute_CreatedBy.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[ColumnMapping]
2 | ADD CONSTRAINT [DC_Attribute_CreatedBy]
3 | DEFAULT (suser_sname())
4 | FOR [CreatedBy]
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_Attribute_CreatedTimestamp.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[ColumnMapping]
2 | ADD CONSTRAINT [DC_Attribute_CreatedTimestamp]
3 | DEFAULT (CONVERT([datetime],(CONVERT([datetimeoffset],getdate()) AT TIME ZONE 'AUS Eastern Standard Time')))
4 | FOR [CreatedTimestamp]
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_Attribute_ModifiedBy.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[ColumnMapping]
2 | ADD CONSTRAINT [DC_Attribute_ModifiedBy]
3 | DEFAULT (suser_sname())
4 | FOR [ModifiedBy]
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_Attribute_ModifiedTimestamp.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[ColumnMapping]
2 | ADD CONSTRAINT [DC_Attribute_ModifiedTimestamp]
3 | DEFAULT (CONVERT([datetime],(CONVERT([datetimeoffset],getdate()) AT TIME ZONE 'AUS Eastern Standard Time')))
4 | FOR [ModifiedTimestamp]
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_CreatedBy.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [DC_IngestDefinition_CreatedBy]
3 | DEFAULT suser_sname()
4 | FOR [CreatedBy]
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_CreatedTimestamp.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [DC_IngestDefinition_CreatedTimestamp]
3 | DEFAULT CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
4 | FOR [CreatedTimestamp]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_DelayL1TransformationFlag.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [DC_IngestDefinition_DelayL1TransformationFlag]
3 | DEFAULT 1
4 | FOR [DelayL1TransformationFlag]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_DelayL2TransformationFlag.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [DC_IngestDefinition_DelayL2TransformationFlag]
3 | DEFAULT 1
4 | FOR [DelayL2TransformationFlag]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_MaxRetries.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [DC_IngestDefinition_MaxRetries]
3 | DEFAULT 3
4 | FOR [MaxRetries]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_ModifiedBy.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [DC_IngestDefinition_ModifiedBy]
3 | DEFAULT suser_sname()
4 | FOR [ModifiedBy]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_ModifiedTimestamp.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [DC_IngestDefinition_ModifiedTimestamp]
3 | DEFAULT CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
4 | FOR [ModifiedTimestamp]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_IngestDefinition_RunSequence.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [DC_IngestDefinition_RunSequence]
3 | DEFAULT 100
4 | FOR [RunSequence]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_L1TransformDefinition_CreatedBy.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformDefinition]
2 | ADD CONSTRAINT [DC_L1TransformDefinition_CreatedBy]
3 | DEFAULT suser_sname()
4 | FOR [CreatedBy]
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_L1TransformDefinition_CreatedTimestamp.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformDefinition]
2 | ADD CONSTRAINT [DC_L1TransformDefinition_CreatedTimestamp]
3 | DEFAULT CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
4 | FOR [CreatedTimestamp]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_L1TransformDefinition_MaxRetries.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformDefinition]
2 | ADD CONSTRAINT [DC_L1TransformDefinition_MaxRetries]
3 | DEFAULT 3
4 | FOR [MaxRetries]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_L1TransformDefinition_ModifiedBy.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformDefinition]
2 | ADD CONSTRAINT [DC_L1TransformDefinition_ModifiedBy]
3 | DEFAULT suser_sname()
4 | FOR [ModifiedBy]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_L1TransformDefinition_ModifiedTimestamp.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformDefinition]
2 | ADD CONSTRAINT [DC_L1TransformDefinition_ModifiedTimestamp]
3 | DEFAULT CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
4 | FOR [ModifiedTimestamp]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_L2TransformDefinition.RunSequence.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [DC_L2TransformDefinition_RunSequence]
3 | DEFAULT 100
4 | FOR [RunSequence]
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_L2TransformDefinition_CreatedBy.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [DC_L2TransformDefinition_CreatedBy]
3 | DEFAULT suser_sname()
4 | FOR [CreatedBy]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_L2TransformDefinition_CreatedTimestamp.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [DC_L2TransformDefinition_CreatedTimestamp]
3 | DEFAULT CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
4 | FOR [CreatedTimestamp]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_L2TransformDefinition_MaxRetries.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [DC_L2TransformDefinition_MaxRetries]
3 | DEFAULT 3
4 | FOR [MaxRetries]
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_L2TransformDefinition_ModifiedBy.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [DC_L2TransformDefinition_ModifiedBy]
3 | DEFAULT suser_sname()
4 | FOR [ModifiedBy]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/DC_L2TransformDefinition_ModifiedTimestamp.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [DC_L2TransformDefinition_ModifiedTimestamp]
3 | DEFAULT CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
4 | FOR [ModifiedTimestamp]
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Constraints/PK_L1TransformInstance.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformInstance]
2 | ADD CONSTRAINT [PK_L1TransformInstance]
3 | PRIMARY KEY ([L1TransformInstanceID])
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Database.sqlproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | reactions
6 | Microsoft.Data.Tools.Schema.Sql.SqlAzureV12DatabaseSchemaProvider
7 | 1033, CI
8 | {00000000-0000-0000-0000-000000000000}
9 |
10 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Functions/uf_GetAestDateTime.sql:
--------------------------------------------------------------------------------
1 | CREATE FUNCTION ELT.[uf_GetAestDateTime]()
2 | RETURNS DATETIME
3 | WITH EXECUTE AS CALLER
4 | AS
5 | BEGIN
6 | RETURN CONVERT(datetime,CONVERT(datetimeoffset, getdate()) AT TIME ZONE 'AUS Eastern Standard Time')
7 | END
8 | GO
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Functions/uf_GetColumnMapping.sql:
--------------------------------------------------------------------------------
1 | CREATE FUNCTION [ELT].[uf_GetColumnMapping]
2 | (
3 | @IngestID int NULL,
4 | @L1TransformID int NULL,
5 | @L2TransformID int NULL
6 | )
7 | RETURNS varchar(max)
8 | AS BEGIN
9 | DECLARE @mapping NVARCHAR(MAX)
10 | BEGIN
11 | WITH
12 | cte
13 | AS
14 | (
15 | select
16 | Mapping = CASE
17 | WHEN SourceName IS NULL THEN NULL
18 | ELSE '{"source":{"name":"' + SourceName + '"},"sink":{"name":"' + TargetName + '"}},'
19 | END
20 | from
21 | [ELT].[ColumnMapping]
22 | WHERE
23 | IngestID = @IngestID
24 | or L1TransformID = @L1TransformID
25 | or L2TransformID = @L2TransformID
26 | and ActiveFlag = 1
27 | )
28 |
29 | SELECT
30 | @mapping = CASE
31 | WHEN STRING_AGG (Mapping, '') IS NULL THEN NULL
32 | ELSE COALESCE(concat('{"type":"TabularTranslator","mappings":[', SUBSTRING(STRING_AGG (Mapping, ''),1,LEN(STRING_AGG (Mapping, ''))-1), ']}'), NULL)
33 | END
34 | FROM cte;
35 | END
36 |
37 | -- Return the result of the function
38 | Return @mapping
39 |
40 | END
41 | GO
42 |
43 |
44 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Functions/uf_GetIngestionColumnList.sql:
--------------------------------------------------------------------------------
1 | CREATE FUNCTION [ELT].[uf_GetIngestionColumnList]
2 | (
3 | @IngestID INT
4 | )
5 |
6 | RETURNS varchar(max)
7 | AS BEGIN
8 | DECLARE @Columns NVARCHAR(MAX)
9 |
10 | --Columns
11 | SET @Columns = (
12 | SELECT
13 | DISTINCT
14 | stuff((
15 | SELECT ', ' + SourceName + ' as ' + TargetName
16 | FROM
17 | [ELT].[ColumnMapping]
18 | WHERE IngestID = @IngestID
19 | and ActiveFlag = 1
20 | ORDER BY TargetOrdinalPosition ASC
21 | FOR XML PATH('')
22 | ),1,1,'') as Columns
23 | FROM [ELT].[ColumnMapping]
24 | WHERE IngestID = @IngestID
25 | and ActiveFlag = 1
26 | GROUP BY SourceName
27 | )
28 | SET @Columns = COALESCE(@Columns, ' * ')
29 |
30 | -- Return the columns of the function
31 | RETURN @Columns
32 | END
33 | GO
34 |
35 |
36 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Functions/uf_GetSalesforceQuery.sql:
--------------------------------------------------------------------------------
1 | CREATE FUNCTION [ELT].[uf_GetSalesforceQuery]
2 | (
3 | @QueryType varchar(20), --SourceQuery|StatQuery
4 | @IngestID INT,
5 | @EntityName varchar(100),
6 | @WatermarkColName varchar(20),
7 | @FromDate datetime2=NULL,
8 | @ToDate datetime2=NULL,
9 | @MaxIntervalMinutes int=NULL
10 | )
11 | RETURNS varchar(MAX)
12 | AS
13 | BEGIN
14 | Declare @Query varchar(MAX)
15 | DECLARE @MappingExists INT
16 | DECLARE @Columns NVARCHAR(MAX)
17 | Declare @From datetime2
18 | DECLARE @FromStr varchar(27), @ToStr varchar(27)
19 |
20 | SET @From = (CASE
21 | WHEN @FromDate is NULL THEN '1900-01-01 00:00:00'
22 | ELSE @FromDate
23 | END)
24 |
25 | --Datetime Strings
26 | SET @ToStr = FORMAT(CAST((
27 | CASE
28 | WHEN @ToDate IS NOT NULL
29 | THEN @ToDate
30 | WHEN @ToDate is NULL AND @MaxIntervalMinutes IS NULL
31 | THEN GETDATE()
32 | WHEN @ToDate is NULL AND @MaxIntervalMinutes IS NOT NULL AND DATEADD(DAY,@MaxIntervalMinutes/1440,@From) > GETDATE()
33 | THEN GETDATE()
34 | WHEN @ToDate is NULL AND @MaxIntervalMinutes IS NOT NULL AND DATEADD(DAY,@MaxIntervalMinutes/1440,@From) <= GETDATE()
35 | THEN DATEADD(DAY,@MaxIntervalMinutes/1440,@From)
36 | ELSE GETDATE()
37 | END
38 | )
39 | as datetime), 'yyyy-MM-ddTHH:mm:ssZ')
40 |
41 | SET @FromStr = (CASE
42 | WHEN @FromDate is NULL THEN '1900-01-01T00:00:00Z'
43 | ELSE FORMAT(CAST(@FromDate as datetime), 'yyyy-MM-ddTHH:mm:ssZ')
44 | END)
45 |
46 | --Set Columns
47 | SET @Columns = (
48 | SELECT
49 | DISTINCT
50 | STUFF((
51 | SELECT
52 | ', ' + SourceName
53 | FROM
54 | [ELT].[ColumnMapping]
55 | WHERE IngestID = @IngestID
56 | AND ActiveFlag = 1
57 | ORDER BY TargetOrdinalPosition ASC
58 | FOR XML PATH('')
59 | ),1,1,'') AS ColumnList
60 | FROM
61 | [ELT].[ColumnMapping]
62 | WHERE IngestID = @IngestID
63 | and ActiveFlag = 1
64 | GROUP BY SourceName
65 | )
66 |
67 | --SourceQuery
68 | IF @QueryType ='SourceQuery'
69 | BEGIN
70 | SET @Query =
71 | 'SELECT ' + COALESCE(@Columns, ' * ')
72 | + ' FROM ' + @EntityName
73 | + CASE
74 | WHEN @WatermarkColName IS NOT NULL
75 | THEN ' WHERE ' + @WatermarkColName + ' > ' + @FromStr + ' AND ' + @WatermarkColName + ' <= ' + @ToStr
76 | ELSE ''
77 | END
78 | END
79 |
80 | --StatQuery
81 | IF @QueryType ='StatQuery'
82 | BEGIN
83 | SET @Query = CASE
84 | WHEN @WatermarkColName IS NOT NULL
85 | THEN 'SELECT MIN('+@WatermarkColName+') AS DataFromTimestamp,' + ' MAX('+@WatermarkColName+') AS DataToTimestamp,'+ 'COUNT(*) AS SourceCount'
86 | WHEN @WatermarkColName IS NULL
87 | THEN 'SELECT ''1900-01-01T00:00:00Z'' AS DataFromTimestamp, '''+FORMAT(CAST(GETDATE() as datetime), 'yyyy-MM-ddTHH:mm:ssZ')+''' AS DataToTimestamp, COUNT(*) AS SourceCount '
88 | END
89 | + ' FROM ' + @EntityName
90 | + CASE
91 | WHEN @WatermarkColName IS NOT NULL
92 | THEN ' WHERE ' + @WatermarkColName + ' > ' + '''' + FORMAT(CAST(@FromStr as datetime), 'yyyy-MM-dd HH:mm:ss') + '''' + ' AND ' + @WatermarkColName + ' <= ' + '''' + FORMAT(CAST(@ToStr as datetime), 'yyyy-MM-dd HH:mm:ss') + ''''
93 | ELSE ''
94 | END
95 | END
96 |
97 | -- Return the result of the function
98 | Return @Query
99 |
100 | END
101 |
102 |
103 | GO
104 |
105 |
106 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Functions/uf_GetTabularTranslatorMappingJson.sql:
--------------------------------------------------------------------------------
1 | CREATE FUNCTION [ELT].[uf_GetTabularTranslatorMappingJson]
2 | (
3 | @DataMappingJson VARCHAR(MAX)
4 | )
5 | RETURNS VARCHAR(MAX)
6 | AS
7 | BEGIN
8 | -- Uses standard ADF Explicit Schema Mapping
9 | -- https://docs.microsoft.com/en-us/azure/data-factory/copy-activity-schema-and-type-mapping
10 | DECLARE @TabularTranslatorJson VARCHAR(MAX) = '{
11 | "type": "TabularTranslator",
12 | "mappings":
13 | }'
14 | RETURN REPLACE(@TabularTranslatorJson, '', @DataMappingJson)
15 | END
16 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Indexes/UI_IngestDefinition.sql:
--------------------------------------------------------------------------------
1 | CREATE UNIQUE INDEX [UI_IngestDefinition]
2 | ON [ELT].[IngestDefinition]
3 | ([SourceSystemName],[StreamName])
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Indexes/UI_IngestInstance.sql:
--------------------------------------------------------------------------------
1 | CREATE INDEX [UI_IngestInstance]
2 | ON [ELT].[IngestInstance]
3 | ([DestinationRawFileSystem],[DestinationRawFolder],[DestinationRawFile])
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Indexes/UI_L1TransformDefinition.sql:
--------------------------------------------------------------------------------
1 | CREATE INDEX [UI_L1TransformDefinition]
2 | ON [ELT].[L1TransformDefinition]
3 | ([InputRawFileSystem],[InputRawFileFolder],[InputRawFile],[OutputL1CurateFileSystem],[OutputL1CuratedFolder],[OutputL1CuratedFile])
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Indexes/UI_L1TransformInstance.sql:
--------------------------------------------------------------------------------
1 | CREATE UNIQUE INDEX [UI_L1TransformInstance]
2 | ON [ELT].[L1TransformInstance]
3 | ([InputRawFileSystem],[InputRawFileFolder],[InputRawFile],[OutputL1CurateFileSystem],[OutputL1CuratedFolder],[OutputL1CuratedFile])
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Indexes/UI_L2TransformDefinition.sql:
--------------------------------------------------------------------------------
1 | CREATE UNIQUE INDEX [UI_L2TransformDefinition]
2 | ON [ELT].[L2TransformDefinition]
3 | ([InputFileSystem],[InputFileFolder],[InputFile],[InputDWTable],[OutputL2CurateFileSystem],[OutputL2CuratedFolder],[OutputL2CuratedFile])
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Indexes/UI_L2TransformInstance.sql:
--------------------------------------------------------------------------------
1 | CREATE INDEX [UI_L2TransformInstance]
2 | ON [ELT].[L2TransformInstance]
3 | ([InputFileSystem],[InputFileFolder],[InputFile],[InputDWTable],[OutputL2CurateFileSystem],[OutputL2CuratedFolder],[OutputL2CuratedFile])
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/FK_Attribute_IngestID.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[ColumnMapping]
2 | ADD CONSTRAINT [FK_Attribute_IngestID]
3 | FOREIGN KEY([IngestID])
4 | REFERENCES [ELT].[IngestDefinition] ([IngestID])
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/FK_Attribute_L1TransformID.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[ColumnMapping]
2 | ADD CONSTRAINT [FK_Attribute_L1TransformID]
3 | FOREIGN KEY([L1TransformID])
4 | REFERENCES [ELT].[L1TransformDefinition] ([L1TransformID])
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/FK_Attribute_L2TransformID.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[ColumnMapping]
2 | ADD CONSTRAINT [FK_Attribute_L2TransformID]
3 | FOREIGN KEY([L2TransformID])
4 | REFERENCES [ELT].[L2TransformDefinition] ([L2TransformID])
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/FK_IngestInstance_IngestID.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestInstance]
2 | ADD CONSTRAINT [FK_IngestInstance_IngestID]
3 | FOREIGN KEY (IngestID)
4 | REFERENCES [ELT].[IngestDefinition] (IngestID)
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/FK_L1TransformDefinition_IngestID.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformDefinition]
2 | ADD CONSTRAINT [FK_L1TransformDefinition_IngestID]
3 | FOREIGN KEY ([IngestID])
4 | REFERENCES [ELT].[IngestDefinition] ([IngestID])
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/FK_L1TransformInstance_IngestID.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformInstance]
2 | ADD CONSTRAINT [FK_L1TransformInstance_IngestID]
3 | FOREIGN KEY ([IngestID])
4 | REFERENCES [ELT].[IngestDefinition] ([IngestID])
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/FK_L1TransformInstance_L1TransformID.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformInstance]
2 | ADD CONSTRAINT [FK_L1TransformInstance_L1TransformID]
3 | FOREIGN KEY (L1TransformID)
4 | REFERENCES [ELT].[L1TransformDefinition] (L1TransformID)
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/FK_L2TransformDefinition_IngestID.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [FK_L2TransformDefinition_IngestID]
3 | FOREIGN KEY (IngestID)
4 | REFERENCES [ELT].[IngestDefinition] (IngestID)
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/FK_L2TransformInstance_IngestID.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformInstance]
2 | ADD CONSTRAINT [FK_L2TransformInstance_IngestID]
3 | FOREIGN KEY (IngestID)
4 | REFERENCES [ELT].[IngestDefinition] (IngestID)
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/FK_L2TransformInstance_L1TransformID.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformInstance]
2 | ADD CONSTRAINT [FK_L2TransformInstance_L1TransformID]
3 | FOREIGN KEY (L1TransformID)
4 | REFERENCES [ELT].[L1TransformDefinition] (L1TransformID)
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/FK_L2TransformInstance_L2TransformID.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformInstance]
2 | ADD CONSTRAINT [FK_L2TransformInstance_L2TransformID]
3 | FOREIGN KEY (L2TransformID)
4 | REFERENCES [ELT].[L2TransformDefinition] (L2TransformID)
5 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/PK_Attribute.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[ColumnMapping]
2 | ADD CONSTRAINT [PK_Attribute]
3 | PRIMARY KEY CLUSTERED ([MappingID])
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/PK_IngestDefinition.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestDefinition]
2 | ADD CONSTRAINT [PK_IngestDefinition]
3 | PRIMARY KEY ([IngestID])
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/PK_IngestInstance.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[IngestInstance]
2 | ADD CONSTRAINT [PK_IngestInstance]
3 | PRIMARY KEY ([IngestInstanceID])
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/PK_L1TransformDefinition.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L1TransformDefinition]
2 | ADD CONSTRAINT [PK_L1TransformDefinition]
3 | PRIMARY KEY ([L1TransformID])
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/PK_L2TransformDefinition.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformDefinition]
2 | ADD CONSTRAINT [PK_L2TransformDefinition]
3 | PRIMARY KEY (L2TransformID)
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Keys/PK_L2TransformInstance.sql:
--------------------------------------------------------------------------------
1 | ALTER TABLE [ELT].[L2TransformInstance]
2 | ADD CONSTRAINT [PK_L2TransformInstance]
3 | PRIMARY KEY ([L2TransformInstanceID])
4 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/GetIngestDefinition.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[GetIngestDefinition]
2 | @SourceSystemName varchar(20),
3 | @StreamName VARCHAR(100) = '%', --Default =All Streams
4 | @MaxIngestInstance INT = 10
5 |
6 | AS
7 | BEGIN
8 |
9 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time');
10 |
11 | with
12 | cte
13 | as
14 |
15 | ( --Normal Run
16 | SELECT
17 | [IngestID]
18 | ,[SourceSystemName]
19 | ,[StreamName]
20 | ,[Backend]
21 | ,[EntityName]
22 | ,[WatermarkColName]
23 |
24 | --Delta Dates
25 | ,[LastDeltaDate]
26 | ,[DataFromTimestamp] =
27 | CASE
28 | WHEN ([EntityName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL) THEN [LastDeltaDate]
29 | ELSE CAST('1900-01-01' AS DateTime)
30 | END
31 | ,[DataToTimestamp] =
32 | CASE
33 | WHEN ([EntityName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL AND [MaxIntervalMinutes] IS NOT NULL AND datediff_big(minute,[LastDeltaDate],@localdate) > [MaxIntervalMinutes]) THEN DateAdd(minute,[MaxIntervalMinutes],[LastDeltaDate])
34 | WHEN ([EntityName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL AND [MaxIntervalMinutes] IS NOT NULL AND datediff_big(minute,[LastDeltaDate],@localdate) <= [MaxIntervalMinutes]) THEN CONVERT(VARCHAR(30),@localdate,120)
35 | ELSE CONVERT(VARCHAR(30),@localdate,120)
36 | END
37 |
38 | --Delta Numbers
39 | ,[LastDeltaNumber]
40 | ,[DataFromNumber] =
41 | CASE
42 | WHEN ([EntityName] IS NOT NULL AND [LastDeltaNumber] IS NOT NULL) THEN [LastDeltaNumber]
43 | END
44 | ,[DataToNumber] =
45 | CASE
46 | WHEN ([EntityName] IS NOT NULL AND [LastDeltaNumber] IS NOT NULL) THEN ([LastDeltaNumber] + [MaxIntervalNumber])
47 | END
48 |
49 | ,[DataFormat]
50 | ,[SourceStructure]
51 | ,[MaxIntervalMinutes]
52 | ,[MaxIntervalNumber]
53 | ,[DataMapping]
54 | ,[RunSequence]
55 | ,[ActiveFlag]
56 | ,[L1TransformationReqdFlag]
57 | ,[L2TransformationReqdFlag]
58 | ,[DelayL1TransformationFlag]
59 | ,[DelayL2TransformationFlag]
60 | ,[DestinationRawFileSystem]
61 |
62 | --Derived Fields
63 | ,[DestinationRawFolder] =
64 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE([DestinationRawFolder] COLLATE SQL_Latin1_General_CP1_CS_AS
65 | ,'YYYY',CAST(Year(COALESCE([LastDeltaDate],@localdate)) as varchar(4)))
66 | ,'MM',Right('0'+ CAST(Month(COALESCE([LastDeltaDate],@localdate)) AS varchar(2)),2))
67 | ,'DD',Right('0'+Cast(Day(COALESCE([LastDeltaDate],@localdate)) as varchar(2)),2))
68 | ,'HH',Right('0'+ CAST(DatePart(hh,COALESCE([LastDeltaDate],@localdate)) as varchar(2)),2))
69 | ,'MI',Right('0'+ CAST(DatePart(mi,COALESCE([LastDeltaDate],@localdate)) as varchar(2)),2))
70 | ,'SS',Right('0'+ CAST(DatePart(ss,COALESCE([LastDeltaDate],@localdate)) as varchar(2)),2))
71 |
72 | ,[DestinationRawFile] =
73 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE([DestinationRawFile] COLLATE SQL_Latin1_General_CP1_CS_AS
74 | ,'YYYY',CAST(Year(COALESCE([LastDeltaDate],@localdate)) AS varchar(4)))
75 | ,'MM',Right('0'+ CAST(Month(COALESCE([LastDeltaDate],@localdate)) AS varchar(2)),2))
76 | ,'DD',Right('0'+Cast(Day(COALESCE([LastDeltaDate],@localdate)) as varchar(2)),2))
77 | ,'HH',Right('0'+ CAST(DatePart(hh,COALESCE([LastDeltaDate],@localdate)) AS varchar(2)),2))
78 | ,'MI',Right('0'+ CAST(DatePart(mi,COALESCE([LastDeltaDate],@localdate)) AS varchar(2)),2))
79 | ,'SS',Right('0'+ CAST(DatePart(ss,COALESCE([LastDeltaDate],@localdate)) AS varchar(2)),2))
80 |
81 |
82 | --Query
83 | ,SourceSQL =
84 | CASE
85 | -- Customized for simple Purview ATLAS API
86 | WHEN Backend IN ('ATLAS REST API','AZURE REST API') THEN EntityName
87 |
88 | --DEFAULT ANSI SQL for Delta Table
89 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL
90 | THEN
91 | CASE
92 | WHEN datediff_big(minute,[LastDeltaDate],@localdate) > [MaxIntervalMinutes]
93 | THEN
94 | 'SELECT * FROM ' + [EntityName] + ' WHERE '
95 | + [WatermarkColName] + ' > ' + ''''+CONVERT(VARCHAR(30),[LastDeltaDate],121) +''''+ ' AND ' + [WatermarkColName] + '<=' + ''''+ CONVERT(VARCHAR(30), DATEADD(minute,[MaxIntervalMinutes],[LastDeltaDate]),121) +''''
96 | ELSE
97 | 'SELECT * FROM ' + [EntityName] + ' WHERE '
98 | + [WatermarkColName] + ' > ' + ''''+ CONVERT(VARCHAR(30),[LastDeltaDate],121) +''''+ ' AND ' + [WatermarkColName] + '<=' + ''''+ CONVERT(VARCHAR(30), @localdate,120) +''''
99 | END
100 | --DEFAULT ANSI SQL for Full Table
101 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NULL
102 | THEN
103 | 'SELECT * FROM ' + [EntityName]
104 | --Running Number
105 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaNumber] IS NOT NULL
106 | THEN 'SELECT * FROM ' + [EntityName] + ' WHERE '
107 | + [WatermarkColName] + ' > ' + ''''+CONVERT(VARCHAR,[LastDeltaNumber]) +'''' + [WatermarkColName] + ' <= ' + ''''+CONVERT(VARCHAR,([LastDeltaNumber] + [MaxIntervalNumber])) +''''
108 | ELSE NULL
109 | END
110 |
111 | --Stats Query
112 | ,StatSQL =
113 |
114 | CASE
115 | -- Customized for simple Purview ATLAS API
116 | WHEN Backend IN ('ATLAS REST API','AZURE REST API') THEN EntityName
117 |
118 | --DEFAULT ANSI SQL For Delta Table
119 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL
120 | THEN
121 | CASE
122 | WHEN datediff_big(minute,[LastDeltaDate],@localdate) > [MaxIntervalMinutes]
123 | THEN
124 | 'SELECT MIN('+[WatermarkColName]+') AS DataFromTimestamp, MAX('+[WatermarkColName]+') AS DataToTimestamp, count(1) as SourceCount FROM ' + [EntityName] + ' WHERE '
125 | + [WatermarkColName] + ' > ' + ''''+CONVERT(varchar(30),LastDeltaDate,121)+''''+ ' AND ' + [WatermarkColName] + ' <= ' + ''''+CONVERT(varchar(30), DATEADD(minute,[MaxIntervalMinutes],[LastDeltaDate]),121)+''''
126 | ELSE
127 | 'SELECT MIN('+[WatermarkColName]+') AS DataFromTimestamp, MAX('+[WatermarkColName]+') AS DataToTimestamp, count(1) as SourceCount FROM ' + [EntityName] + ' WHERE '
128 | + [WatermarkColName] + ' > ' + ''''+CONVERT(varchar(30),[LastDeltaDate],121) +''''+ ' AND ' + [WatermarkColName] + ' <= ' + ''''+ CONVERT(varchar(30),(@localdate),120)+''''
129 | END
130 | --Common No Delta
131 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NULL
132 | THEN 'SELECT ''1900-01-01 00:00:00'' AS DataFromTimestamp, '''+CONVERT(VARCHAR(30),ELT.uf_GetAestDateTime(),120)+''' AS DataToTimestamp, COUNT(*) AS SourceCount FROM ' + [EntityName]
133 | --Running Number
134 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaNumber] IS NOT NULL
135 | THEN 'SELECT MIN('+[WatermarkColName]+') AS DataFromTimestamp,' + ' MAX('+[WatermarkColName]+') AS DataToTimestamp,'+ 'COUNT(*) AS SourceCount FROM ' + [EntityName]
136 | + [WatermarkColName] + ' > ' + ''''+CONVERT(VARCHAR,[LastDeltaNumber])+'''' + ' AND ' + [WatermarkColName] + ' <= ' + ''''+CONVERT(VARCHAR,([LastDeltaNumber] + [MaxIntervalNumber]))+''''
137 | ELSE NULL
138 | END
139 |
140 | , CAST(0 AS BIT) AS [ReloadFlag]
141 | , NULL AS [ADFPipelineRunID]
142 | FROM
143 | [ELT].[IngestDefinition]
144 | WHERE
145 | [SourceSystemName]=@SourceSystemName
146 | AND [StreamName] LIKE COALESCE(@StreamName, [StreamName])
147 | AND [ActiveFlag]=1
148 |
149 | --ReRun
150 | UNION
151 | SELECT
152 | [ID].[IngestID]
153 | ,[SourceSystemName]
154 | ,[StreamName]
155 | ,[Backend]
156 | ,[EntityName]
157 | ,[WatermarkColName]
158 | ,[LastDeltaDate]
159 | ,II.[DataFromTimestamp]
160 | ,II.[DataToTimestamp]
161 | ,ID.[LastDeltaNumber]
162 | ,II.[DataFromNumber]
163 | ,II.[DataToNumber]
164 | ,[DataFormat]
165 | ,[SourceStructure]
166 | ,ID.[MaxIntervalMinutes]
167 | ,ID.[MaxIntervalNumber]
168 | ,ID.[DataMapping]
169 | ,ID.[RunSequence]
170 | ,[ActiveFlag]
171 | ,[L1TransformationReqdFlag]
172 | ,[L2TransformationReqdFlag]
173 | ,[DelayL1TransformationFlag]
174 | ,[DelayL2TransformationFlag]
175 | ,II.[DestinationRawFileSystem]
176 | ,II.[DestinationRawFolder]
177 | ,II.[DestinationRawFile]
178 |
179 | --Derived Fields
180 | ,SourceSQL =
181 | CASE
182 | -- Customized for simple Purview ATLAS API
183 | WHEN Backend IN ('ATLAS REST API','AZURE REST API') THEN EntityName
184 | --DEFAULT ANSI SQL for Delta Table
185 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL
186 | THEN 'SELECT * FROM ' + [EntityName] + ' WHERE '
187 | + [WatermarkColName] + ' > ' + ''''+ CONVERT(varchar(30),II.[DataFromTimestamp],121)+''''+ ' AND ' + [WatermarkColName] + ' <= ' + ''''+ CONVERT(varchar(30),II.[DataToTimestamp],121)+''''
188 | --Common No Delta
189 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NULL
190 | THEN 'SELECT * FROM ' + [EntityName]
191 | --Running Number
192 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaNumber] IS NOT NULL
193 | THEN 'SELECT * FROM ' + [EntityName] + ' WHERE '
194 | + [WatermarkColName] + ' > ' + ''''+CONVERT(VARCHAR,II.[DataFromNumber])+'''' + ' AND ' + [WatermarkColName] + ' <= ' + ''''+CONVERT(VARCHAR,II.[DataToNumber])+''''
195 |
196 | ELSE NULL
197 | END
198 |
199 | ,StatSQL =
200 | CASE
201 | -- Customized for simple Purview ATLAS API
202 | WHEN Backend IN ('ATLAS REST API','AZURE REST API') THEN EntityName
203 |
204 | --DEFAULT ANSI SQL for Delta Table
205 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaDate] IS NOT NULL THEN
206 | 'SELECT MIN('+[WatermarkColName]+') AS DataFromTimestamp, MAX('+[WatermarkColName]+') AS DataToTimestamp, count(1) as SourceCount FROM '
207 | + [EntityName] + ' WHERE ' + [WatermarkColName] + '>' + ''''+CONVERT(varchar(30),II.DataFromTimestamp,121)+''''+ ' AND ' + [WatermarkColName] + '<='+ ''''+CONVERT(varchar(30),II.[DataToTimestamp],121)+''''
208 | --Common No Delta
209 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NULL AND [LastDeltaDate] IS NOT NULL
210 | THEN 'SELECT MIN('+[WatermarkColName]+') AS DataFromTimestamp,' + ' MAX('+[WatermarkColName]+') AS DataToTimestamp,'+ 'COUNT(*) AS SourceCount FROM ' + [EntityName]
211 | --Common No Delta
212 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NULL
213 | THEN 'SELECT SELECT ''1900-01-01 00:00:00'' AS DataFromTimestamp, '''+CONVERT(VARCHAR(30),ELT.uf_GetAestDateTime(),120)+''' AS DataToTimestamp, COUNT(*) AS SourceCount FROM ' + [EntityName]
214 | --Running Number
215 | WHEN [EntityName] IS NOT NULL AND [WatermarkColName] IS NOT NULL AND [LastDeltaNumber] IS NOT NULL
216 | THEN 'SELECT MIN('+[WatermarkColName]+') AS DataFromTimestamp,' + ' MAX('+[WatermarkColName]+') AS DataToTimestamp,'+ 'COUNT(*) AS SourceCount FROM ' + [EntityName]
217 | + [WatermarkColName] + ' > ' + ''''+CONVERT(VARCHAR,II.[DataFromNumber]) +'''' + ' AND ' + [WatermarkColName] + ' <= ' + ''''+CONVERT(VARCHAR,II.[DataToNumber])+''''
218 | ELSE NULL
219 | END
220 |
221 | ,II.[ReloadFlag]
222 | , II.[ADFIngestPipelineRunID]
223 | FROM
224 | [ELT].[IngestDefinition] ID
225 | INNER JOIN [ELT].[IngestInstance] AS II
226 | ON II.[IngestID]= ID.[IngestID]
227 | AND II.[ReloadFlag]=1
228 | AND (II.[IngestStatus] is NULL OR II.[IngestStatus] != 'Running') --Fetch new instances and ignore instances that are currently running
229 | WHERE
230 | ID.[SourceSystemName]=@SourceSystemName
231 | AND ID.[StreamName] LIKE COALESCE(@StreamName, [StreamName])
232 | AND ID.[ActiveFlag]=1
233 | AND ISNULL(II.RetryCount,0) <= ID.MaxRetries
234 |
235 | )
236 | SELECT
237 | TOP (@MaxIngestInstance) *
238 | FROM CTE
239 | ORDER BY
240 | [RunSequence] ASC, [DataFromTimestamp] DESC, [DataToTimestamp] DESC
241 |
242 | END
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/GetIngestDefinition_FileDrop.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[GetIngestDefinition_FileDrop]
2 | @SourceSystemName varchar(20),
3 | @StreamName VARCHAR(100),
4 | @SourceFolder VARCHAR(250), --where SourceFolder=Container/Folder
5 | @SourceFile VARCHAR(200)
6 | AS
7 | --/*
8 | --Generic Stored Procedure to be used for File Drop Pipelines instead of the [ELT].[GetIngestDefinition] procedure which is relevant for database ingest pipelines
9 | --*/
10 | BEGIN
11 | --For Testing
12 |
13 |
14 | DECLARE @IngestID INT
15 | SELECT @IngestID = [IngestID] FROM [ELT].[IngestDefinition] WHERE [SourceSystemName]=@SourceSystemName AND [StreamName]=@StreamName and [ActiveFlag]=1
16 |
17 | --If the Source File already exists (Reload)
18 | IF EXISTS ( SELECT 1 FROM [ELT].[IngestInstance]
19 | WHERE (
20 | CASE
21 | WHEN [SourceFileDropFileSystem] IS NULL
22 | THEN [SourceFileDropFolder] --In most cases container will be null as it is part of folder name from event trigger
23 | WHEN [SourceFileDropFileSystem] IS NOT NULL AND LEN(RTRIM([SourceFileDropFolder])) =0
24 | THEN [SourceFileDropFolder]
25 | ELSE [SourceFileDropFileSystem] +'/' + [SourceFileDropFolder]
26 | END ) =@SourceFolder
27 | AND [SourceFileDropFile]=@SourceFile)
28 |
29 | BEGIN
30 | SELECT
31 | top 1
32 | II.[IngestID]
33 | , ID.[SourceSystemName]
34 | , ID.[StreamName]
35 | ,ID.[Backend]
36 | ,ID.[DataFormat]
37 | , II.[SourceFileDropFileSystem]
38 | , REPLACE(II.[SourceFileDropFolder],ID.[SourceFileDropFileSystem] +'/','') AS [SourceFileDropFolder] --Remove container name from folder path
39 | , II.[SourceFileDropFile]
40 | , ID.[SourceFileDelimiter]
41 | , ID.[SourceFileHeaderFlag]
42 | , II.[DestinationRawFileSystem]
43 | , II.[DestinationRawFolder]
44 | , II.[DestinationRawFile]
45 | , ID.[DataMapping]
46 | , CAST(1 AS BIT) AS [ReloadFlag]
47 | , ID.[L1TransformationReqdFlag]
48 | , ID.[L2TransformationReqdFlag]
49 | , ID.[DelayL1TransformationFlag]
50 | , ID.[DelayL2TransformationFlag]
51 | , II.[ADFIngestPipelineRunID]
52 | FROM
53 | [ELT].[IngestInstance] AS II
54 | INNER JOIN [ELT].[IngestDefinition] AS ID
55 | ON II.[IngestID] =ID.[IngestID]
56 | AND II.[IngestID]=@IngestID
57 | AND (
58 | CASE
59 | WHEN II.[SourceFileDropFileSystem] IS NULL
60 | THEN II.[SourceFileDropFolder] --In most cases container will be null as it is part of folder name from event trigger
61 | WHEN II.[SourceFileDropFolder] IS NOT NULL AND LEN(RTRIM(II.[SourceFileDropFileSystem])) =0
62 | THEN II.[SourceFileDropFolder]
63 | ELSE II.[SourceFileDropFileSystem] +'/' + II.[SourceFileDropFolder]
64 | END ) = @SourceFolder
65 | AND II.[SourceFileDropFile]=@SourceFile
66 | ORDER BY [IngestInstanceID] DESC
67 | END
68 | ELSE
69 | --If this is a new file
70 | BEGIN
71 |
72 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
73 |
74 | SELECT
75 | [IngestID]
76 | , [SourceSystemName]
77 | , [StreamName]
78 | ,[Backend]
79 | ,[DataFormat]
80 | , [SourceFileDropFileSystem]
81 | , REPLACE(@SourceFolder,[SourceFileDropFileSystem] +'/','') AS [SourceFileDropFolder] --Remove container name from folder path
82 | , @SourceFile AS [SourceFileDropFile]
83 | , [SourceFileDelimiter]
84 | , [SourceFileHeaderFlag]
85 | , [DestinationRawFileSystem]
86 | , [DestinationRawFolder] =
87 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE([DestinationRawFolder] COLLATE SQL_Latin1_General_CP1_CS_AS
88 | ,'YYYY',CAST(Year(@localdate) as varchar(4)))
89 | ,'MM',Right('0'+ CAST(Month(@localdate) AS varchar(2)),2))
90 | ,'DD',Right('0'+Cast(Day(@localdate) as varchar(2)),2))
91 | ,'HH',Right('0'+ CAST(DatePart(hh,@localdate) as varchar(2)),2))
92 | ,'MI',Right('0'+ CAST(DatePart(mi,@localdate) as varchar(2)),2))
93 | ,'SS',Right('0'+ CAST(DatePart(ss,@localdate) as varchar(2)),2))
94 |
95 | , [DestinationRawFile] =
96 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE([DestinationRawFile] COLLATE SQL_Latin1_General_CP1_CS_AS
97 | ,'YYYY',CAST(Year(@localdate) AS varchar(4)))
98 | ,'MM',Right('0'+ CAST(Month(@localdate) AS varchar(2)),2))
99 | ,'DD',Right('0'+Cast(Day(@localdate) as varchar(2)),2))
100 | ,'HH',Right('0'+ CAST(DatePart(hh,@localdate) AS varchar(2)),2))
101 | ,'MI',Right('0'+ CAST(DatePart(mi,@localdate) AS varchar(2)),2))
102 | ,'SS',Right('0'+ CAST(DatePart(ss,@localdate) AS varchar(2)),2))
103 | , [DataMapping]
104 | , CAST(0 AS BIT) AS [ReloadFlag]
105 | , [L1TransformationReqdFlag]
106 | , [L2TransformationReqdFlag]
107 | , [DelayL1TransformationFlag]
108 | , [DelayL2TransformationFlag]
109 | , NULL AS [ADFPipelineRunID]
110 |
111 | FROM
112 | [ELT].[IngestDefinition]
113 | WHERE
114 | [IngestID]=@IngestID
115 | END
116 | END
117 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/GetRunningTransformInstances.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[GetRunningTransformInstances]
2 | @timespanHrs int = 18 --Default Timespan 18hrs
3 | AS
4 | --Get the count of transform instances (L1 and L2) still active and running in last @timespanHrs
5 | DECLARE @L1Count INT, @L2Count INT
6 |
7 | IF @timespanHrs IS NULL OR @timespanHrs<=0
8 | SET @timespanHrs=24
9 |
10 | SELECT @L1Count = COUNT(1)
11 | FROM ELT.L1TransformInstance
12 | WHERE L1TransformStatus='Running'
13 | AND DateDiff(hour,L1TransformStartTimestamp,getdate()) <= @timespanHrs
14 |
15 | SELECT @L2Count= COUNT(1)
16 | FROM ELT.L2TransformInstance
17 | WHERE L2TransformStatus='Running'
18 | AND DateDiff(hour,L2TransformStartTimestamp,getdate()) <= @timespanHrs
19 |
20 | SELECT (@L1Count + @L2Count) AS RunningCount
21 |
22 | RETURN
23 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/GetTransformDefinition_L1.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[GetTransformDefinition_L1]
2 | @IngestID int,
3 | @DeltaDate datetime = null
4 | AS
5 | --declare @IngestID int
6 | DECLARE @localdate datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) AT TIME ZONE 'AUS Eastern Standard Time')
7 | DECLARE @CuratedDate datetime
8 | SET @CuratedDate = COALESCE(@DeltaDate,@localdate)
9 |
10 |
11 | SELECT
12 | --PK/FK
13 | TD.[L1TransformID]
14 | , TD.[IngestID]
15 |
16 |
17 | --Databricks
18 | , TD.[ComputeName]
19 | , TD.[ComputePath]
20 |
21 | --Custom
22 | , TD.[CustomParameters]
23 |
24 | --Raw
25 | ,TD.[InputRawFileDelimiter]
26 | ,TD.[InputFileHeaderFlag]
27 |
28 | --Curated File
29 | --Need to test how this performs with a file drop and filedrop reload.
30 | --Possibly use date drop file was dropped?
31 | ,TD.[OutputL1CurateFileSystem]
32 |
33 | ,[OutputL1CuratedFolder] =
34 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(TD.[OutputL1CuratedFolder] COLLATE SQL_Latin1_General_CP1_CS_AS
35 | ,'YYYY',CAST(Year(@CuratedDate) as varchar(4)))
36 | ,'MM',Right('0'+ CAST(Month(@CuratedDate) AS varchar(2)),2))
37 | ,'DD',Right('0'+Cast(Day(@CuratedDate) as varchar(2)),2))
38 | ,'HH',Right('0'+ CAST(DatePart(hh,@CuratedDate) as varchar(2)),2))
39 | ,'MI',Right('0'+ CAST(DatePart(mi,@CuratedDate) as varchar(2)),2))
40 | ,'SS',Right('0'+ CAST(DatePart(ss,@CuratedDate) as varchar(2)),2))
41 |
42 | ,[OutputL1CuratedFile] =
43 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(TD.[OutputL1CuratedFile] COLLATE SQL_Latin1_General_CP1_CS_AS
44 | ,'YYYY',CAST(Year(@CuratedDate) as varchar(4)))
45 | ,'MM',Right('0'+ CAST(Month(@CuratedDate) AS varchar(2)),2))
46 | ,'DD',Right('0'+Cast(Day(@CuratedDate) as varchar(2)),2))
47 | ,'HH',Right('0'+ CAST(DatePart(hh,@CuratedDate) as varchar(2)),2))
48 | ,'MI',Right('0'+ CAST(DatePart(mi,@CuratedDate) as varchar(2)),2))
49 | ,'SS',Right('0'+ CAST(DatePart(ss,@CuratedDate) as varchar(2)),2))
50 |
51 | , TD.[OutputL1CuratedFileDelimiter]
52 | , TD.[OutputL1CuratedFileFormat]
53 | , TD.[OutputL1CuratedFileWriteMode]
54 |
55 | --SQL
56 | , [LookupColumns]
57 | , TD.[OutputDWStagingTable]
58 | , TD.[OutputDWTable]
59 | , TD.[OutputDWTableWriteMode]
60 |
61 | --Max Retries
62 | , TD.[MaxRetries]
63 |
64 |
65 | FROM
66 | [ELT].[L1TransformDefinition] TD
67 | LEFT JOIN [ELT].[IngestDefinition] ID
68 | ON TD.[IngestID] = ID.[IngestID]
69 | WHERE
70 | TD.[IngestID] = @IngestID and
71 | TD.[ActiveFlag] = 1
72 | and ID.[ActiveFlag] = 1
73 | and ID.[L1TransformationReqdFlag] =1
74 |
75 | GO
76 |
77 |
78 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/GetTransformDefinition_L2.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[GetTransformDefinition_L2]
2 | @IngestID int,
3 | @DeltaDate datetime = null,
4 | @InputType varchar(15) = '%'
5 | AS
6 | --declare @IngestID int
7 | DECLARE @localdate datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
8 |
9 | --Should be using L2DeltaTransformDate, if null then LocalDate
10 | SELECT
11 | --PK/FK
12 | TD.[L2TransformID]
13 | , TD.[IngestID]
14 | , TD.[L1TransformID]
15 |
16 | --Databricks
17 | , TD.[ComputeName]
18 | , TD.[ComputePath]
19 |
20 | --Custom
21 | , TD.[CustomParameters]
22 |
23 | --InputType
24 | ,TD.[InputType]
25 |
26 | --Raw
27 | ,TD.[InputFileSystem]
28 | ,TD.[InputFileFolder]
29 | ,TD.[InputFile]
30 | ,TD.[InputFileDelimiter]
31 | ,TD.[InputFileHeaderFlag]
32 | ,TD.[InputDWTable]
33 |
34 | --Deltas
35 | ,TD.[WatermarkColName]
36 | ,[DataFromTimestamp] = NULL
37 | ,[DataToTimestamp] = NULL
38 | ,[DataFromNumber] = NULL
39 | ,[DataToNumber] = NULL
40 | ,TD.[MaxIntervalMinutes]
41 | ,TD.[MaxIntervalNumber]
42 |
43 |
44 | --Retry
45 | ,TD.[MaxRetries]
46 |
47 | --Curated File
48 | --Need to test how this performs with a file drop and filedrop reload. Possibly use date drop file was dropped?
49 | ,TD.[OutputL2CurateFileSystem]
50 |
51 | ,[OutputL2CuratedFolder] =
52 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(TD.[OutputL2CuratedFolder] COLLATE SQL_Latin1_General_CP1_CS_AS
53 | ,'YYYY',CAST(Year(COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(4)))
54 | ,'MM',Right('0'+ CAST(Month(COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) AS varchar(2)),2))
55 | ,'DD',Right('0'+Cast(Day(COALESCE(TD.[LastDeltaDate],@localdate)) as varchar(2)),2))
56 | ,'HH',Right('0'+ CAST(DatePart(hh,COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2))
57 | ,'MI',Right('0'+ CAST(DatePart(mi,COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2))
58 | ,'SS',Right('0'+ CAST(DatePart(ss,COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2))
59 |
60 | ,[OutputL2CuratedFile] =
61 | REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(TD.[OutputL2CuratedFile] COLLATE SQL_Latin1_General_CP1_CS_AS
62 | ,'YYYY',CAST(Year(COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(4)))
63 | ,'MM',Right('0'+ CAST(Month(COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) AS varchar(2)),2))
64 | ,'DD',Right('0'+Cast(Day(COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2))
65 | ,'HH',Right('0'+ CAST(DatePart(hh,COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2))
66 | ,'MI',Right('0'+ CAST(DatePart(mi,COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2))
67 | ,'SS',Right('0'+ CAST(DatePart(ss,COALESCE(TD.[LastDeltaDate],@DeltaDate,@localdate)) as varchar(2)),2))
68 |
69 | , TD.[OutputL2CuratedFileDelimiter]
70 | , TD.[OutputL2CuratedFileFormat]
71 | , TD.[OutputL2CuratedFileWriteMode]
72 |
73 | --SQL
74 | ,TD.[OutputDWStagingTable]
75 | ,TD.[LookupColumns]
76 | ,TD.[OutputDWTable]
77 | ,TD.[OutputDWTableWriteMode]
78 |
79 |
80 | FROM
81 | [ELT].[L2TransformDefinition] TD
82 | LEFT JOIN [ELT].[IngestDefinition] ID
83 | ON TD.[IngestID] = ID.[IngestID]
84 | WHERE
85 | TD.[IngestID] = @IngestID and
86 | TD.[ActiveFlag] = 1
87 | and ID.[ActiveFlag] = 1
88 | and ID.[L2TransformationReqdFlag] =1
89 | and TD.[InputType] like COALESCE(@InputType,TD.[InputType])
90 |
91 | GO
92 |
93 |
94 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/GetTransformInstance_L1.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[GetTransformInstance_L1]
2 | @SourceSystemName varchar(20),
3 | @StreamName varchar(100) = '%',
4 | @MaxTransformInstance int = 10,
5 | @L1TransformInstanceId INT = NULL, --To fetch all transform instances set Parameter as NULL otherwise provide a specific instance id
6 | @DelayL1TransformationFlag INT=NULL --Pass @DelayL1TransformationFlag=0 to fetch all instances that needs to be transformed in the current pipeline (usually the ingestion pipeline). Pass @DelayL1TransformationFlag=1 to fetch all transformations that are scheduled for a later time.
7 | AS
8 | begin
9 | --Limit Number of Transform Instances to prevent queuing at DWH
10 | SELECT top (@MaxTransformInstance)
11 | L1TI.[L1TransformInstanceID]
12 | , L1TI.[L1TransformID]
13 | , L1TI.[IngestID]
14 | , L1TI.[ComputeName]
15 | , L1TI.[ComputePath]
16 | , L1TI.[CustomParameters]
17 | , L1TI.[InputRawFileSystem]
18 | , L1TI.[InputRawFileFolder]
19 | , L1TI.[InputRawFile]
20 | , L1TI.[InputRawFileDelimiter]
21 | , L1TI.[InputFileHeaderFlag]
22 | , L1TI.[OutputL1CurateFileSystem]
23 | , L1TI.[OutputL1CuratedFolder]
24 | , L1TI.[OutputL1CuratedFile]
25 | , L1TI.[OutputL1CuratedFileDelimiter]
26 | , L1TI.[OutputL1CuratedFileFormat]
27 | , L1TI.[OutputL1CuratedFileWriteMode]
28 | , L1TI.[OutputDWStagingTable]
29 | , L1TI.[LookupColumns]
30 | , L1TI.[OutputDWTable]
31 | , L1TI.[OutputDWTableWriteMode]
32 | , L1TI.[ReRunL1TransformFlag]
33 | , L1TD.[MaxRetries]
34 | , L1TD.[WatermarkColName]
35 |
36 | FROM
37 | [ELT].[L1TransformInstance] as L1TI
38 | LEFT JOIN [ELT].[L1TransformDefinition] as L1TD
39 | ON L1TI.[L1TransformID] = L1TD.[L1TransformID]
40 | LEFT JOIN [ELT].[IngestDefinition] as ID
41 | ON id.[IngestID]= L1TD.[IngestID]
42 | WHERE
43 | ID.[SourceSystemName] =@SourceSystemName
44 | AND ID.[StreamName] like COALESCE(@StreamName,id.[StreamName])
45 | AND (ID.ActiveFlag = 1 AND L1TD.ActiveFlag = 1)
46 | AND (L1TI.[ActiveFlag]=1 OR L1TI.[ReRunL1TransformFlag]=1)
47 | AND L1TI.[L1TransformInstanceID] = COALESCE(@L1TransformInstanceId, L1TI.[L1TransformInstanceID])
48 | AND (L1TI.[L1TransformStatus] IS NULL OR L1TI.[L1TransformStatus] NOT IN ('Running','DWUpload')) --Fetch new instances and ignore instances that are currently running
49 | AND ID.[DelayL1TransformationFlag] = COALESCE(@DelayL1TransformationFlag,ID.[DelayL1TransformationFlag])
50 | AND ISNULL(L1TI.RetryCount,0) <= L1TD.MaxRetries
51 | ORDER BY L1TI.[L1TransformInstanceID] ASC
52 | END
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/GetTransformInstance_L2.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[GetTransformInstance_L2]
2 | @SourceSystemName varchar(20),
3 | @StreamName varchar(100) = '%',
4 | @MaxTransformInstance int = 10,
5 | @L2TransformInstanceId INT = NULL, --To fetch all transform instances set Parameter as NULL otherwise provide a specific instance id
6 | @DelayL2TransformationFlag INT=NULL, --Pass @DelayL2TransformationFlag=0 to fetch all instances that needs to be transformed in the current pipeline (usually the ingestion pipeline). Pass =1 to fetch @DelayL2TransformationFlagh all transformations that are scheduled for a later time.
7 | @InputType varchar(15) = '%',
8 | @L2TransformID INT=0 --To Fetch all transformations for a specific stream pass 0 otherwise provide TransformId prersent in L2Transsformdefinition
9 | AS
10 | begin
11 | --Limit Number of Transform Instances to prevent queuing at DWH
12 | SELECT TOP
13 | (@MaxTransformInstance)
14 | L2TI.[L2TransformInstanceID]
15 | , L2TI.[L2TransformID]
16 | , L2TI.[IngestID]
17 | , L2TI.[L1TransformID]
18 | , L2TI.[ComputePath]
19 | , L2TI.[ComputeName]
20 | , L2TI.[CustomParameters]
21 | , L2TD.[InputType]
22 | , L2TI.[InputFileSystem]
23 | , L2TI.[InputFileFolder]
24 | , L2TI.[InputFile]
25 | , L2TI.[InputFileDelimiter]
26 | , L2TI.[InputFileHeaderFlag]
27 | , L2TI.[InputDWTable]
28 | , L2TI.[WatermarkColName]
29 | , L2TI.[DataFromTimestamp]
30 | , L2TI.[DataToTimestamp]
31 | , L2TI.[DataFromNumber]
32 | , L2TI.[DataToNumber]
33 | , L2TI.[OutputL2CurateFileSystem]
34 | , L2TI.[OutputL2CuratedFolder]
35 | , L2TI.[OutputL2CuratedFile]
36 | , L2TI.[OutputL2CuratedFileDelimiter]
37 | , L2TI.[OutputL2CuratedFileFormat]
38 | , L2TI.[OutputL2CuratedFileWriteMode]
39 | , L2TI.[OutputDWStagingTable]
40 | , L2TI.[LookupColumns]
41 | , L2TI.[OutputDWTable]
42 | , L2TI.[OutputDWTableWriteMode]
43 | , L2TI.[ReRunL2TransformFlag]
44 | , L2TD.[MaxRetries]
45 |
46 | FROM
47 | [ELT].[L2TransformInstance] as L2TI
48 | LEFT JOIN [ELT].[L2TransformDefinition] as L2TD
49 | ON L2TI.[L2TransformID] = L2TD.[L2TransformID]
50 | LEFT JOIN [ELT].[IngestDefinition] as ID
51 | ON id.[IngestID]= L2TD.[IngestID]
52 | WHERE
53 | ID.[SourceSystemName] =@SourceSystemName
54 | AND ID.[StreamName] like COALESCE(@StreamName,id.[StreamName])
55 | AND (ID.ActiveFlag = 1 AND L2TD.ActiveFlag = 1)
56 | AND (L2TI.[ActiveFlag]=1 OR L2TI.[ReRunL2TransformFlag]=1)
57 | AND L2TI.[L2TransformInstanceID] = COALESCE(@L2TransformInstanceId, L2TI.[L2TransformInstanceID])
58 | AND (L2TI.[L2TransformStatus] IS NULL OR L2TI.[L2TransformStatus] NOT IN ('Running','DWUpload')) --Fetch new instances and ignore instances that are currently running
59 | AND ID.[DelayL2TransformationFlag] = COALESCE(@DelayL2TransformationFlag,ID.[DelayL2TransformationFlag])
60 | AND L2TD.[InputType] like COALESCE(@InputType,L2TD.[InputType])
61 | --AND ISNULL(L2TI.RetryCount,0) <= L2TD.MaxRetries
62 | AND L2TI.[L2TransformID]= (CASE WHEN @L2TransformID=0 then L2TI.[L2TransformID] ELSE @L2TransformID END)
63 | ORDER BY L2TD.RunSequence ASC, L2TI.[L2TransformInstanceID] ASC
64 | END
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/InsertIngestInstance.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[InsertIngestInstance]
2 | @IngestID INT
3 | ,@SourceFileDropFileSystem varchar(50)=null
4 | ,@SourceFileDropFolder varchar(200)=null
5 | ,@SourceFileDropFile varchar(200)=null
6 | ,@DestinationRawFileSystem varchar(50)=null
7 | ,@DestinationRawFolder varchar(200)=null
8 | ,@DestinationRawFile varchar(200)=null
9 | ,@ReloadFlag bit =0
10 | ,@ADFPipelineRunID UniqueIdentifier=null
11 | AS
12 |
13 | BEGIN
14 |
15 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
16 |
17 |
18 | --NOTE:Potential enhancements for lookup.
19 | -- 1. Job to Purge the Instance Tables to Blob after 3 months
20 | -- 2. Use the AdfPipeline Instance Run ID for the lookup, though this needs to be the individual. Lookup Including Index on the lookup columns.
21 | -- 3. Create a Hash Column on DestinationFileSystem/Folder/File and use as the lookup. Including Index on the lookup columns.
22 |
23 | --Normal Run
24 | IF (@ReloadFlag=0 AND NOT EXISTS (
25 | SELECT 1
26 | FROM [ELT].[IngestInstance]
27 | WHERE [DestinationRawFileSystem] = @DestinationRawFileSystem
28 | AND [DestinationRawFolder] = @DestinationRawFolder
29 | AND [DestinationRawFile] = @DestinationRawFile
30 | )
31 | )
32 | BEGIN
33 | INSERT INTO [ELT].[IngestInstance]
34 | ([IngestID]
35 | ,[SourceFileDropFileSystem]
36 | ,[SourceFileDropFolder]
37 | ,[SourceFileDropFile]
38 | ,[DestinationRawFileSystem]
39 | ,[DestinationRawFolder]
40 | ,[DestinationRawFile]
41 | ,[IngestStartTimestamp]
42 | ,[IngestEndTimestamp]
43 | ,[IngestStatus]
44 | ,[RetryCount]
45 | ,[ReloadFlag]
46 | ,[CreatedBy]
47 | ,[CreatedTimestamp]
48 | ,[ModifiedBy]
49 | ,[ModifiedTimestamp]
50 | ,ADFIngestPipelineRunID)
51 | VALUES
52 | (@IngestID
53 | ,@SourceFileDropFileSystem
54 | ,@SourceFileDropFolder
55 | ,@SourceFileDropFile
56 | ,@DestinationRawFileSystem
57 | ,@DestinationRawFolder
58 | ,@DestinationRawFile
59 | ,@localdate
60 | ,NULL
61 | ,'Running'
62 | ,0
63 | ,0
64 | ,suser_sname()
65 | ,@localdate
66 | ,NULL
67 | ,NULL
68 | ,@ADFPipelineRunID)
69 | END
70 |
71 | --Re-load
72 | IF (@ReloadFlag=1
73 | OR EXISTS (SELECT 1 FROM [ELT].[IngestInstance]
74 | WHERE [DestinationRawFileSystem] = @DestinationRawFileSystem
75 | AND [DestinationRawFolder] = @DestinationRawFolder
76 | AND [DestinationRawFile] = @DestinationRawFile)
77 | )
78 | BEGIN
79 | Update [ELT].[IngestInstance]
80 | SET [IngestStartTimestamp] = @localdate
81 | ,[IngestEndTimestamp] = NULL
82 | ,[SourceCount]=NULL
83 | ,[IngestCount]=NULL
84 | ,[IngestStatus]='Running'
85 | ,[ModifiedBy]=suser_sname()
86 | ,[ModifiedTimestamp] = @localdate
87 | ,ADFIngestPipelineRunID = @ADFPipelineRunID
88 | --Unique Keys
89 | WHERE [DestinationRawFileSystem] = @DestinationRawFileSystem
90 | AND [DestinationRawFolder] = @DestinationRawFolder
91 | AND [DestinationRawFile] = @DestinationRawFile
92 | END
93 | END
94 |
95 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/InsertTransformInstance_L1.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[InsertTransformInstance_L1]
2 | --PK/FK
3 | @L1TransformID int = null,
4 | @IngestInstanceID int=null,
5 | @IngestID int,
6 |
7 | --Databricks Notebook
8 | @ComputeName varchar(100) = null,
9 | @ComputePath varchar(200) = null,
10 |
11 | --Custom
12 | @CustomParameters varchar(max) = null,
13 |
14 | --Input File
15 | @InputRawFileSystem varchar(50) = null,
16 | @InputRawFileFolder varchar(200) = null,
17 | @InputRawFile varchar(200) = null,
18 | @InputRawFileDelimiter char(1) = null,
19 | @InputFileHeaderFlag bit = null,
20 |
21 | --Curated File
22 | @OutputL1CurateFileSystem varchar(50) = null,
23 | @OutputL1CuratedFolder varchar(200) = null,
24 | @OutputL1CuratedFile varchar(200) = null,
25 | @OutputL1CuratedFileDelimiter char(1) = null,
26 | @OutputL1CuratedFileFormat varchar(10) = null,
27 | @OutputL1CuratedFileWriteMode varchar(20) = null,
28 |
29 | --SQL
30 | @OutputDWStagingTable varchar(200) = null,
31 | @LookupColumns varchar(4000) = null,
32 | @OutputDWTable varchar(200) = null,
33 | @OutputDWTableWriteMode varchar(20) = null,
34 | @IngestCount int = null,
35 |
36 | --ADF Pipeline IDs
37 | @IngestADFPipelineRunID uniqueidentifier = null
38 |
39 |
40 |
41 |
42 | AS
43 | BEGIN
44 |
45 |
46 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
47 |
48 | --Check if Transformation records already exists for the input file for same transformation e.g it's a reload
49 | IF NOT EXISTS
50 | (
51 | SELECT 1
52 | FROM
53 | [ELT].[L1TransformInstance]
54 | WHERE
55 | [IngestID] = @IngestID
56 | AND L1TransformID = @L1TransformID
57 | AND InputRawFileSystem = @InputRawFileSystem
58 | AND InputRawFileFolder = @InputRawFileFolder
59 | AND InputRawFile = @InputRawFile
60 |
61 | )
62 |
63 |
64 | BEGIN
65 | --If this is a new transformation
66 | INSERT INTO [ELT].[L1TransformInstance]
67 | (
68 | [L1TransformID]
69 | ,[IngestInstanceID]
70 | ,[IngestID]
71 | ,[ComputeName]
72 | ,[ComputePath]
73 | ,[CustomParameters]
74 | ,[InputRawFileSystem]
75 | ,[InputRawFileFolder]
76 | ,[InputRawFile]
77 | ,[InputRawFileDelimiter]
78 | ,[InputFileHeaderFlag]
79 | ,[OutputL1CurateFileSystem]
80 | ,[OutputL1CuratedFolder]
81 | ,[OutputL1CuratedFile]
82 | ,[OutputL1CuratedFileDelimiter]
83 | ,[OutputL1CuratedFileFormat]
84 | ,[OutputL1CuratedFileWriteMode]
85 | ,[OutputDWStagingTable]
86 | ,[LookupColumns]
87 | ,[OutputDWTable]
88 | ,[OutputDWTableWriteMode]
89 | ,[RetryCount]
90 | ,[ActiveFlag]
91 | ,[IngestCount]
92 | ,[IngestADFPipelineRunID]
93 | ,[CreatedBy]
94 | ,[CreatedTimestamp]
95 |
96 | )
97 | VALUES
98 | (
99 | @L1TransformID
100 | ,@IngestInstanceID
101 | ,@IngestID
102 | ,@ComputeName
103 | ,@ComputePath
104 | ,@CustomParameters
105 | ,@InputRawFileSystem
106 | ,@InputRawFileFolder
107 | ,@InputRawFile
108 | ,@InputRawFileDelimiter
109 | ,@InputFileHeaderFlag
110 | ,@OutputL1CurateFileSystem
111 | ,@OutputL1CuratedFolder
112 | ,@OutputL1CuratedFile
113 | ,@OutputL1CuratedFileDelimiter
114 | ,@OutputL1CuratedFileFormat
115 | ,@OutputL1CuratedFileWriteMode
116 | ,@OutputDWStagingTable
117 | ,@LookupColumns
118 | ,@OutputDWTable
119 | ,@OutputDWTableWriteMode
120 | ,0
121 | ,1
122 | ,@IngestCount
123 | ,@IngestADFPipelineRunID
124 | ,SUSER_SNAME()
125 | ,@localdate
126 | )
127 | END
128 | ELSE
129 | --If this is an existing Transformation
130 | BEGIN
131 | --Just update one record in case if there are duplicates
132 | UPDATE TOP (1) [ELT].[L1TransformInstance]
133 | SET
134 |
135 | [IngestCount] = null
136 | ,[L1TransformInsertCount] = null
137 | ,[L1TransformUpdateCount] = null
138 | ,[L1TransformDeleteCount] = null
139 | ,L1TransformStartTimestamp = null
140 | ,[L1TransformEndTimestamp] = null
141 | ,[L1TransformStatus] = null
142 | ,[RetryCount] = 0
143 | ,[ActiveFlag] = 1
144 | ,[ReRunL1TransformFlag] = 1
145 | ,[IngestADFPipelineRunID] = @IngestADFPipelineRunID
146 | ,[L1TransformADFPipelineRunID] = null
147 | ,[ModifiedBy] = suser_sname()
148 | ,[ModifiedTimestamp] = @localdate
149 |
150 | WHERE
151 | [IngestID] = @IngestID
152 | AND L1TransformID = @L1TransformID
153 | AND InputRawFileSystem = @InputRawFileSystem
154 | AND InputRawFileFolder = @InputRawFileFolder
155 | AND InputRawFile = @InputRawFile
156 | END
157 | END
158 |
159 |
160 |
161 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/InsertTransformInstance_L2.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[InsertTransformInstance_L2]
2 | --PK/FK
3 | @L2TransformID int = null,
4 | @IngestID int,
5 | @L1TransformID int,
6 |
7 | --Databricks Notebook
8 | @ComputeName varchar(100) = null,
9 | @ComputePath varchar(200) = null,
10 |
11 | --Custom
12 | @CustomParameters varchar(max) = null,
13 |
14 | --Input File
15 | @InputFileSystem varchar(50) = null,
16 | @InputFileFolder varchar(200) = null,
17 | @InputFile varchar(200) = null,
18 | @InputFileDelimiter char(1) = null,
19 | @InputFileHeaderFlag bit = null,
20 | @InputDWTable varchar(200) = null,
21 |
22 | --Delta
23 | @WatermarkColName varchar(50) = null,
24 | @DataFromTimestamp Datetime2 =null,
25 | @DataToTimestamp Datetime2 =null,
26 | @DataFromNumber int =null,
27 | @DataToNumber int =null,
28 | @LastDeltaDate datetime = null,
29 | @LastDeltaNumber int = null,
30 |
31 |
32 | --Curated File
33 | @OutputL2CurateFileSystem varchar(50),
34 | @OutputL2CuratedFolder varchar(200),
35 | @OutputL2CuratedFile varchar(200),
36 | @OutputL2CuratedFileDelimiter char(1) = null,
37 | @OutputL2CuratedFileFormat varchar(10) = null,
38 | @OutputL2CuratedFileWriteMode varchar(20) = null,
39 |
40 | --SQL
41 | @OutputDWStagingTable varchar(200) = null,
42 | @LookupColumns varchar(4000) = null,
43 | @OutputDWTable varchar(200) = null,
44 | @OutputDWTableWriteMode varchar(20) = null,
45 |
46 |
47 | --ADF Pipeline IDs
48 | @IngestADFPipelineRunID uniqueidentifier = null,
49 | @L1TransformADFPipelineRunID uniqueidentifier = null,
50 |
51 | --Max Retries
52 | @MaxRetries int = null
53 |
54 | AS
55 | BEGIN
56 |
57 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
58 |
59 | --Check if Transformation records already exists for the input file for same transformation e.g it's a reload
60 | IF NOT EXISTS
61 | (
62 | SELECT 1
63 | FROM
64 | [ELT].[L2TransformInstance]
65 | WHERE
66 | [IngestID] = @IngestID
67 | AND L2TransformID = @L2TransformID
68 | --Check does curated file record already exist
69 | AND OutputL2CurateFileSystem = @OutputL2CurateFileSystem
70 | AND OutputL2CuratedFolder = @OutputL2CuratedFolder
71 | AND OutputL2CuratedFile = @OutputL2CuratedFile
72 |
73 | )
74 |
75 |
76 | BEGIN
77 | --If this is a new transformation
78 | INSERT INTO [ELT].[L2TransformInstance]
79 | (
80 | [L2TransformID]
81 | ,[IngestID]
82 | ,[L1TransformID]
83 | ,[ComputePath]
84 | ,[ComputeName]
85 | ,[CustomParameters]
86 | ,[InputFileSystem]
87 | ,[InputFileFolder]
88 | ,[InputFile]
89 | ,[InputFileDelimiter]
90 | ,[InputFileHeaderFlag]
91 | ,[InputDWTable]
92 | ,[WatermarkColName]
93 | ,[OutputL2CurateFileSystem]
94 | ,[OutputL2CuratedFolder]
95 | ,[OutputL2CuratedFile]
96 | ,[OutputL2CuratedFileDelimiter]
97 | ,[OutputL2CuratedFileFormat]
98 | ,[OutputL2CuratedFileWriteMode]
99 | ,[OutputDWStagingTable]
100 | ,[LookupColumns]
101 | ,[OutputDWTable]
102 | ,[OutputDWTableWriteMode]
103 | ,[RetryCount]
104 | ,[ActiveFlag]
105 | ,[IngestADFPipelineRunID]
106 | ,[L1TransformADFPipelineRunID] --1555
107 | ,[CreatedBy]
108 | ,[CreatedTimestamp]
109 |
110 |
111 | )
112 | VALUES
113 | (
114 | @L2TransformID,
115 | @IngestID,
116 | @L1TransformID,
117 | @ComputePath,
118 | @ComputeName,
119 | @CustomParameters,
120 | @InputFileSystem,
121 | @InputFileFolder,
122 | @InputFile,
123 | @InputFileDelimiter,
124 | @InputFileHeaderFlag,
125 | @InputDWTable,
126 | @WatermarkColName,
127 | @OutputL2CurateFileSystem,
128 | @OutputL2CuratedFolder,
129 | @OutputL2CuratedFile,
130 | @OutputL2CuratedFileDelimiter,
131 | @OutputL2CuratedFileFormat,
132 | @OutputL2CuratedFileWriteMode,
133 | @OutputDWStagingTable,
134 | @LookupColumns,
135 | @OutputDWTable,
136 | @OutputDWTableWriteMode,
137 | 0,
138 | 1,
139 | @IngestADFPipelineRunID,
140 | @L1TransformADFPipelineRunID, --1555
141 | SUSER_NAME(),
142 | @localdate
143 | )
144 | END
145 | ELSE
146 | --If this is an existing Transformation
147 | BEGIN
148 | --Just update one record in case if there are duplicates
149 | UPDATE TOP (1) [ELT].[L2TransformInstance]
150 | SET
151 | [InputCount] = null
152 | ,[L2TransformInsertCount] = null
153 | ,[L2TransformUpdateCount] = null
154 | ,[L2TransformDeleteCount] = null
155 | ,[L2TransformStartTimestamp] = null
156 | ,[L2TransformEndTimestamp] = null
157 | ,[L2TransformStatus] = null
158 | ,[RetryCount] = 0
159 | ,[ActiveFlag] = 1
160 | ,[ReRunL2TransformFlag] = 1
161 | ,[IngestADFPipelineRunID] = @IngestADFPipelineRunID
162 | ,[L1TransformADFPipelineRunID] = null
163 | ,[L2TransformADFPipelineRunID] = null
164 | ,[ModifiedBy] = suser_sname()
165 | ,[ModifiedTimestamp] = @localdate
166 | WHERE
167 | [IngestID] = @IngestID
168 | AND L2TransformID = @L2TransformID
169 | --Check does curated file record already exist
170 | AND OutputL2CurateFileSystem = @OutputL2CurateFileSystem
171 | AND OutputL2CuratedFolder = @OutputL2CuratedFolder
172 | AND OutputL2CuratedFile = @OutputL2CuratedFile
173 | AND (ActiveFlag = 0 AND ISNULL(RetryCount,0) >= @MaxRetries)
174 |
175 | END
176 | END
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/UpdateIngestDefinition.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[UpdateIngestDefinition]
2 | @IngestID INT,
3 | @LastDeltaDate Datetime2=null,
4 | @LastDeltaNumer int=null,
5 | @IngestStatus varchar(20),
6 | @ReloadFlag bit=0
7 | AS
8 | BEGIN
9 |
10 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
11 |
12 | Update
13 | [ELT].[IngestDefinition]
14 | SET
15 | [LastDeltaDate] =
16 | CASE
17 | --When Successful and the DataToDate does not move forward since LastDeltaDate, Increase LastDeltaDate by the Interval
18 | WHEN @LastDeltaDate IS NOT NULL AND @ReloadFlag <> 1 AND @IngestStatus IN ('Success','ReRunSuccess') AND @LastDeltaDate = [LastDeltaDate]
19 | and [MaxIntervalMinutes] is NOT NULL
20 | THEN
21 | CASE
22 | WHEN
23 | DateAdd(minute,[MaxIntervalMinutes],@LastDeltaDate) > ELT.[uf_GetAestDateTime]()
24 | THEN CONVERT(VARCHAR(30),ELT.[uf_GetAestDateTime](),120)
25 | ELSE
26 | DateAdd(minute,[MaxIntervalMinutes],[LastDeltaDate])
27 | END
28 | --Re-run delta date is later than existing delta date
29 | WHEN @LastDeltaDate IS NOT NULL AND @IngestStatus IN ('Success','ReRunSuccess') AND datediff_big(ss,[LastDeltaDate],@LastDeltaDate) >= 0
30 | THEN @LastDeltaDate
31 | --Re-run delta date is earlier than existing delta date
32 | WHEN @LastDeltaDate IS NOT NULL AND @IngestStatus IN ('Success','ReRunSuccess') AND datediff_big(ss,@LastDeltaDate,[LastDeltaDate]) >=0
33 | THEN [LastDeltaDate]
34 | ELSE [LastDeltaDate]
35 | END
36 | , [LastDeltaNumber] =
37 | CASE
38 | WHEN @LastDeltaNumer IS NOT NULL AND @IngestStatus IN ('Success','ReRunSuccess')
39 | THEN @LastDeltaNumer
40 | WHEN @LastDeltaNumer IS NOT NULL AND @IngestStatus IN ('Failure','ReRunFailure')
41 | THEN [LastDeltaNumber]
42 | WHEN @LastDeltaNumer IS NULL AND @ReloadFlag <> 1 AND @IngestStatus IN ('Success','ReRunSuccess')
43 | THEN ([LastDeltaNumber] + [MaxIntervalNumber])
44 | ELSE [LastDeltaNumber]
45 | END
46 | ,[ModifiedBy] =suser_sname()
47 | , [ModifiedTimestamp]=@localdate
48 | WHERE [IngestID]=@IngestID
49 | END
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/UpdateIngestInstance.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[UpdateIngestInstance]
2 | @ADFIngestPipelineRunID Uniqueidentifier,
3 | @IngestStatus varchar(20) =null,
4 | @DataFromTimestamp Datetime2 =null,
5 | @DataToTimestamp Datetime2 =null,
6 | @DataFromNumber int =null,
7 | @DataToNumber int =null,
8 | @SourceCount int=null,
9 | @IngestCount int=null,
10 | @ReloadFlag bit
11 | AS
12 | BEGIN
13 |
14 | DECLARE @localdate as datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
15 |
16 | Update
17 | [ELT].[IngestInstance]
18 | SET
19 | [DataFromTimestamp] = @DataFromTimestamp
20 | ,[DataToTimestamp]=@DataToTimestamp
21 | ,[DataFromNumber] = @DataFromNumber
22 | ,[DataToNumber]=@DataToNumber
23 | ,[SourceCount] =@SourceCount
24 | ,[IngestCount]=@IngestCount
25 | ,[IngestEndTimestamp] =@localdate
26 | ,[IngestStatus] =(CASE WHEN @ReloadFlag=1 AND @IngestStatus='Success' THEN 'ReRunSuccess'
27 | WHEN @ReloadFlag=1 AND @IngestStatus='Failure' THEN 'ReRunFailure'
28 | ELSE @IngestStatus
29 | END)
30 | ,[RetryCount] = (CASE WHEN @IngestStatus IN ('Success','ReRunSuccess') THEN 0
31 | WHEN @IngestStatus IN ('Failure','ReRunFailure') THEN ISNULL([RetryCount],0) +1
32 | END)
33 | ,[ReloadFlag] =(CASE WHEN @ReloadFlag=1 AND @IngestStatus IN ('Success','ReRunSuccess') THEN 0
34 | WHEN @ReloadFlag=1 AND @IngestStatus IN ('Failure','ReRunFailure') THEN 1
35 | ELSE 0
36 | END)
37 | ,[ModifiedBy]=suser_sname()
38 | ,[ModifiedTimestamp] = @localdate
39 | WHERE
40 | ADFIngestPipelineRunID =@ADFIngestPipelineRunID
41 | END
42 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/UpdateTransformDefinition_L2.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[UpdateTransformDefinition_L2]
2 | @L2TransformID INT,
3 | @LastDeltaDate Datetime2 =null,
4 | @LastDeltaNumber int =null
5 | as
6 | BEGIN
7 | Update [ELT].[L2TransformDefinition]
8 | SET
9 | [ModifiedBy] =suser_sname(),
10 | [ModifiedTimestamp]=GETDATE(),
11 | [LastDeltaDate] = COALESCE(@LastDeltaDate,[LastDeltaDate],ELT.uf_GetAestDateTime()),
12 | [LastDeltaNumber] = COALESCE(@LastDeltaNumber,[LastDeltaNumber])
13 |
14 | WHERE [L2TransformID] = @L2TransformID
15 | END
16 | GO
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/UpdateTransformInstance_L1.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[UpdateTransformInstance_L1]
2 | @L1TransformInstanceId INT
3 | , @L1TransformStatus VARCHAR(20)
4 | , @L1TransformADFPipelineRunID UNIQUEIDENTIFIER
5 | , @IngestCount INT = NULL
6 | , @L1TransformInsertCount INT = NULL
7 | , @L1TransformUpdateCount INT = NULL
8 | , @L1TransformDeleteCount INT = NULL
9 | , @MaxRetries int = null
10 | AS
11 | BEGIN
12 |
13 | DECLARE @localdate datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
14 |
15 | Update
16 | [ELT].[L1TransformInstance]
17 | SET
18 | [L1TransformStartTimestamp] = CASE
19 | WHEN @L1TransformStatus IN ('Running')
20 | THEN @localdate
21 | ELSE [L1TransformStartTimestamp]
22 | END
23 |
24 | , [L1TransformEndTimestamp] = CASE
25 | WHEN @L1TransformStatus IN ('Success','Failure','ReRunSuccess','ReRunFailure')
26 | THEN @localdate
27 | ELSE NULL
28 | END
29 |
30 | , [L1TransformStatus] = CASE
31 | WHEN ([ReRunL1TransformFlag] = 1 OR [RetryCount] >0) AND @L1TransformStatus='Success'
32 | THEN 'ReRunSuccess'
33 | WHEN [ReRunL1TransformFlag] = 1 AND @L1TransformStatus='Failure'
34 | THEN 'ReRunFailure'
35 | ELSE @L1TransformStatus
36 | END
37 | , [ActiveFlag] = CASE
38 | WHEN @L1TransformStatus IN ('Success','ReRunSuccess')
39 | THEN 0
40 | WHEN @L1TransformStatus = 'Failure' and ISNULL(RetryCount,0) +1 >= @MaxRetries
41 | THEN 0
42 | ELSE 1
43 | END
44 | , [ReRunL1TransformFlag] = CASE
45 | WHEN [ReRunL1TransformFlag] = 1 AND @L1TransformStatus IN ('Success','ReRunSuccess')
46 | THEN 0
47 | WHEN @L1TransformStatus in ('Failure', 'ReRunFailure') and ISNULL(RetryCount,0) +1 >= @MaxRetries
48 | THEN 0
49 | ELSE [ReRunL1TransformFlag]
50 | END
51 | , [RetryCount] = CASE
52 | WHEN
53 | @L1TransformStatus in ('Success', 'ReRunSuccess')
54 | THEN 0
55 | WHEN
56 | @L1TransformStatus in ('Failure', 'ReRunFailure')
57 | THEN ISNULL([RetryCount],0) + 1
58 | Else [RetryCount]
59 | END
60 |
61 | , [ModifiedBy] =suser_sname()
62 | , [ModifiedTimestamp]=@localdate
63 | , [L1TransformADFPipelineRunID] = @L1TransformADFPipelineRunID
64 | , [IngestCount] = ISNULL(@IngestCount,[IngestCount])
65 | , [L1TransformInsertCount] = ISNULL(@L1TransformInsertCount,[L1TransformInsertCount])
66 | , [L1TransformUpdateCount] = ISNULL(@L1TransformUpdateCount,[L1TransformUpdateCount])
67 | , [L1TransformDeleteCount] = ISNULL(@L1TransformDeleteCount,[L1TransformDeleteCount])
68 | WHERE
69 | [L1TransformInstanceID] = @L1TransformInstanceId
70 | END
71 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Stored Procedures/UpdateTransformInstance_L2.sql:
--------------------------------------------------------------------------------
1 | CREATE PROCEDURE [ELT].[UpdateTransformInstance_L2]
2 | @L2TransformInstanceId INT
3 | ,@L2TransformStatus VARCHAR(20)
4 | ,@L2TransformADFPipelineRunID UNIQUEIDENTIFIER
5 | ,@InputCount INT = NULL
6 | ,@L2TransformInsertCount INT = NULL
7 | ,@L2TransformUpdateCount INT = NULL
8 | ,@L2TransformDeleteCount INT = NULL
9 | ,@DataFromTimestamp Datetime2 = null
10 | ,@DataToTimestamp Datetime2 = null
11 | ,@DataFromNumber int = null
12 | ,@DataToNumber int = null
13 | ,@MaxRetries int = null
14 | AS
15 | BEGIN
16 |
17 | DECLARE @localdate datetime = CONVERT(datetime,CONVERT(datetimeoffset, getdate()) at time zone 'AUS Eastern Standard Time')
18 |
19 | Update
20 | [ELT].[L2TransformInstance]
21 | SET
22 | [L2TransformStartTimestamp] = CASE
23 | WHEN @L2TransformStatus IN ('Running')
24 | THEN @localdate
25 | ELSE [L2TransformStartTimestamp]
26 | END
27 |
28 | , [L2TransformEndTimestamp] = CASE
29 | WHEN @L2TransformStatus IN ('Success','Failure','ReRunSuccess','ReRunFailure')
30 | THEN @localdate
31 | ELSE NULL
32 | END
33 | , [DataFromTimestamp] = @DataFromTimestamp
34 | , [DataToTimestamp] = @DataToTimestamp
35 | , [DataFromNumber] = @DataFromNumber
36 | , [DataToNumber] = @DataToNumber
37 | , [L2TransformStatus] = CASE
38 | WHEN ([ReRunL2TransformFlag] = 1 OR [RetryCount] > 0) AND @L2TransformStatus = 'Success'
39 | THEN 'ReRunSuccess'
40 | WHEN [ReRunL2TransformFlag] = 1 AND @L2TransformStatus = 'Failure'
41 | THEN 'ReRunFailure'
42 | ELSE @L2TransformStatus
43 | END
44 | , [ActiveFlag] = CASE
45 | WHEN @L2TransformStatus IN ('Success','ReRunSuccess')
46 | THEN 0
47 | WHEN @L2TransformStatus = 'Failure' and ISNULL(RetryCount,0) +1 >= @MaxRetries
48 | THEN 0
49 | ELSE 1
50 | END
51 | , [ReRunL2TransformFlag] = CASE
52 | WHEN [ReRunL2TransformFlag] =1 AND @L2TransformStatus IN ('Success','ReRunSuccess')
53 | THEN 0
54 | WHEN @L2TransformStatus in ('Failure', 'ReRunFailure') and ISNULL(RetryCount,0) +1 >= @MaxRetries
55 | THEN 0
56 | ELSE [ReRunL2TransformFlag]
57 | END
58 | , [RetryCount] = CASE
59 | WHEN
60 | @L2TransformStatus in ('Success', 'ReRunSuccess')
61 | THEN 0
62 | WHEN
63 | @L2TransformStatus in ('Failure', 'ReRunFailure')
64 | THEN ISNULL([RetryCount],0) + 1
65 | ELSE [RetryCount]
66 | END
67 |
68 | , [ModifiedBy] =suser_sname()
69 | , [ModifiedTimestamp]=@localdate
70 | , [L2TransformADFPipelineRunID] = @L2TransformADFPipelineRunID
71 | , [InputCount] = ISNULL(@InputCount,[InputCount])
72 | , [L2TransformInsertCount] = ISNULL(@L2TransformInsertCount,[L2TransformInsertCount])
73 | , [L2TransformUpdateCount] = ISNULL(@L2TransformUpdateCount,[L2TransformUpdateCount])
74 | , [L2TransformDeleteCount] = ISNULL(@L2TransformDeleteCount,[L2TransformDeleteCount])
75 | WHERE
76 | [L2TransformInstanceID] = @L2TransformInstanceId
77 | END
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Tables/ColumnMapping.sql:
--------------------------------------------------------------------------------
1 | GO
2 |
3 | CREATE TABLE [ELT].[ColumnMapping](
4 | [MappingID] int NOT NULL IDENTITY,
5 | [IngestID] int NULL,
6 | [L1TransformID] int NULL,
7 | [L2TransformID] int NULL,
8 | [SourceName] nvarchar(150) NOT NULL,
9 | [TargetName] nvarchar(150) NOT NULL,
10 | [Description] nvarchar(250) NULL,
11 | [TargetOrdinalPosition] int NOT NULL,
12 | [ActiveFlag] bit NULL,
13 | [CreatedBy] varchar(128) NOT NULL,
14 | [CreatedTimestamp] datetime NOT NULL,
15 | [ModifiedBy] varchar(128) NULL,
16 | [ModifiedTimestamp] datetime NULL)
17 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Tables/IngestDefinition.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE [ELT].[IngestDefinition]
2 | (
3 | [IngestID] int not null identity,
4 | [SourceSystemName] varchar(50) not null,
5 | [StreamName] varchar(100) null,
6 | [SourceSystemDescription] varchar(200) null,
7 | [Backend] varchar(30) null,
8 | [DataFormat] varchar(10) null,
9 | [EntityName] varchar(100) null,
10 | [WatermarkColName] varchar(50) null,
11 | [DeltaFormat] varchar(30) null,
12 | [LastDeltaDate] datetime2 null,
13 | [LastDeltaNumber] int null,
14 | [LastDeltaString] varchar(50) null,
15 | [MaxIntervalMinutes] int null,
16 | [MaxIntervalNumber] int null,
17 | [DataMapping] varchar(max) null,
18 | [SourceFileDropFileSystem] varchar(50) null,
19 | [SourceFileDropFolder] varchar(200) null,
20 | [SourceFileDropFile] varchar(200) null,
21 | [SourceFileDelimiter] char(1) null,
22 | [SourceFileHeaderFlag] bit null,
23 | [SourceStructure] varchar(max) null,
24 | [DestinationRawFileSystem] varchar(50) null,
25 | [DestinationRawFolder] varchar(200) null,
26 | [DestinationRawFile] varchar(200) null,
27 | [RunSequence] int null,
28 | [MaxRetries] int null,
29 | [ActiveFlag] bit not null,
30 | [L1TransformationReqdFlag] bit not null,
31 | [L2TransformationReqdFlag] bit not null,
32 | [DelayL1TransformationFlag] bit not null,
33 | [DelayL2TransformationFlag] bit not null,
34 | [CreatedBy] nvarchar(128) not null,
35 | [CreatedTimestamp] datetime not null,
36 | [ModifiedBy] nvarchar(128) null,
37 | [ModifiedTimestamp] datetime null,
38 | )
39 |
40 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Tables/IngestInstance.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE [ELT].[IngestInstance]
2 | (
3 | [IngestInstanceID] int not null identity,
4 | [IngestID] int not null,
5 | [SourceFileDropFileSystem] varchar(50) null,
6 | [SourceFileDropFolder] varchar(200) null,
7 | [SourceFileDropFile] varchar(200) null,
8 | [DestinationRawFileSystem] varchar(50) not null,
9 | [DestinationRawFolder] varchar(200) not null,
10 | [DestinationRawFile] varchar(200) not null,
11 | [DataFromTimestamp] dateTime2 null,
12 | [DataToTimestamp] datetime2 null,
13 | [DataFromNumber] Int null,
14 | [DataToNumber] Int null,
15 | [SourceCount] int null,
16 | [IngestCount] int null,
17 | [IngestStartTimestamp] datetime null,
18 | [IngestEndTimestamp] datetime null,
19 | [IngestStatus] varchar(20) null,
20 | [RetryCount] int null,
21 | [ReloadFlag] bit null,
22 | [CreatedBy] nvarchar(128) not null,
23 | [CreatedTimestamp] datetime not null,
24 | [ModifiedBy] nvarchar(128) null,
25 | [ModifiedTimestamp] datetime null,
26 | [ADFIngestPipelineRunID] uniqueidentifier null
27 | )
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Tables/L1TransformDefinition.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE [ELT].[L1TransformDefinition]
2 | (
3 | [L1TransformID] int not null identity,
4 | [IngestID] int not null,
5 | [ComputePath] varchar(200) null,
6 | [ComputeName] varchar(100) null,
7 | [CustomParameters] varchar(max) null,
8 | [InputRawFileSystem] varchar(50) not null,
9 | [InputRawFileFolder] varchar(200) not null,
10 | [InputRawFile] varchar(200) not null,
11 | [InputRawFileDelimiter] char(1) null,
12 | [InputFileHeaderFlag] bit null,
13 | [OutputL1CurateFileSystem] varchar(50) not null,
14 | [OutputL1CuratedFolder] varchar(200) not null,
15 | [OutputL1CuratedFile] varchar(200) not null,
16 | [OutputL1CuratedFileDelimiter] char(1) null,
17 | [OutputL1CuratedFileFormat] varchar(10) null,
18 | [OutputL1CuratedFileWriteMode] varchar(20) null,
19 | --Output SQL DW Table
20 | [OutputDWStagingTable] varchar(200) null,
21 | [LookupColumns] varchar(4000) null,
22 | [OutputDWTable] varchar(200) null,
23 | [OutputDWTableWriteMode] varchar(20) null,
24 | [MaxRetries] int null,
25 | [WatermarkColName] varchar(50) null,
26 | [ActiveFlag] bit not null,
27 | [CreatedBy] nvarchar(128) not null,
28 | [CreatedTimestamp] datetime not null,
29 | [ModifiedBy] nvarchar(128) null,
30 | [ModifiedTimestamp] datetime null
31 | )
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Tables/L1TransformInstance.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE [ELT].[L1TransformInstance]
2 | (
3 | [L1TransformInstanceID] int not null identity,
4 | [L1TransformID] int not null,
5 | [IngestInstanceID] int null,
6 | [IngestID] int not null,
7 | [ComputeName] varchar(100) null,
8 | [ComputePath] varchar(200) null,
9 | [CustomParameters] varchar(max) null,
10 | [InputRawFileSystem] varchar(50) not null,
11 | [InputRawFileFolder] varchar(200) not null,
12 | [InputRawFile] varchar(200) not null,
13 | [InputRawFileDelimiter] char(1) null,
14 | [InputFileHeaderFlag] bit null,
15 | [OutputL1CurateFileSystem] varchar(50) not null,
16 | [OutputL1CuratedFolder] varchar(200) not null,
17 | [OutputL1CuratedFile] varchar(200) not null,
18 | [OutputL1CuratedFileDelimiter] char(1) null,
19 | [OutputL1CuratedFileFormat] varchar(10) null,
20 | [OutputL1CuratedFileWriteMode] varchar(20) null,
21 | [OutputDWStagingTable] varchar(200) null,
22 | [LookupColumns] varchar(4000) null,
23 | [OutputDWTable] varchar(200) null,
24 | [OutputDWTableWriteMode] varchar(20) null,
25 | [IngestCount] int null,
26 | [L1TransformInsertCount] int null,
27 | [L1TransformUpdateCount] int null,
28 | [L1TransformDeleteCount] int null,
29 | [L1TransformStartTimestamp] datetime null,
30 | [L1TransformEndTimestamp] datetime null,
31 | [L1TransformStatus] varchar(20) null,
32 | [RetryCount] int null,
33 | [ActiveFlag] bit not null,
34 | [ReRunL1TransformFlag] bit null,
35 | [IngestADFPipelineRunID] uniqueidentifier null,
36 | [L1TransformADFPipelineRunID] uniqueidentifier null,
37 | [CreatedBy] nvarchar(128) not null,
38 | [CreatedTimestamp] datetime not null,
39 | [ModifiedBy] nvarchar(128) null,
40 | [ModifiedTimestamp] datetime null
41 | )
42 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Tables/L2TransformDefinition.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE [ELT].[L2TransformDefinition]
2 | (
3 | [L2TransformID] int not null identity,
4 | [IngestID] int null,
5 | [L1TransformID] int null,
6 | [ComputePath] varchar(200) null,
7 | [ComputeName] varchar(100) null,
8 | [CustomParameters] varchar(max) null,
9 | [InputType] varchar(15) null,
10 | [InputFileSystem] varchar(50) null,
11 | [InputFileFolder] varchar(200) null,
12 | [InputFile] varchar(200) null,
13 | [InputFileDelimiter] char(1) null,
14 | [InputFileHeaderFlag] bit null,
15 | [InputDWTable] varchar(200) null,
16 | [WatermarkColName] varchar(50) null,
17 | [LastDeltaDate] datetime2 null,
18 | [LastDeltaNumber] int null,
19 | [MaxIntervalMinutes] int null,
20 | [MaxIntervalNumber] int null,
21 | [MaxRetries] int null,
22 | [OutputL2CurateFileSystem] varchar(50) null,
23 | [OutputL2CuratedFolder] varchar(200) null,
24 | [OutputL2CuratedFile] varchar(200) null,
25 | [OutputL2CuratedFileDelimiter] char(1) null,
26 | [OutputL2CuratedFileFormat] varchar(10) null,
27 | [OutputL2CuratedFileWriteMode] varchar(20) null,
28 | --Output SQL DW Table
29 | [OutputDWStagingTable] varchar(200) null,
30 | [LookupColumns] varchar(4000) null,
31 | [OutputDWTable] varchar(200) null,
32 | [OutputDWTableWriteMode] varchar(20) null,
33 | [ActiveFlag] bit not null,
34 | [RunSequence] int not null,
35 | [CreatedBy] nvarchar(128) not null,
36 | [CreatedTimestamp] datetime not null,
37 | [ModifiedBy] nvarchar(128) null,
38 | [ModifiedTimestamp] datetime null
39 | )
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/Tables/L2TransformInstance.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE [ELT].[L2TransformInstance]
2 | (
3 | [L2TransformInstanceID] int not null identity,
4 | [L2TransformID] int null,
5 | [IngestID] int null,
6 | [L1TransformID] int null,
7 | [ComputePath] varchar(200) null,
8 | [ComputeName] varchar(100) null,
9 | [CustomParameters] varchar(max) null,
10 | [InputFileSystem] varchar(50) null,
11 | [InputFileFolder] varchar(200) null,
12 | [InputFile] varchar(200) null,
13 | [InputFileDelimiter] char(1) null,
14 | [InputFileHeaderFlag] bit null,
15 | [InputDWTable] varchar(200) null,
16 | [WatermarkColName] varchar(50) null,
17 | [DataFromTimestamp] dateTime2 null,
18 | [DataToTimestamp] datetime2 null,
19 | [DataFromNumber] int null,
20 | [DataToNumber] int null,
21 | [OutputL2CurateFileSystem] varchar(50) not null,
22 | [OutputL2CuratedFolder] varchar(200) not null,
23 | [OutputL2CuratedFile] varchar(200) not null,
24 | [OutputL2CuratedFileDelimiter] char(1) null,
25 | [OutputL2CuratedFileFormat] varchar(10) null,
26 | [OutputL2CuratedFileWriteMode] varchar(20) null,
27 | [OutputDWStagingTable] varchar(200) null,
28 | [LookupColumns] varchar(4000) null,
29 | [OutputDWTable] varchar(200) null,
30 | [OutputDWTableWriteMode] varchar(20) null,
31 | [InputCount] int null,
32 | [L2TransformInsertCount] int null,
33 | [L2TransformUpdateCount] int null,
34 | [L2TransformDeleteCount] int null,
35 | [L2TransformStartTimestamp] datetime null,
36 | [L2TransformEndTimestamp] datetime null,
37 | [L2TransformStatus] varchar(20) null,
38 | [RetryCount] int null,
39 | [ActiveFlag] bit not null,
40 | [ReRunL2TransformFlag] bit null,
41 | [IngestADFPipelineRunID] uniqueidentifier null,
42 | [L1TransformADFPipelineRunID] uniqueidentifier null,
43 | [L2TransformADFPipelineRunID] uniqueidentifier null,
44 | [CreatedBy] nvarchar(128) not null,
45 | [CreatedTimestamp] datetime not null,
46 | [ModifiedBy] nvarchar(128) null,
47 | [ModifiedTimestamp] datetime null,
48 | [L2SnapshotGroupID] INT NULL,
49 | [L2SnapshotInstanceID] INT NULL
50 | )
51 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/ELT/elt.sql:
--------------------------------------------------------------------------------
1 | CREATE SCHEMA [ELT]
--------------------------------------------------------------------------------
/elt-framework/ControlDB/Post Deployment Script/Script.PostDeployment.sql:
--------------------------------------------------------------------------------
1 | /*
2 | Post-Deployment Script Template
3 | --------------------------------------------------------------------------------------
4 | This file contains SQL statements that will be appended to the build script.
5 | Use SQLCMD syntax to include a file in the post-deployment script.
6 | Example: :r .\myfile.sql
7 | Use SQLCMD syntax to reference a variable in the post-deployment script.
8 | Example: :setvar TableName MyTable
9 | SELECT * FROM [$(TableName)]
10 | --------------------------------------------------------------------------------------
11 | */
12 |
13 | --IngestDefinition
14 | :r .\datasources\RestAPI\IngestDefinition\PurviewRestAPI.sql
15 | :r .\datasources\RestAPI\IngestDefinition\AzureRestAPI.sql
16 | :r .\datasources\UploadFiles\IngestDefinition\CustomFormRecognizerModel.sql
17 |
18 | --L1TransformDefinition
19 | :r .\datasources\RestAPI\L1TransformDefinition\L1T_PurviewRestAPI.sql
20 | :r .\datasources\RestAPI\L1TransformDefinition\L1T_AzureRestAPI.sql
21 | :r .\datasources\UploadFiles\L1TransformDefinition\L1T_CustomFormRecognizerModel.sql
22 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/Post Deployment Script/datasources/RestAPI/IngestDefinition/AzureRestAPI.sql:
--------------------------------------------------------------------------------
1 | /*
2 | ELT Configuration for REST API Data Sources
3 | */
4 |
5 | IF OBJECT_ID('tempdb..#AzureRestAPI_Ingest') IS NOT NULL DROP TABLE #AzureRestAPI_Ingest;
6 |
7 | --Create Temp table with same structure as IngestDefinition
8 | CREATE TABLE #AzureRestAPI_Ingest
9 | (
10 | [SourceSystemName] [varchar](50) NOT NULL,
11 | [StreamName] [varchar](100) NULL,
12 | [SourceSystemDescription] [varchar](200) NULL,
13 | [Backend] [varchar](30) NULL,
14 | [EntityName] [varchar](100) NULL,
15 | [WatermarkColName] [varchar](50) NULL,
16 | [LastDeltaDate] [datetime2](7) NULL,
17 | [LastDeltaNumber] [int] NULL,
18 | [LastDeltaString] [varchar](50) NULL,
19 | [MaxIntervalMinutes] [int] NULL,
20 | [MaxIntervalNumber] [int] NULL,
21 | [DataMapping] [varchar](max) NULL,
22 | [DestinationRawFileSystem] [varchar](50) NULL,
23 | [DestinationRawFolder] [varchar](200) NULL,
24 | [DestinationRawFile] [varchar](200) NULL,
25 | [RunSequence] [int] NULL,
26 | [MaxRetries] [int] NULL,
27 | [ActiveFlag] [bit] NOT NULL,
28 | [L1TransformationReqdFlag] [bit] NOT NULL,
29 | [L2TransformationReqdFlag] [bit] NOT NULL,
30 | [DelayL1TransformationFlag] [bit] NOT NULL,
31 | [DelayL2TransformationFlag] [bit] NOT NULL
32 | );
33 |
34 |
35 | --Insert Into Temp Table
36 | INSERT INTO #AzureRestAPI_Ingest
37 | --Operations
38 | SELECT 'Azure' AS [SourceSystemName],
39 | 'operations' AS [StreamName],
40 | 'Azure APIs' AS [SourceSystemDescription],
41 | 'AZURE REST API' AS [Backend],
42 | '/providers/Microsoft.Purview/operations?api-version=2020-12-01-preview' AS [EntityName],
43 | NULL AS [WatermarkColName],
44 | NULL AS [LastDeltaDate],
45 | NULL AS [LastDeltaNumber],
46 | NULL AS [LastDeltaString],
47 | NULL AS [MaxIntervalMinutes],
48 | NULL AS [MaxIntervalNumber],
49 | '{
50 | "type": "TabularTranslator",
51 | "mappings": [
52 | {
53 | "source": {
54 | "path": "[''name'']"
55 | },
56 | "sink": {
57 | "name": "name",
58 | "type": "String"
59 | }
60 | },
61 | {
62 | "source": {
63 | "path": "[''display''][''provider'']"
64 | },
65 | "sink": {
66 | "name": "provider",
67 | "type": "String"
68 | }
69 | },
70 | {
71 | "source": {
72 | "path": "[''display''][''resource'']"
73 | },
74 | "sink": {
75 | "name": "resource",
76 | "type": "String"
77 | }
78 | },
79 | {
80 | "source": {
81 | "path": "[''display''][''operation'']"
82 | },
83 | "sink": {
84 | "name": "operation",
85 | "type": "String"
86 | }
87 | },
88 | {
89 | "source": {
90 | "path": "[''display''][''description'']"
91 | },
92 | "sink": {
93 | "name": "description",
94 | "type": "String"
95 | }
96 | }
97 | ],
98 | "collectionReference": "$[''value'']",
99 | "mapComplexValuesToString": true
100 | }'AS [DataMapping],
101 | 'raw-bronze' AS [DestinationRawFileSystem],
102 | 'purview/operations/YYYY-MM' AS [DestinationRawFolder] ,
103 | 'operations_YYYYMMDD.parquet' AS [DestinationRawFile],
104 | 1 AS [RunSequence],
105 | 3 AS [MaxRetries],
106 | 1 AS [ActiveFlag],
107 | 1 AS [L1TransformationReqdFlag],
108 | 1 AS [L2TransformationReqdFlag],
109 | 1 AS [DelayL1TransformationFlag],
110 | 1 AS [DelayL2TransformationFlag] ;
111 |
112 | --Merge with Temp table for re-runnability
113 |
114 | MERGE INTO [ELT].[IngestDefinition] AS tgt
115 | USING #AzureRestAPI_Ingest AS src
116 | ON src.[SourceSystemName]=tgt.[SourceSystemName] AND src.[StreamName] =tgt.[StreamName]
117 | WHEN MATCHED THEN
118 | UPDATE SET tgt.[SourceSystemDescription] = src.[SourceSystemDescription],
119 | tgt.[Backend] = src.[Backend],
120 | tgt.[EntityName] = src.[EntityName],
121 | tgt.[WatermarkColName] = src.[WatermarkColName],
122 | tgt.[LastDeltaDate] = src.[LastDeltaDate],
123 | tgt.[LastDeltaNumber] = src.[LastDeltaNumber],
124 | tgt.[LastDeltaString] = src.[LastDeltaString],
125 | tgt.[MaxIntervalMinutes] = src.[MaxIntervalMinutes],
126 | tgt.[MaxIntervalNumber] = src.[MaxIntervalNumber],
127 | tgt.[DataMapping] = src.[DataMapping],
128 | tgt.[DestinationRawFileSystem] = src.[DestinationRawFileSystem],
129 | tgt.[DestinationRawFolder] = src.[DestinationRawFolder],
130 | tgt.[DestinationRawFile] = src.[DestinationRawFile],
131 | tgt.[RunSequence] = src.[RunSequence],
132 | tgt.[MaxRetries] = src.[MaxRetries],
133 | tgt.[ActiveFlag] = src.[ActiveFlag],
134 | tgt.[L1TransformationReqdFlag] = src.[L1TransformationReqdFlag],
135 | tgt.[L2TransformationReqdFlag] = src.[L2TransformationReqdFlag],
136 | tgt.[DelayL1TransformationFlag] = src.[DelayL1TransformationFlag],
137 | tgt.[DelayL2TransformationFlag] = src.[DelayL2TransformationFlag],
138 | tgt.[ModifiedBy] = USER_NAME(),
139 | tgt.[ModifiedTimestamp] = GetDate()
140 | WHEN NOT MATCHED BY TARGET THEN
141 | INSERT([SourceSystemName],
142 | [StreamName],
143 | [SourceSystemDescription],
144 | [Backend],
145 | [EntityName],
146 | [WatermarkColName],
147 | [LastDeltaDate],
148 | [LastDeltaNumber],
149 | [LastDeltaString],
150 | [MaxIntervalMinutes],
151 | [MaxIntervalNumber],
152 | [DataMapping],
153 | [DestinationRawFileSystem],
154 | [DestinationRawFolder],
155 | [DestinationRawFile],
156 | [RunSequence],
157 | [MaxRetries],
158 | [ActiveFlag],
159 | [L1TransformationReqdFlag],
160 | [L2TransformationReqdFlag],
161 | [DelayL1TransformationFlag],
162 | [DelayL2TransformationFlag],
163 | [CreatedBy],
164 | [CreatedTimestamp] )
165 | VALUES (src.[SourceSystemName],
166 | src.[StreamName],
167 | src.[SourceSystemDescription],
168 | src.[Backend],
169 | src.[EntityName],
170 | src.[WatermarkColName],
171 | src.[LastDeltaDate],
172 | src.[LastDeltaNumber],
173 | src.[LastDeltaString],
174 | src.[MaxIntervalMinutes],
175 | src.[MaxIntervalNumber],
176 | src.[DataMapping],
177 | src.[DestinationRawFileSystem],
178 | src.[DestinationRawFolder],
179 | src.[DestinationRawFile],
180 | src.[RunSequence],
181 | src.[MaxRetries],
182 | src.[ActiveFlag],
183 | src.[L1TransformationReqdFlag],
184 | src.[L2TransformationReqdFlag],
185 | src.[DelayL1TransformationFlag],
186 | src.[DelayL2TransformationFlag],
187 | USER_NAME(),
188 | GETDATE());
--------------------------------------------------------------------------------
/elt-framework/ControlDB/Post Deployment Script/datasources/RestAPI/IngestDefinition/PurviewRestAPI.sql:
--------------------------------------------------------------------------------
1 | /*
2 | ELT Configuration for REST API Data Sources
3 | */
4 |
5 | IF OBJECT_ID('tempdb..#PurviewRestAPI_Ingest') IS NOT NULL DROP TABLE #PurviewRestAPI_Ingest;
6 |
7 | --Create Temp table with same structure as IngestDefinition
8 | CREATE TABLE #PurviewRestAPI_Ingest
9 | (
10 | [SourceSystemName] [varchar](50) NOT NULL,
11 | [StreamName] [varchar](100) NULL,
12 | [SourceSystemDescription] [varchar](200) NULL,
13 | [Backend] [varchar](30) NULL,
14 | [EntityName] [varchar](100) NULL,
15 | [WatermarkColName] [varchar](50) NULL,
16 | [LastDeltaDate] [datetime2](7) NULL,
17 | [LastDeltaNumber] [int] NULL,
18 | [LastDeltaString] [varchar](50) NULL,
19 | [MaxIntervalMinutes] [int] NULL,
20 | [MaxIntervalNumber] [int] NULL,
21 | [DataMapping] [varchar](max) NULL,
22 | [DestinationRawFileSystem] [varchar](50) NULL,
23 | [DestinationRawFolder] [varchar](200) NULL,
24 | [DestinationRawFile] [varchar](200) NULL,
25 | [RunSequence] [int] NULL,
26 | [MaxRetries] [int] NULL,
27 | [ActiveFlag] [bit] NOT NULL,
28 | [L1TransformationReqdFlag] [bit] NOT NULL,
29 | [L2TransformationReqdFlag] [bit] NOT NULL,
30 | [DelayL1TransformationFlag] [bit] NOT NULL,
31 | [DelayL2TransformationFlag] [bit] NOT NULL
32 | );
33 |
34 | --Insert Into Temp Table
35 | INSERT INTO #PurviewRestAPI_Ingest
36 | --datasources
37 | SELECT 'Purview' AS [SourceSystemName],
38 | 'datasources' AS [StreamName],
39 | 'Microsoft Purview APIs' AS [SourceSystemDescription],
40 | 'ATLAS REST API' AS [Backend],
41 | '/scan/datasources?api-version=2022-02-01-preview' AS [EntityName],
42 | NULL AS [WatermarkColName],
43 | NULL AS [LastDeltaDate],
44 | NULL AS [LastDeltaNumber],
45 | NULL AS [LastDeltaString],
46 | NULL AS [MaxIntervalMinutes],
47 | NULL AS [MaxIntervalNumber],
48 | '{
49 | "type": "TabularTranslator",
50 | "mappings": [
51 | {
52 | "source": {
53 | "path": "[''properties''][''endpoint'']"
54 | },
55 | "sink": {
56 | "name": "endpoint",
57 | "type": "String"
58 | }
59 | },
60 | {
61 | "source": {
62 | "path": "[''properties''][''resourceGroup'']"
63 | },
64 | "sink": {
65 | "name": "resourceGroup",
66 | "type": "String"
67 | }
68 | },
69 | {
70 | "source": {
71 | "path": "[''properties''][''subscriptionId'']"
72 | },
73 | "sink": {
74 | "name": "subscriptionId",
75 | "type": "String"
76 | }
77 | },
78 | {
79 | "source": {
80 | "path": "[''properties''][''location'']"
81 | },
82 | "sink": {
83 | "name": "location",
84 | "type": "String"
85 | }
86 | },
87 | {
88 | "source": {
89 | "path": "[''properties''][''resourceName'']"
90 | },
91 | "sink": {
92 | "name": "resourceName",
93 | "type": "String"
94 | }
95 | },
96 | {
97 | "source": {
98 | "path": "[''properties''][''resourceId'']"
99 | },
100 | "sink": {
101 | "name": "resourceId",
102 | "type": "String"
103 | }
104 | },
105 | {
106 | "source": {
107 | "path": "[''properties''][''dataUseGovernance'']"
108 | },
109 | "sink": {
110 | "name": "dataUseGovernance",
111 | "type": "String"
112 | }
113 | },
114 | {
115 | "source": {
116 | "path": "[''properties''][''createdAt'']"
117 | },
118 | "sink": {
119 | "name": "createdAt",
120 | "type": "String"
121 | }
122 | },
123 | {
124 | "source": {
125 | "path": "[''properties''][''lastModifiedAt'']"
126 | },
127 | "sink": {
128 | "name": "lastModifiedAt",
129 | "type": "String"
130 | }
131 | },
132 | {
133 | "source": {
134 | "path": "[''properties''][''parentCollection'']"
135 | },
136 | "sink": {
137 | "name": "parentCollection",
138 | "type": "String"
139 | }
140 | },
141 | {
142 | "source": {
143 | "path": "[''properties''][''collection''][''lastModifiedAt'']"
144 | },
145 | "sink": {
146 | "name": "Collection_lastModifiedAt",
147 | "type": "String"
148 | }
149 | },
150 | {
151 | "source": {
152 | "path": "[''properties''][''collection''][''referenceName'']"
153 | },
154 | "sink": {
155 | "name": "referenceName",
156 | "type": "String"
157 | }
158 | },
159 | {
160 | "source": {
161 | "path": "[''properties''][''collection''][''type'']"
162 | },
163 | "sink": {
164 | "name": "type",
165 | "type": "String"
166 | }
167 | },
168 | {
169 | "source": {
170 | "path": "[''properties''][''dataSourceCollectionMovingState'']"
171 | },
172 | "sink": {
173 | "name": "dataSourceCollectionMovingState",
174 | "type": "String"
175 | }
176 | },
177 | {
178 | "source": {
179 | "path": "[''kind'']"
180 | },
181 | "sink": {
182 | "name": "kind",
183 | "type": "String"
184 | }
185 | },
186 | {
187 | "source": {
188 | "path": "[''id'']"
189 | },
190 | "sink": {
191 | "name": "id",
192 | "type": "String"
193 | }
194 | },
195 | {
196 | "source": {
197 | "path": "[''name'']"
198 | },
199 | "sink": {
200 | "name": "name",
201 | "type": "String"
202 | }
203 | },
204 | {
205 | "source": {
206 | "path": "$[''count'']"
207 | },
208 | "sink": {
209 | "name": "count",
210 | "type": "String"
211 | }
212 | }
213 | ],
214 | "collectionReference": "$[''value'']",
215 | "mapComplexValuesToString": true
216 | }'AS [DataMapping],
217 | 'raw-bronze' AS [DestinationRawFileSystem],
218 | 'purview/datasources/YYYY-MM' AS [DestinationRawFolder] ,
219 | 'datasources_YYYYMMDD.parquet' AS [DestinationRawFile],
220 | 1 AS [RunSequence],
221 | 3 AS [MaxRetries],
222 | 1 AS [ActiveFlag],
223 | 1 AS [L1TransformationReqdFlag],
224 | 1 AS [L2TransformationReqdFlag],
225 | 1 AS [DelayL1TransformationFlag],
226 | 1 AS [DelayL2TransformationFlag]
227 |
228 | UNION
229 | --Collections
230 | SELECT 'Purview' AS [SourceSystemName],
231 | 'collections' AS [StreamName],
232 | 'Microsoft Purview APIs' AS [SourceSystemDescription],
233 | 'ATLAS REST API' AS [Backend],
234 | '/collections?api-version=2019-11-01-preview' AS [EntityName],
235 | NULL AS [WatermarkColName],
236 | NULL AS [LastDeltaDate],
237 | NULL AS [LastDeltaNumber],
238 | NULL AS [LastDeltaString],
239 | NULL AS [MaxIntervalMinutes],
240 | NULL AS [MaxIntervalNumber],
241 | '{
242 | "type": "TabularTranslator",
243 | "mappings": [
244 | {
245 | "source": {
246 | "path": "[''name'']"
247 | },
248 | "sink": {
249 | "name": "name",
250 | "type": "String"
251 | }
252 | },
253 | {
254 | "source": {
255 | "path": "[''friendlyName'']"
256 | },
257 | "sink": {
258 | "name": "friendlyName",
259 | "type": "String"
260 | }
261 | },
262 | {
263 | "source": {
264 | "path": "[''description'']"
265 | },
266 | "sink": {
267 | "name": "description",
268 | "type": "String"
269 | }
270 | },
271 | {
272 | "source": {
273 | "path": "[''systemData''][''createdBy'']"
274 | },
275 | "sink": {
276 | "name": "createdBy",
277 | "type": "String"
278 | }
279 | },
280 | {
281 | "source": {
282 | "path": "[''systemData''][''createdByType'']"
283 | },
284 | "sink": {
285 | "name": "createdByType",
286 | "type": "String"
287 | }
288 | },
289 | {
290 | "source": {
291 | "path": "[''systemData''][''createdAt'']"
292 | },
293 | "sink": {
294 | "name": "createdAt",
295 | "type": "DateTime"
296 | }
297 | },
298 | {
299 | "source": {
300 | "path": "[''systemData''][''lastModifiedByType'']"
301 | },
302 | "sink": {
303 | "name": "lastModifiedByType",
304 | "type": "String"
305 | }
306 | },
307 | {
308 | "source": {
309 | "path": "[''systemData''][''lastModifiedAt'']"
310 | },
311 | "sink": {
312 | "name": "lastModifiedAt",
313 | "type": "DateTime"
314 | }
315 | },
316 | {
317 | "source": {
318 | "path": "[''collectionProvisioningState'']"
319 | },
320 | "sink": {
321 | "name": "collectionProvisioningState",
322 | "type": "String"
323 | }
324 | },
325 | {
326 | "source": {
327 | "path": "[''parentCollection''][''type'']"
328 | },
329 | "sink": {
330 | "name": "parentCollectionType",
331 | "type": "String"
332 | }
333 | },
334 | {
335 | "source": {
336 | "path": "[''parentCollection''][''referenceName'']"
337 | },
338 | "sink": {
339 | "name": "parentCollectionReferenceName",
340 | "type": "String"
341 | }
342 | }
343 | ],
344 | "collectionReference": "$[''value'']",
345 | "mapComplexValuesToString": true
346 | }' AS [DataMapping],
347 | 'raw-bronze' AS [DestinationRawFileSystem],
348 | 'purview/collections/YYYY-MM' AS [DestinationRawFolder] ,
349 | 'collections_YYYYMMDD.parquet' AS [DestinationRawFile],
350 | 2 AS [RunSequence],
351 | 3 AS [MaxRetries],
352 | 1 AS [ActiveFlag],
353 | 1 AS [L1TransformationReqdFlag],
354 | 1 AS [L2TransformationReqdFlag],
355 | 1 AS [DelayL1TransformationFlag],
356 | 1 AS [DelayL2TransformationFlag]
357 | ;
358 |
359 |
360 | --Merge with Temp table for re-runnability
361 |
362 | MERGE INTO [ELT].[IngestDefinition] AS tgt
363 | USING #PurviewRestAPI_Ingest AS src
364 | ON src.[SourceSystemName]=tgt.[SourceSystemName] AND src.[StreamName] =tgt.[StreamName]
365 | WHEN MATCHED THEN
366 | UPDATE SET tgt.[SourceSystemDescription] = src.[SourceSystemDescription],
367 | tgt.[Backend] = src.[Backend],
368 | tgt.[EntityName] = src.[EntityName],
369 | tgt.[WatermarkColName] = src.[WatermarkColName],
370 | tgt.[LastDeltaDate] = src.[LastDeltaDate],
371 | tgt.[LastDeltaNumber] = src.[LastDeltaNumber],
372 | tgt.[LastDeltaString] = src.[LastDeltaString],
373 | tgt.[MaxIntervalMinutes] = src.[MaxIntervalMinutes],
374 | tgt.[MaxIntervalNumber] = src.[MaxIntervalNumber],
375 | tgt.[DataMapping] = src.[DataMapping],
376 | tgt.[DestinationRawFileSystem] = src.[DestinationRawFileSystem],
377 | tgt.[DestinationRawFolder] = src.[DestinationRawFolder],
378 | tgt.[DestinationRawFile] = src.[DestinationRawFile],
379 | tgt.[RunSequence] = src.[RunSequence],
380 | tgt.[MaxRetries] = src.[MaxRetries],
381 | tgt.[ActiveFlag] = src.[ActiveFlag],
382 | tgt.[L1TransformationReqdFlag] = src.[L1TransformationReqdFlag],
383 | tgt.[L2TransformationReqdFlag] = src.[L2TransformationReqdFlag],
384 | tgt.[DelayL1TransformationFlag] = src.[DelayL1TransformationFlag],
385 | tgt.[DelayL2TransformationFlag] = src.[DelayL2TransformationFlag],
386 | tgt.[ModifiedBy] = USER_NAME(),
387 | tgt.[ModifiedTimestamp] = GetDate()
388 | WHEN NOT MATCHED BY TARGET THEN
389 | INSERT([SourceSystemName],
390 | [StreamName],
391 | [SourceSystemDescription],
392 | [Backend],
393 | [EntityName],
394 | [WatermarkColName],
395 | [LastDeltaDate],
396 | [LastDeltaNumber],
397 | [LastDeltaString],
398 | [MaxIntervalMinutes],
399 | [MaxIntervalNumber],
400 | [DataMapping],
401 | [DestinationRawFileSystem],
402 | [DestinationRawFolder],
403 | [DestinationRawFile],
404 | [RunSequence],
405 | [MaxRetries],
406 | [ActiveFlag],
407 | [L1TransformationReqdFlag],
408 | [L2TransformationReqdFlag],
409 | [DelayL1TransformationFlag],
410 | [DelayL2TransformationFlag],
411 | [CreatedBy],
412 | [CreatedTimestamp] )
413 | VALUES (src.[SourceSystemName],
414 | src.[StreamName],
415 | src.[SourceSystemDescription],
416 | src.[Backend],
417 | src.[EntityName],
418 | src.[WatermarkColName],
419 | src.[LastDeltaDate],
420 | src.[LastDeltaNumber],
421 | src.[LastDeltaString],
422 | src.[MaxIntervalMinutes],
423 | src.[MaxIntervalNumber],
424 | src.[DataMapping],
425 | src.[DestinationRawFileSystem],
426 | src.[DestinationRawFolder],
427 | src.[DestinationRawFile],
428 | src.[RunSequence],
429 | src.[MaxRetries],
430 | src.[ActiveFlag],
431 | src.[L1TransformationReqdFlag],
432 | src.[L2TransformationReqdFlag],
433 | src.[DelayL1TransformationFlag],
434 | src.[DelayL2TransformationFlag],
435 | USER_NAME(),
436 | GETDATE());
437 |
438 | GO
--------------------------------------------------------------------------------
/elt-framework/ControlDB/Post Deployment Script/datasources/RestAPI/L1TransformDefinition/L1T_AzureRestAPI.sql:
--------------------------------------------------------------------------------
1 | /*
2 | ELT Configuration for REST API Data Sources
3 | */
4 | Declare @SourceSystem VARCHAR(50), @StreamName VARCHAR(100)
5 | SET @SourceSystem='Azure'
6 | SET @StreamName ='%'
7 |
8 | IF OBJECT_ID('tempdb..#AzureRestAPI_L1') IS NOT NULL DROP TABLE #AzureRestAPI_L1;
9 | --Create Temp table with same structure as L1TransformDefinition
10 | CREATE TABLE #AzureRestAPI_L1
11 | (
12 | [IngestID] int not null,
13 | [ComputePath] varchar(200) null,
14 | [ComputeName] varchar(100) null,
15 | [CustomParameters] varchar(max) null,
16 | [InputRawFileSystem] varchar(50) not null,
17 | [InputRawFileFolder] varchar(200) not null,
18 | [InputRawFile] varchar(200) not null,
19 | [InputRawFileDelimiter] char(1) null,
20 | [InputFileHeaderFlag] bit null,
21 | [OutputL1CurateFileSystem] varchar(50) not null,
22 | [OutputL1CuratedFolder] varchar(200) not null,
23 | [OutputL1CuratedFile] varchar(200) not null,
24 | [OutputL1CuratedFileDelimiter] char(1) null,
25 | [OutputL1CuratedFileFormat] varchar(10) null,
26 | [OutputL1CuratedFileWriteMode] varchar(20) null,
27 | [OutputDWStagingTable] varchar(200) null,
28 | [LookupColumns] varchar(4000) null,
29 | [OutputDWTable] varchar(200) null,
30 | [OutputDWTableWriteMode] varchar(20) null,
31 | [MaxRetries] int null,
32 | [WatermarkColName] varchar(50) null,
33 | [ActiveFlag] bit not null
34 | );
35 |
36 | --Insert Into Temp Table
37 | INSERT INTO #AzureRestAPI_L1
38 | SELECT [IngestID]
39 | ,'L1Transform' AS [ComputePath]
40 | ,'L1Transform-Generic-Synapse' AS [ComputeName]
41 | , NULL AS [CustomParameters]
42 | ,[DestinationRawFileSystem] AS [InputRawFileSystem]
43 | ,[DestinationRawFolder] AS [InputRawFileFolder]
44 | ,[DestinationRawFile] AS [InputRawFile]
45 | , NULL AS [InputRawFileDelimiter]
46 | , 1 AS [InputFileHeaderFlag]
47 | , 'curated-silver' AS [OutputL1CurateFileSystem]
48 | , [DestinationRawFolder] AS [OutputL1CuratedFolder]
49 | , 'standardized_'+ [DestinationRawFile] AS [OutputL1CuratedFile]
50 | , NULL AS [OutputL1CuratedFileDelimiter]
51 | , 'parquet' AS [OutputL1CuratedFileFormat]
52 | , 'overwrite' AS [OutputL1CuratedFileWriteMode]
53 | , 'stg.merge_' + [SourceSystemName] +'_'+ StreamName AS [OutputDWStagingTable]
54 | , CASE WHEN StreamName = 'operations' THEN '[''name'']'
55 | ELSE NULL
56 | END AS [LookupColumns]
57 | , SourceSystemName + '.' + StreamName AS [OutputDWTable]
58 | , 'append' AS [OutputDWTableWriteMode]
59 | ,3 AS [MaxRetries]
60 | , NULL AS [WatermarkColName]
61 | , 1 AS [ActiveFlag]
62 | FROM [ELT].[IngestDefinition]
63 | WHERE [SourceSystemName]=@SourceSystem
64 | AND [StreamName] like @StreamName;
65 |
66 |
67 | --Merge with Temp table for re-runnability
68 |
69 | MERGE INTO [ELT].[L1TransformDefinition] AS tgt
70 | USING #AzureRestAPI_L1 AS src
71 | ON src.[InputRawFileSystem] = tgt.[InputRawFileSystem]
72 | AND src.[InputRawFileFolder] = tgt.[InputRawFileFolder]
73 | AND src.[InputRawFile] = tgt.[InputRawFile]
74 | AND src.[OutputL1CurateFileSystem] = tgt.[OutputL1CurateFileSystem]
75 | AND src.[OutputL1CuratedFolder] = tgt.[OutputL1CuratedFolder]
76 | AND src.[OutputL1CuratedFile] = tgt.[OutputL1CuratedFile]
77 | WHEN MATCHED THEN
78 | UPDATE SET tgt.[IngestID] =src.[IngestID],
79 | tgt.[ComputePath] =src.[ComputePath],
80 | tgt.[ComputeName] =src.[ComputeName],
81 | tgt.[CustomParameters] =src.[CustomParameters],
82 | tgt.[InputRawFileSystem] =src.[InputRawFileSystem],
83 | tgt.[InputRawFileFolder] =src.[InputRawFileFolder],
84 | tgt.[InputRawFile] =src.[InputRawFile],
85 | tgt.[InputRawFileDelimiter] =src.[InputRawFileDelimiter],
86 | tgt.[InputFileHeaderFlag] =src.[InputFileHeaderFlag],
87 | tgt.[OutputL1CurateFileSystem] =src.[OutputL1CurateFileSystem],
88 | tgt.[OutputL1CuratedFolder] =src.[OutputL1CuratedFolder],
89 | tgt.[OutputL1CuratedFile] =src.[OutputL1CuratedFile],
90 | tgt.[OutputL1CuratedFileDelimiter] =src.[OutputL1CuratedFileDelimiter],
91 | tgt.[OutputL1CuratedFileFormat] =src.[OutputL1CuratedFileFormat],
92 | tgt.[OutputL1CuratedFileWriteMode] =src.[OutputL1CuratedFileWriteMode],
93 | tgt.[OutputDWStagingTable] =src.[OutputDWStagingTable],
94 | tgt.[LookupColumns] =src.[LookupColumns],
95 | tgt.[OutputDWTable] =src.[OutputDWTable],
96 | tgt.[OutputDWTableWriteMode] =src.[OutputDWTableWriteMode],
97 | tgt.[MaxRetries] =src.[MaxRetries],
98 | tgt.[WatermarkColName] =src.[WatermarkColName],
99 | tgt.[ActiveFlag] =src.[ActiveFlag],
100 | tgt.[ModifiedBy] = USER_NAME(),
101 | tgt.[ModifiedTimestamp] = GetDate()
102 | WHEN NOT MATCHED BY TARGET THEN
103 | INSERT([IngestID],
104 | [ComputePath],
105 | [ComputeName],
106 | [CustomParameters],
107 | [InputRawFileSystem],
108 | [InputRawFileFolder],
109 | [InputRawFile],
110 | [InputRawFileDelimiter],
111 | [InputFileHeaderFlag],
112 | [OutputL1CurateFileSystem],
113 | [OutputL1CuratedFolder],
114 | [OutputL1CuratedFile],
115 | [OutputL1CuratedFileDelimiter],
116 | [OutputL1CuratedFileFormat],
117 | [OutputL1CuratedFileWriteMode],
118 | [OutputDWStagingTable],
119 | [LookupColumns],
120 | [OutputDWTable],
121 | [OutputDWTableWriteMode],
122 | [MaxRetries],
123 | [WatermarkColName],
124 | [ActiveFlag],
125 | [CreatedBy],
126 | [CreatedTimestamp] )
127 | VALUES (src.[IngestID],
128 | src.[ComputePath],
129 | src.[ComputeName],
130 | src.[CustomParameters],
131 | src.[InputRawFileSystem],
132 | src.[InputRawFileFolder],
133 | src.[InputRawFile],
134 | src.[InputRawFileDelimiter],
135 | src.[InputFileHeaderFlag],
136 | src.[OutputL1CurateFileSystem],
137 | src.[OutputL1CuratedFolder],
138 | src.[OutputL1CuratedFile],
139 | src.[OutputL1CuratedFileDelimiter],
140 | src.[OutputL1CuratedFileFormat],
141 | src.[OutputL1CuratedFileWriteMode],
142 | src.[OutputDWStagingTable],
143 | src.[LookupColumns],
144 | src.[OutputDWTable],
145 | src.[OutputDWTableWriteMode],
146 | src.[MaxRetries],
147 | src.[WatermarkColName],
148 | src.[ActiveFlag],
149 | USER_NAME(),
150 | GETDATE());
151 |
152 | GO
--------------------------------------------------------------------------------
/elt-framework/ControlDB/Post Deployment Script/datasources/RestAPI/L1TransformDefinition/L1T_PurviewRestAPI.sql:
--------------------------------------------------------------------------------
1 | /*
2 | ELT Configuration for REST API Data Sources
3 | */
4 | Declare @SourceSystem VARCHAR(50), @StreamName VARCHAR(100)
5 | SET @SourceSystem='Purview'
6 | SET @StreamName ='%'
7 |
8 | IF OBJECT_ID('tempdb..#PurviewRestAPI_L1') IS NOT NULL DROP TABLE #PurviewRestAPI_L1;
9 | --Create Temp table with same structure as L1TransformDefinition
10 | CREATE TABLE #PurviewRestAPI_L1
11 | (
12 | [IngestID] int not null,
13 | [ComputePath] varchar(200) null,
14 | [ComputeName] varchar(100) null,
15 | [CustomParameters] varchar(max) null,
16 | [InputRawFileSystem] varchar(50) not null,
17 | [InputRawFileFolder] varchar(200) not null,
18 | [InputRawFile] varchar(200) not null,
19 | [InputRawFileDelimiter] char(1) null,
20 | [InputFileHeaderFlag] bit null,
21 | [OutputL1CurateFileSystem] varchar(50) not null,
22 | [OutputL1CuratedFolder] varchar(200) not null,
23 | [OutputL1CuratedFile] varchar(200) not null,
24 | [OutputL1CuratedFileDelimiter] char(1) null,
25 | [OutputL1CuratedFileFormat] varchar(10) null,
26 | [OutputL1CuratedFileWriteMode] varchar(20) null,
27 | [OutputDWStagingTable] varchar(200) null,
28 | [LookupColumns] varchar(4000) null,
29 | [OutputDWTable] varchar(200) null,
30 | [OutputDWTableWriteMode] varchar(20) null,
31 | [MaxRetries] int null,
32 | [WatermarkColName] varchar(50) null,
33 | [ActiveFlag] bit not null
34 | );
35 |
36 | --Insert Into Temp Table
37 | INSERT INTO #PurviewRestAPI_L1
38 | SELECT [IngestID]
39 | ,'L1Transform' AS [ComputePath]
40 | ,'L1Transform-Generic-Synapse' AS [ComputeName]
41 | , NULL AS [CustomParameters]
42 | ,[DestinationRawFileSystem] AS [InputRawFileSystem]
43 | ,[DestinationRawFolder] AS [InputRawFileFolder]
44 | ,[DestinationRawFile] AS [InputRawFile]
45 | , NULL AS [InputRawFileDelimiter]
46 | , 1 AS [InputFileHeaderFlag]
47 | , 'curated-silver' AS [OutputL1CurateFileSystem]
48 | , [DestinationRawFolder] AS [OutputL1CuratedFolder]
49 | , 'standardized_'+ [DestinationRawFile] AS [OutputL1CuratedFile]
50 | , NULL AS [OutputL1CuratedFileDelimiter]
51 | , 'parquet' AS [OutputL1CuratedFileFormat]
52 | , 'overwrite' AS [OutputL1CuratedFileWriteMode]
53 | , 'stg.merge_' + [SourceSystemName] +'_'+ StreamName AS [OutputDWStagingTable]
54 | , CASE WHEN StreamName = 'datasources' THEN '[''resourceName'']'
55 | WHEN StreamName = 'collections' THEN '[''name'']'
56 | END AS [LookupColumns]
57 | , SourceSystemName + '.' + StreamName AS [OutputDWTable]
58 | , 'append' AS [OutputDWTableWriteMode]
59 | ,3 AS [MaxRetries]
60 | , CASE WHEN StreamName = 'datasources' THEN 'lastModifiedAt'
61 | WHEN StreamName = 'collections' THEN 'lastModifiedAt'
62 | END AS [WatermarkColName]
63 | , 1 AS [ActiveFlag]
64 | FROM [ELT].[IngestDefinition]
65 | WHERE [SourceSystemName]=@SourceSystem
66 | AND [StreamName] like @StreamName;
67 |
68 |
69 | --Merge with Temp table for re-runnability
70 |
71 | MERGE INTO [ELT].[L1TransformDefinition] AS tgt
72 | USING #PurviewRestAPI_L1 AS src
73 | ON src.[InputRawFileSystem] = tgt.[InputRawFileSystem]
74 | AND src.[InputRawFileFolder] = tgt.[InputRawFileFolder]
75 | AND src.[InputRawFile] = tgt.[InputRawFile]
76 | AND src.[OutputL1CurateFileSystem] = tgt.[OutputL1CurateFileSystem]
77 | AND src.[OutputL1CuratedFolder] = tgt.[OutputL1CuratedFolder]
78 | AND src.[OutputL1CuratedFile] = tgt.[OutputL1CuratedFile]
79 | WHEN MATCHED THEN
80 | UPDATE SET tgt.[IngestID] =src.[IngestID],
81 | tgt.[ComputePath] =src.[ComputePath],
82 | tgt.[ComputeName] =src.[ComputeName],
83 | tgt.[CustomParameters] =src.[CustomParameters],
84 | tgt.[InputRawFileSystem] =src.[InputRawFileSystem],
85 | tgt.[InputRawFileFolder] =src.[InputRawFileFolder],
86 | tgt.[InputRawFile] =src.[InputRawFile],
87 | tgt.[InputRawFileDelimiter] =src.[InputRawFileDelimiter],
88 | tgt.[InputFileHeaderFlag] =src.[InputFileHeaderFlag],
89 | tgt.[OutputL1CurateFileSystem] =src.[OutputL1CurateFileSystem],
90 | tgt.[OutputL1CuratedFolder] =src.[OutputL1CuratedFolder],
91 | tgt.[OutputL1CuratedFile] =src.[OutputL1CuratedFile],
92 | tgt.[OutputL1CuratedFileDelimiter] =src.[OutputL1CuratedFileDelimiter],
93 | tgt.[OutputL1CuratedFileFormat] =src.[OutputL1CuratedFileFormat],
94 | tgt.[OutputL1CuratedFileWriteMode] =src.[OutputL1CuratedFileWriteMode],
95 | tgt.[OutputDWStagingTable] =src.[OutputDWStagingTable],
96 | tgt.[LookupColumns] =src.[LookupColumns],
97 | tgt.[OutputDWTable] =src.[OutputDWTable],
98 | tgt.[OutputDWTableWriteMode] =src.[OutputDWTableWriteMode],
99 | tgt.[MaxRetries] =src.[MaxRetries],
100 | tgt.[WatermarkColName] =src.[WatermarkColName],
101 | tgt.[ActiveFlag] =src.[ActiveFlag],
102 | tgt.[ModifiedBy] = USER_NAME(),
103 | tgt.[ModifiedTimestamp] = GetDate()
104 | WHEN NOT MATCHED BY TARGET THEN
105 | INSERT([IngestID],
106 | [ComputePath],
107 | [ComputeName],
108 | [CustomParameters],
109 | [InputRawFileSystem],
110 | [InputRawFileFolder],
111 | [InputRawFile],
112 | [InputRawFileDelimiter],
113 | [InputFileHeaderFlag],
114 | [OutputL1CurateFileSystem],
115 | [OutputL1CuratedFolder],
116 | [OutputL1CuratedFile],
117 | [OutputL1CuratedFileDelimiter],
118 | [OutputL1CuratedFileFormat],
119 | [OutputL1CuratedFileWriteMode],
120 | [OutputDWStagingTable],
121 | [LookupColumns],
122 | [OutputDWTable],
123 | [OutputDWTableWriteMode],
124 | [MaxRetries],
125 | [WatermarkColName],
126 | [ActiveFlag],
127 | [CreatedBy],
128 | [CreatedTimestamp] )
129 | VALUES (src.[IngestID],
130 | src.[ComputePath],
131 | src.[ComputeName],
132 | src.[CustomParameters],
133 | src.[InputRawFileSystem],
134 | src.[InputRawFileFolder],
135 | src.[InputRawFile],
136 | src.[InputRawFileDelimiter],
137 | src.[InputFileHeaderFlag],
138 | src.[OutputL1CurateFileSystem],
139 | src.[OutputL1CuratedFolder],
140 | src.[OutputL1CuratedFile],
141 | src.[OutputL1CuratedFileDelimiter],
142 | src.[OutputL1CuratedFileFormat],
143 | src.[OutputL1CuratedFileWriteMode],
144 | src.[OutputDWStagingTable],
145 | src.[LookupColumns],
146 | src.[OutputDWTable],
147 | src.[OutputDWTableWriteMode],
148 | src.[MaxRetries],
149 | src.[WatermarkColName],
150 | src.[ActiveFlag],
151 | USER_NAME(),
152 | GETDATE());
153 |
154 | GO
--------------------------------------------------------------------------------
/elt-framework/ControlDB/Post Deployment Script/datasources/UploadFiles/IngestDefinition/CustomFormRecognizerModel.sql:
--------------------------------------------------------------------------------
1 | /*
2 | ELT Configuration for CustomFormRecognizerModel JSON Data Source
3 | */
4 |
5 | IF OBJECT_ID('tempdb..#CFRM_Ingest') IS NOT NULL DROP TABLE #CFRM_Ingest;
6 |
7 | --Create Temp table with similar schema as IngestDefinition
8 | CREATE TABLE #CFRM_Ingest
9 | (
10 | [SourceSystemName] [varchar](50) NOT NULL,
11 | [StreamName] [varchar](100) NULL,
12 | [SourceSystemDescription] [varchar](200) NULL,
13 | [Backend] [varchar](30) NULL,
14 | [DataFormat] [varchar](10) NULL,
15 | [SourceFileDropFileSystem] [varchar](50) NULL,
16 | [SourceFileDropFolder] [varchar](200) NULL,
17 | [SourceFileDropFile] [varchar](200) NULL,
18 | [DestinationRawFileSystem] [varchar](50) NULL,
19 | [DestinationRawFolder] [varchar](200) NULL,
20 | [DestinationRawFile] [varchar](200) NULL,
21 | [ActiveFlag] [bit] NOT NULL,
22 | [L1TransformationReqdFlag] [bit] NOT NULL,
23 | [L2TransformationReqdFlag] [bit] NOT NULL,
24 | [DelayL1TransformationFlag] [bit] NOT NULL,
25 | [DelayL2TransformationFlag] [bit] NOT NULL
26 | );
27 |
28 |
29 | --Insert Into Temp Table
30 | INSERT INTO #CFRM_Ingest
31 | --Operations
32 | SELECT 'AIML-OCR' AS [SourceSystemName],
33 | 'restatements' AS [StreamName],
34 | 'Custom Document Intelligence Model output for Real Estate Statements' AS [SourceSystemDescription],
35 | 'Azure AI DocI REST API' AS [Backend],
36 | 'JSON' AS [DataFormat],
37 | 'doci' AS [SourceFileDropFileSystem],
38 | 'inference/re-statements' AS [SourceFileDropFolder],
39 | '*.json' AS [SourceFileDropFile],
40 | 'raw-bronze' AS [DestinationRawFileSystem],
41 | 're-statements/YYYY/MM' AS [DestinationRawFolder],
42 | 'YYYY-MM-DD_HHMISS_re-statement.json' AS [DestinationRawFile],
43 | 1 AS [ActiveFlag],
44 | 1 AS [L1TransformationReqdFlag],
45 | 0 AS [L2TransformationReqdFlag],
46 | 0 AS [DelayL1TransformationFlag],
47 | 1 AS [DelayL2TransformationFlag]
48 | UNION
49 | SELECT 'AIML-OCR' AS [SourceSystemName],
50 | 'form10q' AS [StreamName],
51 | 'Custom Document Intelligence Model output for Form 10-Q' AS [SourceSystemDescription],
52 | 'Azure AI DocI REST API' AS [Backend],
53 | 'JSON' AS [DataFormat],
54 | 'doci' AS [SourceFileDropFileSystem],
55 | 'inference/form10q' AS [SourceFileDropFolder],
56 | '*.json' AS [SourceFileDropFile],
57 | 'raw-bronze' AS [DestinationRawFileSystem],
58 | 'form10q/YYYY/MM' AS [DestinationRawFolder],
59 | 'YYYY-MM-DD_HHMISS_form10q.json' AS [DestinationRawFile],
60 | 1 AS [ActiveFlag],
61 | 1 AS [L1TransformationReqdFlag],
62 | 0 AS [L2TransformationReqdFlag],
63 | 0 AS [DelayL1TransformationFlag],
64 | 1 AS [DelayL2TransformationFlag]
65 |
66 | --Merge with Temp table for re-runnability
67 |
68 | MERGE INTO [ELT].[IngestDefinition] AS tgt
69 | USING #CFRM_Ingest AS src
70 | ON src.[SourceSystemName]=tgt.[SourceSystemName] AND src.[StreamName]=tgt.[StreamName]
71 | WHEN MATCHED THEN
72 | UPDATE SET tgt.[SourceSystemDescription] = src.[SourceSystemDescription],
73 | tgt.[Backend] = src.[Backend],
74 | tgt.[DataFormat] = src.[DataFormat],
75 | tgt.[SourceFileDropFileSystem] = src.[SourceFileDropFileSystem],
76 | tgt.[SourceFileDropFolder] =src.[SourceFileDropFolder],
77 | tgt.[SourceFileDropFile] = src.[SourceFileDropFile],
78 | tgt.[DestinationRawFileSystem] = src.[DestinationRawFileSystem],
79 | tgt.[DestinationRawFolder] = src.[DestinationRawFolder],
80 | tgt.[DestinationRawFile] = src.[DestinationRawFile],
81 | tgt.[ActiveFlag] = src.[ActiveFlag],
82 | tgt.[L1TransformationReqdFlag] = src.[L1TransformationReqdFlag],
83 | tgt.[L2TransformationReqdFlag] = src.[L2TransformationReqdFlag],
84 | tgt.[DelayL1TransformationFlag] = src.[DelayL1TransformationFlag],
85 | tgt.[DelayL2TransformationFlag] = src.[DelayL2TransformationFlag],
86 | tgt.[ModifiedBy] = USER_NAME(),
87 | tgt.[ModifiedTimestamp] = GetDate()
88 | WHEN NOT MATCHED BY TARGET THEN
89 | INSERT( [SourceSystemName],
90 | [StreamName],
91 | [SourceSystemDescription],
92 | [Backend],
93 | [DataFormat],
94 | [SourceFileDropFileSystem],
95 | [SourceFileDropFolder],
96 | [SourceFileDropFile],
97 | [DestinationRawFileSystem],
98 | [DestinationRawFolder],
99 | [DestinationRawFile],
100 | [ActiveFlag],
101 | [L1TransformationReqdFlag],
102 | [L2TransformationReqdFlag],
103 | [DelayL1TransformationFlag],
104 | [DelayL2TransformationFlag],
105 | [ModifiedBy],
106 | [ModifiedTimestamp])
107 | VALUES (src.[SourceSystemName],
108 | src.[StreamName],
109 | src.[SourceSystemDescription],
110 | src.[Backend],
111 | src.[DataFormat],
112 | src.[SourceFileDropFileSystem],
113 | src.[SourceFileDropFolder],
114 | src.[SourceFileDropFile],
115 | src.[DestinationRawFileSystem],
116 | src.[DestinationRawFolder],
117 | src.[DestinationRawFile],
118 | src.[ActiveFlag],
119 | src.[L1TransformationReqdFlag],
120 | src.[L2TransformationReqdFlag],
121 | src.[DelayL1TransformationFlag],
122 | src.[DelayL2TransformationFlag],
123 | USER_NAME(),
124 | GETDATE());
--------------------------------------------------------------------------------
/elt-framework/ControlDB/Post Deployment Script/datasources/UploadFiles/L1TransformDefinition/L1T_CustomFormRecognizerModel.sql:
--------------------------------------------------------------------------------
1 | /*
2 | ELT Configuration for CustomFormRecognizerModel JSON Data Source
3 | */
4 | Declare @SourceSystem VARCHAR(50), @StreamName VARCHAR(100)
5 | SET @SourceSystem='AIML-OCR'
6 | SET @StreamName ='%'
7 |
8 | IF OBJECT_ID('tempdb..#CFRM_L1') IS NOT NULL DROP TABLE #CFRM_L1;
9 | --Create Temp table with same structure as L1TransformDefinition
10 | CREATE TABLE #CFRM_L1
11 | (
12 | [IngestID] int not null,
13 | [ComputePath] varchar(200) null,
14 | [ComputeName] varchar(100) null,
15 | [InputRawFileSystem] varchar(50) not null,
16 | [InputRawFileFolder] varchar(200) not null,
17 | [InputRawFile] varchar(200) not null,
18 | [OutputL1CurateFileSystem] varchar(50) not null,
19 | [OutputL1CuratedFolder] varchar(200) not null,
20 | [OutputL1CuratedFile] varchar(200) not null,
21 | [OutputL1CuratedFileFormat] varchar(10) null,
22 | [OutputL1CuratedFileWriteMode] varchar(20) null,
23 | [OutputDWStagingTable] varchar(200) null,
24 | [LookupColumns] varchar(4000) null,
25 | [OutputDWTable] varchar(200) null,
26 | [OutputDWTableWriteMode] varchar(20) null,
27 | [ActiveFlag] bit not null
28 | );
29 |
30 | --Insert Into Temp Table
31 | INSERT INTO #CFRM_L1
32 | SELECT [IngestID]
33 | ,'L1Transform' AS [ComputePath]
34 | ,'L1Transform-ReStatement' AS [ComputeName]
35 | ,[DestinationRawFileSystem] AS [InputRawFileSystem]
36 | ,[DestinationRawFolder] AS [InputRawFileFolder]
37 | ,[DestinationRawFile] AS [InputRawFile]
38 | , 'curated-silver' AS [OutputL1CurateFileSystem]
39 | , [DestinationRawFolder] AS [OutputL1CuratedFolder]
40 | , 'standardized_'+ [DestinationRawFile] AS [OutputL1CuratedFile]
41 | , 'json' AS [OutputL1CuratedFileFormat]
42 | , 'overwrite' AS [OutputL1CuratedFileWriteMode]
43 | , null as [OutputDWStagingTable]
44 | , null as [LookupColumns]
45 | , null as [OutputDWTable]
46 | , null as [OutputDWTableWriteMode]
47 | , 1 AS [ActiveFlag]
48 | FROM [ELT].[IngestDefinition]
49 | WHERE [SourceSystemName]=@SourceSystem
50 | AND [StreamName] ='restatements'
51 |
52 | UNION
53 | SELECT [IngestID]
54 | ,'L1Transform' AS [ComputePath]
55 | ,'L1Transform-SEC-Form10Q' AS [ComputeName]
56 | ,[DestinationRawFileSystem] AS [InputRawFileSystem]
57 | ,[DestinationRawFolder] AS [InputRawFileFolder]
58 | ,[DestinationRawFile] AS [InputRawFile]
59 | , 'curated-silver' AS [OutputL1CurateFileSystem]
60 | , [DestinationRawFolder] AS [OutputL1CuratedFolder]
61 | , 'standardized_'+ [DestinationRawFile] AS [OutputL1CuratedFile]
62 | , 'json' AS [OutputL1CuratedFileFormat]
63 | , 'overwrite' AS [OutputL1CuratedFileWriteMode]
64 | , '[stg].[merge_sec_form10q]' as [OutputDWStagingTable]
65 | , '[''org_name'',''reporting_quarter'']' as [LookupColumns]
66 | , '[sec].[form10q]' as [OutputDWTable]
67 | , 'append' as [OutputDWTableWriteMode]
68 | , 1 AS [ActiveFlag]
69 | FROM [ELT].[IngestDefinition]
70 | WHERE [SourceSystemName]=@SourceSystem
71 | AND [StreamName] ='form10q'
72 | --Merge with Temp table for re-runnability
73 |
74 | MERGE INTO [ELT].[L1TransformDefinition] AS tgt
75 | USING #CFRM_L1 AS src
76 | ON src.[InputRawFileSystem] = tgt.[InputRawFileSystem]
77 | AND src.[InputRawFileFolder] = tgt.[InputRawFileFolder]
78 | AND src.[InputRawFile] = tgt.[InputRawFile]
79 | AND src.[OutputL1CurateFileSystem] = tgt.[OutputL1CurateFileSystem]
80 | AND src.[OutputL1CuratedFolder] = tgt.[OutputL1CuratedFolder]
81 | AND src.[OutputL1CuratedFile] = tgt.[OutputL1CuratedFile]
82 | WHEN MATCHED THEN
83 | UPDATE SET tgt.[IngestID] =src.[IngestID],
84 | tgt.[ComputePath] =src.[ComputePath],
85 | tgt.[ComputeName] =src.[ComputeName],
86 | tgt.[InputRawFileSystem] =src.[InputRawFileSystem],
87 | tgt.[InputRawFileFolder] =src.[InputRawFileFolder],
88 | tgt.[InputRawFile] =src.[InputRawFile],
89 | tgt.[OutputL1CurateFileSystem] =src.[OutputL1CurateFileSystem],
90 | tgt.[OutputL1CuratedFolder] =src.[OutputL1CuratedFolder],
91 | tgt.[OutputL1CuratedFile] =src.[OutputL1CuratedFile],
92 | tgt.[OutputL1CuratedFileFormat] =src.[OutputL1CuratedFileFormat],
93 | tgt.[OutputL1CuratedFileWriteMode] =src.[OutputL1CuratedFileWriteMode],
94 | tgt.[OutputDWStagingTable] = src.[OutputDWStagingTable],
95 | tgt.[LookupColumns] = src.[LookupColumns],
96 | tgt.[OutputDWTable] = src.[OutputDWTable],
97 | tgt.[OutputDWTableWriteMode]=src.[OutputDWTableWriteMode],
98 | tgt.[ActiveFlag] =src.[ActiveFlag],
99 | tgt.[ModifiedBy] = USER_NAME(),
100 | tgt.[ModifiedTimestamp] = GetDate()
101 | WHEN NOT MATCHED BY TARGET THEN
102 | INSERT([IngestID],
103 | [ComputePath],
104 | [ComputeName],
105 | [InputRawFileSystem],
106 | [InputRawFileFolder],
107 | [InputRawFile],
108 | [OutputL1CurateFileSystem],
109 | [OutputL1CuratedFolder],
110 | [OutputL1CuratedFile],
111 | [OutputL1CuratedFileFormat],
112 | [OutputL1CuratedFileWriteMode],
113 | [ActiveFlag],
114 | [CreatedBy],
115 | [CreatedTimestamp] )
116 | VALUES (src.[IngestID],
117 | src.[ComputePath],
118 | src.[ComputeName],
119 | src.[InputRawFileSystem],
120 | src.[InputRawFileFolder],
121 | src.[InputRawFile],
122 | src.[OutputL1CurateFileSystem],
123 | src.[OutputL1CuratedFolder],
124 | src.[OutputL1CuratedFile],
125 | src.[OutputL1CuratedFileFormat],
126 | src.[OutputL1CuratedFileWriteMode],
127 | src.[ActiveFlag],
128 | USER_NAME(),
129 | GETDATE());
130 |
131 | GO
--------------------------------------------------------------------------------
/elt-framework/ControlDB/Pre Deployment Script/DBRoles.PreDeployment.sql:
--------------------------------------------------------------------------------
1 | /*
2 | Pre-Deployment Script Template
3 | --------------------------------------------------------------------------------------
4 | This file contains SQL statements that will be executed before the build script.
5 | Use SQLCMD syntax to include a file in the pre-deployment script.
6 | Example: :r .\myfile.sql
7 | Use SQLCMD syntax to reference a variable in the pre-deployment script.
8 | Example: :setvar TableName MyTable
9 | SELECT * FROM [$(TableName)]
10 | --------------------------------------------------------------------------------------
11 | */
12 | ----Create database user for Azure Datafactory Managed Identity
13 | --CREATE USER [$(adf_name)] FROM EXTERNAL PROVIDER;
14 |
15 | ----Assign Roles to ADF Managed Identity
16 | --EXEC sp_addrolemember db_datareader, [$(adf_name)]; --Reader
17 | --EXEC sp_addrolemember db_datawriter, [$(adf_name)]; --Writer
18 | --GRANT EXECUTE TO [$(adf_name)]; --Execute Stored Procedures
19 |
--------------------------------------------------------------------------------
/elt-framework/ControlDB/Pre Deployment Script/Script.PreDeployment.sql:
--------------------------------------------------------------------------------
1 | /*
2 | Pre-Deployment Script Template
3 | --------------------------------------------------------------------------------------
4 | This file contains SQL statements that will be executed before the build script.
5 | Use SQLCMD syntax to include a file in the pre-deployment script.
6 | Example: :r .\myfile.sql
7 | Use SQLCMD syntax to reference a variable in the pre-deployment script.
8 | Example: :setvar TableName MyTable
9 | SELECT * FROM [$(TableName)]
10 | --------------------------------------------------------------------------------------
11 | */
12 |
13 | --:r .\DBRoles.PreDeployment.sql
--------------------------------------------------------------------------------
/pictures/Azure Modern Data Platform - Ingest Definition.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/Azure Modern Data Platform - Ingest Definition.png
--------------------------------------------------------------------------------
/pictures/Azure Modern Data Platform - L1 Transform Definition.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/Azure Modern Data Platform - L1 Transform Definition.png
--------------------------------------------------------------------------------
/pictures/Azure Modern Data Platform - L2 Transform Definition.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/Azure Modern Data Platform - L2 Transform Definition.png
--------------------------------------------------------------------------------
/pictures/Azure Modern Data Platform.drawio:
--------------------------------------------------------------------------------
1 | 7L1Zk5xG1jD8axzxfhfjYF8u2QuKnSoKuHGwFVDs+/LrP7IleSS3ZuyJsWxNPOpQq4skCzLPlmfLkz+hXL1JQ9jlWpuk1U8IlGw/ofxPCALjOHT+AS37hxaKxj40ZEORfGiC/tngFEf68ZufWuciSccvOk5tW01F97ER/tAYt02TxtMXbeEwtOuX3Z5tlXzR0IVZ+q7BicPqfeujSKb84yw+TQu0X9Iiyz+9GYY+3qnDT50/Nox5mLTrZ02o8BPKDW07ffhUb1xaAeB9CRfxX9z9dWBD2kx/5AurH/WbqtQVjRLKyhLWrhL/wMkPj1nCav4444+jnfZPIFjzYkqdLozB9Xqi+SeUzae6Oq/g8+M4DW2Zcm3VDm/9Uejt57zzfoQfB72kw5RunzV9HLGUtnU6DfvZ5eNdhPoIvY/kQ6Afr9d/IgP71Cf/DBH0x7bwI/6zXx/9TxCdHz5C6T+AGIa9A1CanCTz8bIdprzN2iashH+2skM7N0kKHgvg8s8+att2H8H4Sqdp/0j/4Ty1XwL5hOCwe+D7P0PIp2v/7RqnP13z28cXfLjaP78y06E4AZAOHxv/JW7Gdh7i9N8A4BNnTuGQpdO/64h+6AjA829xPaRVOBXLlxz3pyPu07g/I3VnGtKwfofPk1M78LGo34QD+/aXGbsP8gVAL/x08Sw2gFYWUHRxSg01jNLKbMdiKtrmvB+109TWn3VgqiIDNyaAePbjG/gknMKfUObDJSJ2TfYTwhUua9grdJWyljl/dOeeC/fs/KSB/3iZY7TzLxfmFTGCDrnIaq7gASi9/aNAL+r8YB74Twj7LA0+wxTOKkXW8hB6iQpslimTP3ZqkWe1wJKLjc7kjLVX7mUcI3Y/O0I6byvOvbbLQDofsgwYqh/J7Q4blxx73mHd9h9iMQ0hKm2j7wi5GqP6XrS92hFptUQILdr3xz4OSn52hM5XUM/L7lbBo+yuzrVouzW+MJG0zdg5hGLouec57It/rwTLtbHGQOLQe/YGs0tcUrbnEGAiq8j7RCb8rzM9W2vV4egxg/VivMYG9DRt/CmPDjyMtB4eN0Zbjz/yy0OhSXoqwXOR0gm7t9dKSG7anBkuep/i2A0CXfCIO654ZjuRvfSSCQybzwFomFmmO8zyaML66KwaxGsy20PceLS9CWZqhVTb7g7xehHy2Z1bmPAEeYCi3HPQGyveoli9XA3qSNGdyxVzBLPSxbTOV6lYZFWwJPoyjZgqnO35+XuveagruIOpt+LymlxCPNQoXJHApEV5M47u7KNcCFfeyp5ToXzHnraSLE6tvrAIQfewKPhG26x7xd+ejNhJV/oqDHNinN/rPd3s02q+Gm7f7otvNf2oCy5O6+lmeGXb2YZeybOZNDabRmG2W7LpnoJRjGDoHtfK4/DJIWMk8KzzH1ZmKZ2QoU0WlouQyyTg0xbB16tO6HFAOFwHK5fn85Upd04VyPyhvPiGgy4hqWlGU9DU4uiYfM38Ec4v2Xxdm8zL8u0+9A6hXEcRfezny1lbcIeaLZTrIRVK9/JM81plFgmAVYbtQZRspCkCYDCO6NWKKMaSEy5WTrI64b9OesyH/vnceoeMHs1tvQIgs4344g8zxXEkoVoiX1mOfKEnk7EnLYpWFV5Q6yIYfShCVpuuzUsFaEvuHB4L3f2+WgaSli3ZyZgk6/1dYJOFutdgDAzdDg+HVDFtWbQrenUTK/RbHNOzKy8mHpFAYig53ZK/XrdTRrEuD+jmURsc0l/KO1Fmds9nWs8KnsKhCbFfMUXK79eswIyJaYyDv78qqnzYjLNyRJDng7vq9Stk4SlwlpFbzNQf3JDWlHCi/HvOrJ56MWoPkkQpZwdx6a2SSTpyiBvzXqM9zfrQ1bmB+dVMVDTuzkjyqJUt/uJyJohTE+t293nOLOGPZmfOD3tS5XZcRC171NJjmxOJlw8JLbSj9O3zfojTCz/hvFwwElq3JitfqRfnP3RpycznhWW9sThf5500JMK++Lw7Jd+2jyK8i7nP6EbPSvGu5TTG6C+/w4To7HxhYkUY1tTzhmymR+OlzxbitNShO0jK0s8i9W8JtbaFwADCWIkJC9Hs5SeL7JU8R9AvEqGzRJ8Ro/K3bFtu1liSz7u+Kod98R/uWMLqSV6Vmi/s4Dkbxb/ql8eo1ClZ84i9HkWPbjbi0goKsFxTOvNIRj24od5FUPn6ZVfMc1sviwoY5lylEXHKSpzvMizP75VeHYEawdorfIJ5nxIXQgQtgbOxaVm6v59EIFaBr/chakQjfdvaUz0+CfzVGpcLTumWjfkNQ2Y81XjrcNNPfIkYkRvmc7UXm55Ntotangnv1eGzbjxith2QObY6nKA9bBpIskzt544pBb62JZ/GUGqKT1CJz+BI+LyM0vvYinOZzpRxsogoURLVpAPcQYixulH1gJGeeZzoZa17RATDk9surzJ1ACZj6NQTIS9xdCutGXNLl0ValmnyxThpi4slzkNXTu3gSFYiXO89uSRG0F8E+f6gi8bqypscQmJLIpY+hwXsrISXQeEuPZjJvRI84zLmObf0Xo/oS0nhpBVPkhBb2T/f/YJWw4DWSyUncXa+HyDHucIXz7YCTyimK9szKBmPcXuruZJ/7fcK3s7vjTR2SgTtXACvLEGKA0XGbJ1Nx4GVyskm7HbVoMhE5NsdE15YnRv9K9q9S3ZzleUiZoaf7d6NDfdKpG6pQ0z6sk1a52XhA72tgGI597aOjBzSzjkkL5daIb3KwZVpHjYlXy9nY/AmRkU5QeoXNNXCi47Xh68kWJmq4zZxywE/udIYm1V1b8KWPfIEIaR90HuMKbnozml8QcVXyjnQRw/X2MM8TNGXQ9MSkbbtn1jS3gZoj/zk9fai+33gs40AyxrEC61I8kMFYTalakDoHXZN5qPN+cILeqbDcZI565CsN73wto66iodM2zLNmMS0hPDFqDwVltLaDId4VDdXeOKZttTscj5pFGQeukLl8xKXSGdZyk5pscKpR1+f0LGATI/2Ld3l+/kphVz+1DhPhYvpyRKSMPMwBgavbs+Koyf6EdKNgWWPJWzJES3oVxCX5ytol0bd152gLlesVe7J44FkzzQ21NLdqwdEcMJV9eCHnKw5GQP1orgDiLNBHc2Sm1J6GM0sCtbi6nVJiSjM4bt2qzWJzGgjoWev2fzTGAFsE5//XVNOqrlqDllxw8YwFe71OlcSgt31D0vhsGjCqLMLB4RPEMh0Lw7kyzlGed/tSKD3hS4ZGuUSkiLtg/aHRJgvD+K+YJoBd+NUAwzIjyrTIkeNydC/YRF5r3r6OW1Gswj96wUwiPo94PscvPVmPgU3IUg4wlqMUxamzh8HItvTo/SkC1rSbi7acTS5QLI3XJ2U0dWDRzd3/Z7K/H2uDsD9pX6w6P26Luks+zYUK7iQhluE2lH8uvG4p6ajknUR8rzQLIsai2fIdVCEdUdwhshwigrfskggAgtQ1a2UltfWYoG3sffOpSSssvxh8vsKl2ucJOlrrz8e+GqnOMk5XL4AWTrHbkN3q6cYU9nfqc3jbjPQc8nypA/xMrO0gTdDlYVJV2aw5HpQGuSF5Nt/5BdIOAM87VxeFRtzIfdWYkCtZpy7a9hXnPNlGdgHH7X204pL0m9g8WJfGrw4Qr8zeMmv2Lvkt7J3YeSd2aS2WXESO8R03fjOeHpnGn1mxXZt0UxvA8TZn3AAyvCjORSfAAPQ/GQRsc+2mT5awzDymaVU1BlYAAsg1+sxDoEYfxvPL2A0P49L9iehAf2N4wEnT0v7s5/3boivYAWFf0agz3/Qb+WVwP8Or8R/4URA/6AT4dMTvxcnAvqOG+T2djZc5uh7YAXmmIf0l3NIv5wD+vO4AabRL6USiv2Mv+MA/CscgH8zPxz6P0bx2B+kePQ7c5th7yieB94q5DRP41NV378Hsgcj+uXjeL4d1f+6yP5tNA+9A/b3TfPEH6V58vuieeK9qzitw2Z6U3veglzfheLDnGjcx2L8xUmHpYjTP1H/IX4TeEFI/A9KfJT+RlhB3jvwtTDOi+YEBaSm4dAUwG/+96Pl46h++TSmT+j55dEO5QgCaX8qopAvEAV/LUJG/JVC6j3zcCDqikAmCIn+vcEzFCa/jJ7R1F8cPvvU8X9ND8DIvxl1+OeIg38HaX8Ffr6vNQuj3rHdpZ3+Hde9xSq/BPc78ffbsGVdJMkHzKZjcYTR2/Ogr4rS9yj4t3T1TuL9mtPx8S0/fZ428TVJ+A/oZwyByS+k4T8+MtEfxsHHp5tgOp8J1S+/0D6f40kbv0XZr0P6L9Y4+B0WPwSpv5Ol7cNgfnlTPE7C+BPXMZQkvsAcjv0xZfvbqRvvDR9xbmIQ2P8ulL9fB/Pn4QCH3ju9iPfaxNewQPxMfSs84O/wIKjA5SIOYZ2up0oFdPJ0Ct+SKL4DxDiW+gswSKNwTH/J0uZUHeI/Ud+D4d/gCPkKjtCvcsrPn4Ttn44k9L2X+P+KlwDFiZ9+yzT/3lOMYl9BDvatMPNejL3DRXbqad2/TFL64zD6j5ds+Et58zXbBf6quEG/lfXyKdzyXyRE/kHF51fE/D6l/X3geE88TloX/zjVgDme5uFkXIQIawCAJhr/SSyfASsJx/yfFsA8VadlzP2arAv9jv5Z/Saf7p1i+h8m3H1K7au3DOQr/7ymURGDNfQ1gq+zZ2tSnO/4LJ2VF3mBF/58pP4qPr5kAfg9CxB/ZbzvPf3fm/EzdP8Gvf+juZKJ+z5XkkUTNPUqPFa5dTT5FYTOS9yIanzmM0ybVAdt3hIpbxCtIforaAIkbToirhTnoRbYvAwErTrQtEQIFV/YQ92p6XaHzYX8EPsnZ3L0JTZ7S8k8r9GJTKIHPPEWZML00+NXygQJlzGqb3DiueYNojbtJvZh3YKcTIK51f05UAmB/pkdSXhRb2SbxEHIOdkn5xp+ui8jiP5X6KXMoXHDKj4KjghkN1BmddvmvW0b40EvL3RYFnEnCXhkbItRrE3mFSZXRIWxlIvIMB0rMItoZgWrgKxDucLumdUZM+p2c409sYa6M1QJWQJ1Hx2BsgbHdiyTYc8n8Ap0o5WLwt2euIzJ0oc0Tce5AkhwqcI+FVlbFDu5WRa63zaL3282c9ucTb5t3KS4LPc8x8CaiiKwi3gR8kWQoAoSRKqk7gJVRhazMbQiQRxVqha3MYliQOdolCxbQtJiNMitdUhFMm1NIZA3mhccOvQSSDyR41IZM52AjON1XromdCc29aDYPgkOhYMEIt2MZXXKEREsti6PXbUaWEgOVgrMaBqgNTicotsg0r0PtFU696PmNSHzNavgWf26MG6d5AHWIQVhXbCcRQpfCoPz1awR8B0AQWsifNJVBzbohwqDrA5Vngr6uNz720k6eL9F/CW0GvGAjlG/FyEA3yMlCueKJgL7Ur2Zt3G5M73JPqdYcKF0tQWek+GLrzKLdXn5mRA4HRRmzg7dW5HpNJ62EohwrsMqYaJM3+dDtwz8EEVfcjvxsdplZjeQbsusEU/XHX0qU1ecShybORshrXwdJLWyc8aeN4JnHc5JHlLsCk/6CjMYWy8WQ/jt89olsag9Pfi+3AeUZu7ZxWk5efE0Q1GU5KQxVj+80uW3gtZZu7pGSsF6S2KrlgiyBmtksgUu3jhZowpBvMKsXT4zJpClZbX2w3CjGl1nPO9jg3S7ulfsZ83dzGRXn4d6nxbTl0dYTU+xyxIzVGKdmYDMquV6uequGO9LnF6osXMY9ULVmKHJxDB1aTQTBQ91yJ2Vb9TE7Lcy07KM12TArutLBjmYYsQYZWm9/Bt5u90sfxuxuwRy4GxSb+4wc00RlWTlsidILq0ds/N8QHnbhQLpK9nkR4ECRzXcATHyWO+14D/2R5dizfEkBec28xpTnFKVZYRgKLn+ZjH4EDq95cqclrMVXWltSphBSl0T/6le8VuwYIgmXmKHsO6JaClttwsXM8q6sPLVUfPuuLjRva2BtEyHCbtsJe2b+iJhFroH1xC/WoVNTJ0RE3R72+GbyAS5wIQKp8iGpTLwHV5BmiufagmXQvBLhhlv2gSQYiTFDvwAeVGy1T8eeL0T46JlzS6V/H16joxkRywqP7rEhx4AF9hMW9NkP5V1HxGiURuiTY7hZk7WCDsu1sa0yNHPSZVl5uDk62sUCKvGhnINSEvtPcbjsJ0cZUS/FKk1P9ZjO6id39l+FV6SVUYRxgfETXaq7eBierqEZMRCr5ZQHQZz8MUeFzawU59bbOt8PnPsCMQaTtdNmbp7VYtxWjFPnoQ8rFu8g4nVIIWKiIed6T2n1VLmWfPPfjVeBtmwYnAKhPN+ccfh9aLMrCXBfcz296rGJKv3wmuLCt1wjR/6g114Oy0gId04xgQPTtgjr+Qs5oSnfJtXG+RXZYrBXOG+71Zu38NbkYWFWNkWJbQoLcgvUn9GHhPe/FafIXa3D6asYXkbr1p3bWyPUYO7brx2nIgLzlpsQ47iBaqDZQAUzLOyyOq+vBUk/BBubG/EPTu5lKXaWmlPDJ5nUAzTDLVATLJUwQKS06zkLhdlzcTXKLyE4zXWL6lTXiQBz1pZ7u+hDNawyX7N+j17gygn6zgQba+mdwQWWfsxHeVyLZBc5bAXkH1Z+zJE7VqXqxOpSw9NZyN4CtwNd1NMsh4xbge2MtQRpUjRoeYgvapuxbmrl3jauAnYawfZrqIzxbqRAVoNcPleGDCEXGCHfcqT3j8xYRjgse0U0omJXcyEXYjBeugI11q6h/7DBfliL+tEOSWFCFHUDG889OixvhjKJBofF+NBSlmFoatLi6Y70LnZWEb6AoPTlhTsqnlNcYsSIocUB2v4OSP5NpBdMptQyFNP2K3uJ4jl9WY1p5s3BvvuFKjeSDK0XyWQzkiQ7Fwdhr+fAugw1WHrhJWJD6zoc12/516g9vMwK8ckK9w23FWtYObJVXrD9nNhmuChvrNQtGxlnopg1TnRfHMu4jhvrMNhGIkwC712LRvnTlCxMxHH4b53SNYiLRV7WEese+UdPhJMmIKLV8RctNRC+hQ35EMJ3Vq7rpLniCHOvphIvCQpLJwvBr8SlSW2SB0z2zMTSGDNhr5zWEmmwSK2wHJzCWj1LroMjyJRSVvyvuMzxZdg1RvNZt/nUTa4uQ5lC462ZOrengy4o4obHN/sULytJzdztANHNppNDUjOG0W5msLnqbiXr7GpIqLEdhqkxPt5N0T75OOmjyhEs0dNoWv9PUtx3q6CaRXJJ3+FpPkhzFvxtAlBvN043DXKBKybzNsvtVB2Rto6r8uIdn9FikaJZf/ijvqwSRNilCtWckpnEIglcU4p+QHLpOdITNylkWELr8EaviULCpV4K53Zqjnum5iCn6wG7AurAfmK4Uzg762GT21/fsT8q37rP2ojfjIigGo/Tu0ADIm/3Mj+BNnf5Hx8xQtNfgWw38ocQ99HU/+F++bbeWl+3Uz5K0zeZwKgX/PNw9Q3A8v7PLxv5aYh/gUY/62b5i8GB/x+G68dnpob9P/YoW2O9P97B5y/y/2qhmX6iwN4/F96YP9bNHyi2t8Q7Xs/LI78jH8lMwL7ZmlF70O0nwtJ6P85RXXO7/8ismDs97H1Vwb/4PduMG4ewukDnqS2Sv4vYulXr+N3gqVPQv/frQLfu4PSfHNQqh8clCyEhRgBOszVbx2UQ2Ih+ss//2pMT5tPlKZQ/RZ9uL1QYKM2Z93lUuGsVeOZRtnj4brPtxLXVhVsUJqGELkW7cCt4/UYSYNa1HW4Oi3TJ6a8ax/ckvTFhsC27htAmvhcKZN9+bVa43NAPz10/bATbQsebtMDX+jxoJa07voASy8vkTTdKXrA5mZwA9jdyF0Ro9/xRYGhNCropcv2VIumpIhqfAylvDNvwLSyEbAT81h0ZMJN7s0iDIHtvdzMc2ETY2JNkBT02YJ1oh97KrahFNd9WR4AgtSv7lA44SCjd3Lmkax3p0839lGHbXAj8rIlH8bz+QFaBEKqG8mp0vXRmqdpcz4Zd9MYvRwaaRwMETu9KxS2XDmd1d5D5ePnG5bbuXZ+v1yl1JptBo/ktcoCfEelphddPCfywclnE6FtuoUCd+A48bQ7pp0nczHsBparblE2AG9URMMIK5DByj2BrT3t53fdBKfRtiteYHuxvdXQopjEcHbWx2JdZrDPjk91wPZ65pUQ43upO8E8ubYbTjMs5pql3lTQS3OXO9Q3O4pSbLSq5qPSsPvANuc4by9Mb+nGTCClcRSSZnBP865QQ2UvewoyiVKdZBk9LL5iDJJ4PL8wHhn6a2bt6Q3YWmkyIRbwe6DPAkqN+fVo40dA4yeBieGLuMxZ55r9tiH7DeFEDQHgwjuWtq7x7ivM2T3g7G3fX8QmkIZyE/cLqfmyQEQye5Qxbk6h03/w1kaY6PHTIniUVKC30FGFOvUXTEZYNs07NNfRF0eXsNuQfn41oYgy9YV1H8GNQlZBJDzDTdEa+K2NA0PTiwMA7SiLFVHBM+KpkmEd9NKkbYMtqynznSmA512uislJT+PFTXVyqKtL4EU9XWItQO6MIMKieQqMwRbnbZHuCusHZjAt5Pgs0DUKuuFqGXzvlDrKahcBuCpsNMeIYld3d+Z94DYDzMyufDFh9fC0Zuw6F0JJgR3eXiB6o6DF8dVoA3cTzp4Bc01C1tWQhLHyGYpUfstjq+H9oMiVDjLBw4TR0aAmsG3nhkPM6KBpc6c7dtj2xT/Bz0qAs9XQmtUA8iVPEYKMlOXxAAxqKKKQ2O3zXNZYIeUSKaeg7Uoo5YZztnkBJQRYT3C83pYm0U6YFuKkCyl2ldw117tq6Yx2uND9bLvoSu1Aj7IchiOLye6J3DS5DWKXcpANHisP6vrb0x6lbXrlJB86M8K/9LLC709shmapn3EF8kX5RVoGBJzybBvXCkvouROCkgcsq2DdpijBC+Gz4BCJKxdz6gnxIyNeBiC/FrLM/ajLsiU8kWBJWmH1BdMx281fij5gopM3+HW/8ARWmhBPmpKtF93ttuYPn1z5nVal1hzk5UOwxmHwcF+nLa8cqwwxq1WcYPOV+xPh2uIi34tGV1W9Yhl8PheB6R7sTD+EjvVo1ctrRaoR7l9BVgVDF7STZBPW9hKel+yZY2GQAU4SWxWF8JjrOytHtZEOEWWtGMDej3I9l1iJpa9gWxdbPtsMhTPau2ulSwU8fa4T7otttiPFA7eTsZfoCqM5s+4Uo7c4H2lLRPg2scqWjSpRQFL8IZB61j/HfH5SzNxYUVcTcSmtw0YE9gMkE3nC6Nvo1rlPTbIaOYBdWuES25EsUixvIQ0WaDkAu6bZzWLGReavhFsyz1UIZolUxHB3mgHpEL/qbn2FcAJZkBBDdbWXHVZI756xsqFhA5I2nQHjC6FvhNZd4MhEj2rkT6wt4VjMkZu9DmV77jYgTzXFeiyz+cl3pnyP72cHaoxqfw9LCBPsSM1cPRBQmrXQIHjMgm08Z+KityOO7MR1XfpI9nTJuTy2iaodtDyCYCylMDM7n+75++RzpcQPGl9eXsY6YeNjAxM0YyWTxZtctg5UxpRXd55LPfIxOnbXrmIbC7T7tbabrYgVPTDHMUjElImRpNqpkj9VOszmklntzRnWO9zUBK2G4WZjo5sZxchdECePSJEHX+MycCHn9ylg6bwx6lzYLrmgwVOt6NMNVxsxHEstIXvJul0vvaZbKKTQ65BZw1tcQSMz5KGQgtG/rLDHL5gCFqfD28He4hugHZmyLXtK7WO2mivOnMDsbvSY5byw3ZAoDqmyvsJ8XgfwjmTNqA+sU0uxvYbuRMyQD/bNW1dZe1znkheEKlD6sr4w6n59XDnr6Xd2xcDNK7iAVSqYtY5a2FaibweTDrj32g/IhYDAQ3n5KBfbQ9adcsTrvcLaCSuVoi+CJAluJ+wWM6GA67gudCjUUmOvb9qRjUd3vQek+3BzGXNoB28buTmfJ4TBimIRRzWXqXbDGzQ9NqBuDGvlcJXktIJAtBCfGamBHx4J477SanTB53Gsmc3dFUNiH5EQb5RqwOoyitQmDfTCjKZMBhRQYPC5Ns1LVr6txS33hii+cd1dHMvGdUZu8/qTTk3OfEA0jYOxlyKDamaF0hmlPPGOGYb6ZiAg8hO7rT+OKETJk6GU3qpSelVcAWAU0Z7wm4ZHYK96T5g48FMvN6Aysk1zmA0Iv8xZLIXPnfY8vz6UxxZY5V1B3goDZKapx4FYYqc2eS5T4njt0K6TWdUgcwboUa0sLUF/LmGCtltDTz6uRqPXDi9nN5au9oZCaOTun+tcdpVNyJZ8Zsfhyb0/bHOWt2sWbMVcyqsFPPWyZ6QVWkc9Qtgj1aUGxw+5jg27Cj3T44DtUzFGqDB8TJuSzHGZ3ugH2c8yegUbq1OiJm5zr95jCoKhMXXugRiaa/TSlekiXgk8DEW1ujDTY4ozDeh/dhqgxt2r3GJFH4iv0wv8wsFAIFOh7XoyKOMxBWaogImWR3JVjEuF+2XXlAQ9Px4qWShcTA0ku5QVuR8GcFZbzZN0mQuIslUKQhDE2td4ZgbWs8HpD0FqyCRML3jdrgmfEwESLxx8S10sB/UQYH+4XBEGle7hs/Lp9ro2om9HgUAOj5ISL4Jq7fwxF44TPPy1sgl0tBcKIcQNbZvOoElCMRqcIbIBrVfiBSPcWpFGwCWMk2NmnUzWMG+ROftwoO1926mXTo/NgHFVGJ91l1AKJ2GePE9foqq/l5pxFSYd9ZSypJBbNxFXzM9qEcU4zR3p3aU8QLiv9sgGMDkzeD6IRDNOIIrk1DwzoKE4gXQogWV2W8aFIDIiMXuD0Y2Mg5VBv+1PvYMec+oSlG9I55p2ahtLdMczOh+HkHl5xW1DXjvfUwp/wxLwwvjiYfnS3RIgF4i36htDNs4F27eJWNZB2wjApV892oOMctEUTYDW3kmGdLnoF+6ixvXzRhuODyoIgdBRcrX9wpJ2wDRh1D7doFiHA9Osm0FIfRCLHmVO2VRObUMxIFfheFWYyqEQLbjRcaKBlC2tAmHIZ2cyauEvc7E9UgoFATllehTRgwTWVkdWKbrZB2skFQ/M03Zcr7u/cRH/JEEoPPVfXTHSyexl5IKE3u1ZQcU06NFAw+skTWlACPSw6/rjIlI6Qr/lO0RXK5npRs8u+uaNZNSoY0qHofAMzTY0QkaNvEUoC2AuLoDK3gJOThSK8ql2TlHxZgiGiQ5w+Hq6qPty8fVIA0Q1UIbXK4Fuz34EwB3INxRBrHpUeTgvUYEjr9pUz5TwaJ2Vghe0Nvw7TDlKE55mL4kmhW4txWzb1ZUmPLk/AbsRx+YGAVR20cs8TQcBCDHkYJ+Gak/RCyZXcqcwy9HdON4Ww1jzzbiI3cr5enIEtR6A+GG+rSlpsrTZpeoFawTsQpE8mBczbY3uT3ZovegmXmEMlKgZTPrels5parnXSmHZHvKsxZ/muR+r9RE91InCEVXgSTYfSc/qjEpmN9GbA2UJsASBgE2SuXMFTVsfPtI5jGC0ZYBYdgDSTmlGw4qJtdEeDYYZUVKqEgDzIPsDJ7O6n+CTd6pDbU7FMXVuQFUfNT3S6ctzqMjhdY9o+n5fU8MgCfVZjnB9mqtXZ9GYZhm6R7+K0EocqLps+hHp/cXwK9y7FhCKRkyQ1nWS289Nv4avyD8W9yJ5QNLEhhajTXcMruHTsER7ySnMjCEAocTJCbeqZOEH/SyupMdmihPiZhDbL2PragrCjYbra3jEjUU3wSSfaXooEe4+bsIFszS7ulUVBow1VodoPOwe/CMo5o1pSJaAoh2UBiFralcpXo12PjVtEwn5TS4ATByOKHqvh92keMzuiNbJBjgPUFjtSGbylJ94wWbGxuYg/wE3roMxgGpFyGZ6xz18+d51oCz4xFgcyqdljPU0VMOVC9YzdIiQY7BvCCVGz0j4oG9ThH/FwjKgTsubUsrQAXk6D05D2PGpiLciXabtzUNB4c/mOtAvNp9tj3PoZCOLMdPsgRQnFd8xKgbAoG4XXJQ6ptbFx8ZR/B1W4RI9uVdzhtzMExLaqwsFNZEPY8QhFPEWEyP0JrqgK0rDm6MXV9FhTpEhNhxvtAkCovQxIgfAVBvS/rWJxcZyi/Qxu4t7U9UFj8yFU/82NcpIDCs5MLDShKHVHRp03+7UKXixJlGweuZRzG4m9lZMtGCfFsZRF3y68Oq5hEBACHhesandPajyMDR6zz1thWmblR6+FGjgp6UL4MFghn9acUUrF7ec10KFAGQ9egXHAkdNnp9GOFh8LUoQS6R2HDmBOwi+5tJpos7zXL/Wu2sqIS4BvJ+CKp85hZL5LJGC5lJ4dzowXQOjmikF94+4JJZQcFda1EtEuUGQBYoT6Vc3pucsrbRzKONm0ISxVMbNCB8FklHWcYeMp+SxYDXot5uGmgmMCu0Vw0zozvmIA+G1XXmvytzBEo1YsLCokEtZE8IRSx2mHusL/WBBilx4KTbTIolkLiqGgEu1+0MhSKSV1NEZoF6OCHd7WZsdzD28tyirMG6jc812hQgv4i43I5opcneKTUZgULIHZxq9bxCe5hoRllEfx272C2M2kEJ3HaZturecNiuFOulmIQOjvGimtXbsdHCxurgu8al8XhZ9AT429MntGmrxqaGtPEsiG9vcb5Yyh7kK9Eon79a6LHi1OGWH8ewVT+Rs4tFdMa9FS5XED2xVbq4skrytNg/fS/nefzPUy5rhaUWf5UKFzbfUprTpNsw0nw80kLEalAXDMkO9KVj8gR5BoglRbJc7vQabGaADKeds0C2swGimGIS6rVQygrU0rN1fkIbIh1V6dfDcgCpaeYWEgAqCnMBBweoER2UI84U0tYILa8vTppUGOipbw/URvxbVjTvpxlR+hen6quXbbrVQNZPQuGOEeYEhtbAHMxKHUyKRSWO5+4NzQsWFl1M2mbccx1iFayZoYxY/UJzhhot6c3ttwOyk9hbrPQ74OsrxsE+pn8pFSvZLYZgMqBHVrl6APGB1N8ZDRIx4fAq5lNrAkhsJqiILPyqj232C1luKDYeqy4+yp5uZZo95aOIUrD+N7/VuaSTNgQqs/wLqCauUMEKHD42xgo17zkEthmktckBwat4ruS5sdw0mCxU59ZR4mVgIWW50kOIJqZIDnfQYowmITTFsH8/nyU+i1+3BPhm2ywtqrCKczw7UE1TREdunDGOxrJ/PiDQwqwutv3TPN22fcEriig/Dxmgn7TyBjSDcj4uMH8/uxMZp4CY2SQ+eOgBZvNmXCwmrykpd/JsNTforo6bWLKc+qtQidpKC4Kii43FdvUHspleu5AEZXG2SeRkxqELdmpXLWHVrFAKb4Fgv9xX6NnC58erUvQrceBNuCYU0s9D3uI1ZQKhr1pTq+VLGs/KWO3UP2J1c9btLnguYqytA33tFbAKWEEOD0btkzlNiIAhAMfzQ8Mcs95u0DW4AJ+rcXmv6STkAMtBluDraDlxUNGUFBAMXPZvU95elUMnycHz+LY6gvzYGkL798pVRA4PAD2Jr754fAoVwEsie5aXrdH0G/vXUHRui3SybGIpAA3UeRP75mARs0YMXjuz26h2siVUsSByKM7UPnu6DjLhE6scW8IVOquFYE+LKKo6OinIg1R2AIukUIfSywaqrskB/Ztxb64bmvTGvKbK8XF8nW94ONJcW5vtVlApbPtWee3+/cmClYGnYCyzVQVFt/y8SSv7zAByK/Yx9vm2F+iLSA0Pwu1APAf8MUe+jPd9sYznyB2L+P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9P6I9/0ejPQhO/UzDn0V7iN9Ee9C/PdqDficnWvxbiP9+BfSPGzN//7DMj7D9TqrJIn9Ltd8/HfrUH4U+/V1BH39f8vJjNcUqLNN3mDmZfvoSkL+tzgtAPX7cU/gflfl9v2/xWVTVp+poTduATl+eAPyx8Uti+PMl2ZeVeX/ndCTkm0mp9xuvzXY9YYpArPw/F502BBCdzj5Ep3lOe14m0KH6EJ2eFvvt/rayl2V7vcZ55WGkPVA8CqGSgQg1u8Hrpi7LdtE7hh5cKza4VYCu1g3Z7GufrxvsWa9wyFnPtvxIzNRuzi9Sk+GJkz2Z2sJIdbV53Xr4qbXkE8uduj1MB0ydPrPl1jNUIa2GFa9dfbcSzWQel4hpBzk7h2ed78l0y2L85rLKgcgopJ/VwmmKZTfGt3jmHDP3IildN+BTHTonIk7/LKOWhOhcGDnH+BQ5t3YTtXU2XotiAuUe5hmd36Lu53/JvlpMLsATI8RNA5RnPfdqJZUeOwVWdN83awRNpHzMcuCl2C3ZI2WBp8uCpfaNoQKsjTXqboDTdDmRB15hJn+OmWUijnw5ThN6a4p8PTYLizD5blIlB3wHOwu8BIGcPbXRMqX0euFNkc+X377lAbqtS8sYvHwhNYagn1nGq3l31V93QY7NhAwzFB8XXboUal5tM/CwL7cdLZX2LpxaqFlikbIswW1YVSyst4JLjTy63cRnwfqa5R6o6vrFVD0g/bCEBhigBE6UcSalXQj0XzmxjSTtFFZdb1G1X6+bgccTH5Uc6VWnUQaU/Bvw+FJky9Cv9JpHva8SriuooSQDR3cp4TiFDQ0nx3yly28x+LBPkHzYndis0Q1j6n2/pw/2GK7ooAFFmIuAycVdsP5gO7cNw5exDnN6G5le0A+/wepHXqs3whxcfR5s4KW8GBDID6jc56vClJWxPNJ4sNtqQIZM9giKMDCK+KiKlQ6ks1uzIRsle568SVp4UH0gW9RhoX6XN3u0NeYAIEl7DqQtGrnL4PjYEsRUUzRY0ksFc0IQjtmK7kyQZFBQFTz/lPVEoq7BpTnGvOAZhMAa7kPWQusf7G5tJHBJ2M5SIgYvqhmwRlIF7vQsQRy9tw4srW5FUeiVBTTmroIvsebf/NFY3L3pr8TzISO0lLgrLnoJJIoR83JOiyGBefd2yJGC8hx16QZgAMm+BmG8fBfeaNOKFIityp6RC2uFAjwfy5U2AHV52cUuzUy+emV1GmUcg+Cogc9vlFA3A7tvUtwW+TPsKLvGM4uwDmHa7C6EORmxSGlmlRko2ke5W1jRr26n4Xe23jUoAPZHPAjBklf2BVw8qnBndlFIt9Zpg5O6bSHvupS3BGMrhUtgZYUzK4qk8ntLERavVtRuhCWKPao0P7kCMAfE9IaOzvjKcxfiEfmtnCOW792twc4uF3/P64HrZ/FCXzRt00/wbs9STJxCEpTm1co0r62HVtbjrn8OC3SQS2Zdjie7ZlJl5QUn3NSiXQbB3FleMC4oeLcJ+VWqBa9CWE2LvXlzr0Mq+2YuJKAe0sNdhmhKNLZOvQvVokWbE1adq3ZPeA2NxRnUva5jPvitC89QYFQ0nzZyMoqS2AUXi86OI4n55cg2c5g95Ag3PkJA3bLBrvssruGR9OK+emjhFL0ycj2pte9GQoMt4Dim02W9GfY904x4mhBSLolDDu6Y6IvlMbgNiI6oUD6y0e7PeSUwz6A4WpTVcSPqDOCof5SEnlnzW0GtMqPvsWKOsQXdn2Tj3iVbHWCWwa7btR+So5b0diCGcoPgon4EM6ELG7rriyaQD3OiFfMZTVTTBC2nAwYAYU0XeBnBwjXHKDqRkCK5wOZNOiySGniRYSqCnhGYyko2+kWtTImRTXdLckSMUluJDHFyX9fmpKyuIbUOTjEVuP0C96KlL56avTVCKHCUmNit+f2oWuzIDOOwy3hQn+0W189pxENVW6nXwXl2v5OLDEQUmwPFASFtjpcpfqsd2TpJ8Frl8YXe8hJY47NSb8D8LaayxmsK2Rc2M584SOHCgN+rsS83m4cPytCqGdEqbILaEXj+pWFM2dYe7nd1faBOTOn91RULT3qqbDMfT/wc48qKkW5XSu/2aohfbhdufPVW5iXhGo6PttlEYrH2Zzylhkfo+5gkCgzs4lZ3wwdzKRtVAX7TEhWUmwnFsMcmMLX6xoWG+7WsYTFVh9ImHjT1ZFHyfsFrcQa4QOR0wQkVGiHUaYpMe420xOirPBakOUdDhl+T60s09Ad3my7e89AC3dvtPjYtV8xZdjEJmE5t1pEaJREVDyueXUTnOWenioilNomowRiZy2Fsz1640TsFN870yo3WMy0gCZ3V+tYr6JtnlMYCegHpbz755ud7S5eDnoXBhYZdSjTHfP1o0Pd67n9cT5z8bZGNT7WlvtBxf0bh92ruNysK/KmCx99tDv7RQ1w+lYf6fcOP+K4Mv0+lh/5X4PypAMzvwxn7ruCMv4fzf39U0VZM3mefP5wvhX+8+uc5ReDi17OlPkNuWkXt+jle3xrOG58svL+GIz6Zyt8Lpv5A0vc3r39HfXnkBvzJO/Z5wbevFWnHyJ9p8v1REH++2Hhfreo2hM34bIc6fHMYfBdS5d8i+PedefifTZhfP+sJQX+m6c+ObP7NAz4w0Mfv/AZvf8IBUPDXXX/RUMTld3Hs0D9H859VG/uVkf/4lpSvKD5fKy8GuOxbHT6Ev5c23x/rfDOO+FQSlfiNRor+jP3nHPLusb89/eV3HvtBQHwzxsPfH3XChlOcn0381w6W+p90uv/HxshvdoF9uv673O3Ee+loDHGensD4+jr3fwNLyG8O6yW+Ug6a/ivxRL7H09+utZH0e6BgJP0eKt+sSu8nVvkvihZ/SU8/IegHUfnH14xfMfMflTP+a+FEfOUEYRAEACkTkFl0KThD6rtQhphjHtJfPhvSOXIUHCj8UTf6K9H163lSvzFWvkL1X1Oivhky30dIpQJoAXbatd8REt+G8/ehDf7EYf8abejXalV/M7R9pVZ1N7zB/7tBGduGQ/J34gz+zljtK26AUzX6fhAGRvOLWYXN34g0hMJ/D2l/KaOR70/O4NPF6N7z2f+kLvsnYe3L407en3f7Ne/bt9NukR+y8XfZDP2+ZCP5Xqd8A9TZpLVJCnI3Ptr50CmgJuA5PT/+4/xlTgPz1MnfTmn4X+TJz2mH+pPsTeJLGfopV/f3DkD5M46pFcKykuZ2mkSFJbVwiUUr+Af6DjF/gZfuhN+wg4gP9DNGU58aQNgH/vnX+/+M+7xd7Z9fmelQnPMHZPCZ9PyNN+93wz1fBcjHfr8b7fmQv/gXBHe+Okrs70Db3wDlD1HmvwvKXzmaa2/Cbnwvzb73/FRV+/ww37u68zzogIi/qZ7Eprpt4cv5N/GoD220h3/4ADYagsJA+MfP2NvmKZHcQTv9Ip500s8e2Ln6RF8f7oFkHchklJT2XNB+/t4pkwryaTqID/dLCjwrB/WLenebUYYG1yDNZ6HiTFsf2SlaaZPGpvkFqrSAfcgs+aT7Yfa7bCwUUIdk4uuJmkodjUmzUcE+sGAZ01AlFyorEpD+Ms/NSD8PHiQsPZwtRVPyWRPIol/RZeLeNjSSu6mRVYLs8KJrDY2F9SzAq8nMU7qWvZE6i74er0RPmpjo9lTHhiQxKpqZCl1SpynfBh3sF98Nqh3m+oE+1RsoRZQzle7g0OzYqA6BI5SvhfAxe1Z3k/BuFg5rCXHDS7vMcbKqxEXBaz1fXI7+eJEjOX7Ew2UnJL8F28bh2nemY+Oux7AQiZfAahdr22gat/vVvGXF4ezzcyIitAt3hbvqEKxVi29BzcPT7O2y31HKXGDyWssYd+u7u7kafRyMiJzTNdwA8DFSgU6DVLKVAS5ttloeIPFUety0lxYRx0Ox0ACc48nWrFjCfexHWAD22mFHVRBEwmkTWUQBBW1ZzVIylRBJIMOHxRruyIxZHdDNzbvp9M14K/ajX4yhgxwBww7c5khwOGF2AZuD5kDPF/5hhYNvs5qPCh3KqrwpTV5RPMfnWnflLdleSYDfDMSviBxDluZ56xbklgTBxdggM1TN40Dj+ckbgBLw5/P2KphD0ClOcS9dKPRgx2rH9qA6Dgt++SBkX3Q4gx35YjhBU+9SK985LYbp6D17LIGSG0ZLa7Ymg61U2WrQ+SHEJgVS76LcA3lR91OTMbKHk6ZHF6/LJLIJ6hvHpfcD5UHIXFyAtE8beYUmI+n2W87eIxEMM1DuOclUYEe/ktYED+qTxOmrZLf+/qrkWcyqaMqXWoYecM6HSWUqqOl7skU7qtTjYNeVOuwF9DKvKRNqEBAPqn2dQHaWUCcqdmfU1/Z6XS60UjW8+IFVnygOqgnRDy1nr0mga6oPDjMV7YxYHU+aeS85jsuBuX6fQ7zLpYORbfh4R7lp7M9lhYhvjqHbjbcKN6HvW4/DTTc4ohcfEtBhtndF6pN23CfjuFXrjPIvTyY87Arp28OIUOk+Y6+tW58wY9c8kAtIIxUzcm35DSQDsqR0dZ0uiASlUJ1RpqLCvDxJkupNB7/cEeYhdXpNru2qNqsBRFTYHazM1s10ElEvkXyYE5Q79IrYrlM7V+xl1dNrlBljV/eCjuETMVLQI36hEkiH9prBcWUYQOYeQBefDo766nSg4hLLvUSCfUve3odpNzLiqODqMHJGgBpUyoPrSZ8P8JCJURBmy+1AGcGUarQv1FL1ACPdkCzSIdDqu8eFga2neZOQx7GK9QJlg7sjtHttjBZQoHFCZ7jfc8MWLsUsEsvqOzQs8fYASy/7cT/OOZ/ta7wTFN8ed6mA23zxe8xjiNSbF1SThE1A80HhRjYYsEuN8Obd2VAdhkphuz0rW4MP8064052jYVMDOcaOeqcP3jfTt1RM5eTKp+30R9Pr3u0OkvrZCb+04jQv4iMwu0WgV7qq9YAX17dtklQNFVVwC5rXM3/khbvLxqz46laMj2B3Z7pwkDtdpumW9qUGSDRc3FSLwF7tB3Gd4dAeusi+VcMBnpYTmxjcVm5LTPySq7osk5VPDzuC4xHfzXUX3CD7dvXtY5bkrI+FkCymnuofjah3uj593HvuYOggpia5XretOqZm9PxH7+JLqIvkVHsA2JfH4xo/fK8y+iqHidEPuXJ4oQZ+6zhuvasBbZ9rmndBBP5+zXM/EvmK3CVZcSSfhvDVvV409WgtvCLuekvL8Ij3r0MvaxQINUFKnlELIMrMTqI8HjS7Ma+m7dYt3CmqTvm4rMURMl3IEEaslUyHvlkYIKardkR9ys7pfJXM29A/WSkmCgbJCScpRKFOlV6UOhpXfbJQTeTuuU99Y9RhqNW7E45kpi79xdbm/Xr1+HQ2jM1JJiN/6sioIUEyqshF4KqQ6oSyXaP7Ldc1Yq+MZOxLokPhwh9tnmh2NjL9oDgVLAu/INOmV1K0N+ZDM3KPUAKDG9z2jidHG8ooAz+zgT2mrDQmDBt7CpLYzAswtrrZo9S6R3adTllmEzDJ0OcMRahK0de9FY9hRI5XP+oXepqspPP6xbfx2+pi8R5trwiJ9HVOuVjfbrIb7wEKkvAlF5OYcaQnD5di9+X2dkzgYongwgrgHXX84YShpxheolFhdji1Syg4CnZA1NGYIa/GV2p8L81J4DjPx5cduxsJhUSk29kXhtjdOz8ZuNAMsU5I+Q2eOli9aHtBboffhImSH3XqNHoi7DP8mOhqTJqRI62t4MOw29yGY3GXIdXinggWjeDcZSdvPTyH4hyQzVyoB4XeLqmFRHBKH/OyVKg8zxt5rtR808QvOJ38a3GDwjhG0PnGPE2p1zd5RQ0jSJ7HVSWutmnMNzAhOT0scBA3sUaOUye7y9X9/qT4rOlo0RWlgr4TNxOoU/JoTW2+z/O4VVXrVlV+kNguIWm+hTci88RTa+tsvjEiH8+PsBFeCpnXCXVpGkPGKsoLn/1xL/OqWWNIGfSW0CiCi9JDYjzh4Tn4KhnCfKXwCkr9XibocnMrbPcJe70asb1Xm1iRuc/nudvtIn1YSsyQ0TYDwbt69Z0/31UVsmznVF6iF0TRfV9KDJfnXy8+gW1NUVBiCFn+wyom3SXpXJ3U+rwJaSgvpkJBNJqbkCsyGqkIF92xaY5fP66hLdcxDqOjOCJGKJ7vecllHMx+qeeoAfXEqYqKqtPeD1NdVYtbme7/r+06dlhFguAfrUyGI5hok0w2N4bosYkmmPn6ZSyttE8btPGINCAETXV1T1dhyWlUEGl+0k6F5HaT47Ex8nuhEnuGVVM0bUGjSUACpNLtTUfV3ZmGY6CQvLTHqepkvJU3d/aWQzLgoHd9j0zWb8iEsIJIV4N3MauvoLYauXXVa+y9A9ZelKAxQSLjCqa7vTOivvHMSA80H9K3zxOmAIsWwGRXE+VEucOCq1IeSU0PLCfU+pzJzkayKvQZI2u09nSbO1zG7FukZffOwVitcufTy2KwVSIQelibV2Z+LnKThGP8+sbIaCo1l0yP+CKNVhB08C5SH3ZjymjShfgux70McbzRxzXvX7Mk0JOYTNH64HXl14HjpuqDY9LUngbTPew+w/5NpoRXbpclOVDua8LogmqQKSBg0iN6AvRNr66893wgcRMfYcWx+IaFz2KAMH4azeXkpmtsvhj9/WUrEUfzIsM7fnfZhmt0mt8KjANctgR0asR1awBAHgQejcMSsdBdXGrVeexsMJu+tk0Vylc++ri3mW/lsV6rKrOLQf/gXdSTsHKTOI0FgZ5ivQ4afeCHWK4PQ9PeBdVHuBRx85Way+GxzZrs6LB21kE3FqwgWPHPuCUF66fqUb3aYrhvw7hEvPkJgB926KOv61O17BS52UPCFRE/WIWG6lN0DSyFk2TSk30PtAkfWv0qbgLnO5ErkYw80RUksZSElVYEBkqZbkIlF5ovjO+WQ++9bRzAwEkvcrmg3a0XVlUZerYLuBdPExbCriwuLgQT3lN39eBBBrV35tk3mBCrRI4vrOzfp5EmmFfjkmM7iunrSoQRsmxDB6yRyTUVKTf4dj8wg1WqKlZE2c1X258OjXMFAkhtyqaICR5kAYD2HDrM97keXBrEUZsQwvMS8uoR7GL+QRUskDTPdXk7z2xRhzCvOVMmFE/B3zpWs5wUbEyimot9TxDXFn5xb9GM1GIrx66vmmOdes27qR8vUbY8EpNsZU83X8Ro7/2BOasx+ksDDWkyN2L1OL6pniiSg0faOWGCpuN0BXboWeWFMWZCChd15DujboF3kxChWgYbIRlzob0MuQNszPsZv+hnbqcGJzUOobmNfTmxkPFjs6VvCQ54+KhXnU32cFcKTZaTZYgKr/dLwd+b5iaFugxR7J/48DwkFkmTg5Y2isiO4X7X+kdfl7TVpNK3KGgoIxbEA+slnnpcwGq2pS2qn2B5RhFv0NpInCl/tIVn7QuWQIdgL9vz9LrOw775s9c4FbQG/ZTyBtJc7IUVnoRqsNVZGQkcC9cju5tG0jT+kZRaFGiAiEarJw8skzIwmwzte6NV3LRViqqi2nQWKi6rsPZdUP5M5fE7Iyq/aSL9YQuV5X/c76V+Z9jxl5ntH1qo7E889z81in47jfLve6T/aMD97zbn/vLE+x/3x/775txxOPX9/OspxSkbGrzbgFf8DA==
--------------------------------------------------------------------------------
/pictures/Azure Modern Data Platform.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/Azure Modern Data Platform.png
--------------------------------------------------------------------------------
/pictures/ELT_Framework_Dataflow.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/ELT_Framework_Dataflow.png
--------------------------------------------------------------------------------
/pictures/Ingestion_Pipeline.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/Ingestion_Pipeline.png
--------------------------------------------------------------------------------
/pictures/L1Transformation_Pipeline.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/L1Transformation_Pipeline.png
--------------------------------------------------------------------------------
/pictures/L2Transformation_Pipeline.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/bennyaustin/elt-framework/e632231e30ca84e68ae11e87eb0369c16d2e18b2/pictures/L2Transformation_Pipeline.png
--------------------------------------------------------------------------------