├── .gitignore ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── SECURITY.md ├── SUPPORT.md ├── images └── Architecture.svg └── src └── DataverseToSql ├── AzureStorageFileProvider ├── AzureStorageDirectoryContents.cs ├── AzureStorageFileInfo.cs ├── AzureStorageFileProvider.cs └── AzureStorageFileProvider.csproj ├── BlockBlobClientRangeCopyExtension ├── BlockBlobClientCopyRangeExtension.cs └── BlockBlobClientRangeCopyExtension.csproj ├── DataverseToSql.Cli ├── Commands │ ├── Deploy.cs │ ├── EntityAdd.cs │ └── Init.cs ├── DataverseToSql.Cli.csproj ├── Program.cs └── Properties │ └── launchSettings.json ├── DataverseToSql.Core ├── Auth │ └── DacAuthProvider.cs ├── BlobLock.cs ├── CdmModel │ ├── CdmAttribute.cs │ ├── CdmEntity.cs │ ├── CdmManifest.cs │ ├── CdmPartition.cs │ └── CdmPartitionGroup.cs ├── CdmSqlExtensions.cs ├── CloudEnvironment.cs ├── Database.cs ├── DataverseToSql.Core.csproj ├── EnvironmentBase.cs ├── EnvironmentConfiguration.cs ├── Jobs │ ├── AddAllEntitiesJob.cs │ ├── AddEntityJob.cs │ ├── DeployJob.cs │ └── IngestionJob.cs ├── LocalEnvironment.cs ├── Model │ ├── BlobToIngest.cs │ ├── ManagedBlob.cs │ ├── ManagedCustomScript.cs │ └── ManagedEntity.cs ├── Naming.cs ├── Notebooks.Designer.cs ├── Notebooks.resx ├── Notebooks │ ├── DataverseToSql_ProcessEntity.ipynb │ └── DataverseToSql_RootNotebook.ipynb ├── SqlBuiltinDataTypeValidator.cs ├── SqlObjects.Designer.cs ├── SqlObjects.resx ├── SqlObjects │ ├── DataverseToSql.BlobsToIngest_Insert_Proc.sql │ ├── DataverseToSql.BlobsToIngest_Table.sql │ ├── DataverseToSql.IngestionJobs_Complete_Proc.sql │ ├── DataverseToSql.IngestionJobs_Get_Proc.sql │ ├── DataverseToSql.ManagedBlobs_Table.sql │ ├── DataverseToSql.ManagedBlobs_Upsert_Proc.sql │ ├── DataverseToSql.ManagedCustomScripts_Table.sql │ ├── DataverseToSql.ManagedCustomScripts_Upsert_Proc.sql │ ├── DataverseToSql.ManagedEntities_Table.sql │ ├── DataverseToSql.ManagedEntities_Upsert_Proc.sql │ ├── DataverseToSql.Types.sql │ ├── DataverseToSql_Schema.sql │ ├── Optionsets_AttributeMetadata.sql │ ├── Optionsets_GlobalOptionsetMetadata.sql │ ├── Optionsets_OptionsetMetadata.sql │ ├── Optionsets_StateMetadata.sql │ ├── Optionsets_StatusMetadata.sql │ └── Optionsets_TargetMetadata.sql ├── StringHashExtension.cs └── TokenCredentialExtensions.cs ├── DataverseToSql.Function ├── .gitignore ├── .vscode │ ├── extensions.json │ ├── launch.json │ ├── settings.json │ └── tasks.json ├── DataverseToSql.Function.csproj ├── Ingestion.cs ├── Properties │ ├── serviceDependencies.json │ └── serviceDependencies.local.json └── host.json └── DataverseToSql.sln /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Ww][Ii][Nn]32/ 27 | [Aa][Rr][Mm]/ 28 | [Aa][Rr][Mm]64/ 29 | bld/ 30 | [Bb]in/ 31 | [Oo]bj/ 32 | [Ll]og/ 33 | [Ll]ogs/ 34 | 35 | # Visual Studio 2015/2017 cache/options directory 36 | .vs/ 37 | # Uncomment if you have tasks that create the project's static files in wwwroot 38 | #wwwroot/ 39 | 40 | # Visual Studio 2017 auto generated files 41 | Generated\ Files/ 42 | 43 | # MSTest test Results 44 | [Tt]est[Rr]esult*/ 45 | [Bb]uild[Ll]og.* 46 | 47 | # NUnit 48 | *.VisualState.xml 49 | TestResult.xml 50 | nunit-*.xml 51 | 52 | # Build Results of an ATL Project 53 | [Dd]ebugPS/ 54 | [Rr]eleasePS/ 55 | dlldata.c 56 | 57 | # Benchmark Results 58 | BenchmarkDotNet.Artifacts/ 59 | 60 | # .NET Core 61 | project.lock.json 62 | project.fragment.lock.json 63 | artifacts/ 64 | 65 | # ASP.NET Scaffolding 66 | ScaffoldingReadMe.txt 67 | 68 | # StyleCop 69 | StyleCopReport.xml 70 | 71 | # Files built by Visual Studio 72 | *_i.c 73 | *_p.c 74 | *_h.h 75 | *.ilk 76 | *.meta 77 | *.obj 78 | *.iobj 79 | *.pch 80 | *.pdb 81 | *.ipdb 82 | *.pgc 83 | *.pgd 84 | *.rsp 85 | *.sbr 86 | *.tlb 87 | *.tli 88 | *.tlh 89 | *.tmp 90 | *.tmp_proj 91 | *_wpftmp.csproj 92 | *.log 93 | *.tlog 94 | *.vspscc 95 | *.vssscc 96 | .builds 97 | *.pidb 98 | *.svclog 99 | *.scc 100 | 101 | # Chutzpah Test files 102 | _Chutzpah* 103 | 104 | # Visual C++ cache files 105 | ipch/ 106 | *.aps 107 | *.ncb 108 | *.opendb 109 | *.opensdf 110 | *.sdf 111 | *.cachefile 112 | *.VC.db 113 | *.VC.VC.opendb 114 | 115 | # Visual Studio profiler 116 | *.psess 117 | *.vsp 118 | *.vspx 119 | *.sap 120 | 121 | # Visual Studio Trace Files 122 | *.e2e 123 | 124 | # TFS 2012 Local Workspace 125 | $tf/ 126 | 127 | # Guidance Automation Toolkit 128 | *.gpState 129 | 130 | # ReSharper is a .NET coding add-in 131 | _ReSharper*/ 132 | *.[Rr]e[Ss]harper 133 | *.DotSettings.user 134 | 135 | # TeamCity is a build add-in 136 | _TeamCity* 137 | 138 | # DotCover is a Code Coverage Tool 139 | *.dotCover 140 | 141 | # AxoCover is a Code Coverage Tool 142 | .axoCover/* 143 | !.axoCover/settings.json 144 | 145 | # Coverlet is a free, cross platform Code Coverage Tool 146 | coverage*.json 147 | coverage*.xml 148 | coverage*.info 149 | 150 | # Visual Studio code coverage results 151 | *.coverage 152 | *.coveragexml 153 | 154 | # NCrunch 155 | _NCrunch_* 156 | .*crunch*.local.xml 157 | nCrunchTemp_* 158 | 159 | # MightyMoose 160 | *.mm.* 161 | AutoTest.Net/ 162 | 163 | # Web workbench (sass) 164 | .sass-cache/ 165 | 166 | # Installshield output folder 167 | [Ee]xpress/ 168 | 169 | # DocProject is a documentation generator add-in 170 | DocProject/buildhelp/ 171 | DocProject/Help/*.HxT 172 | DocProject/Help/*.HxC 173 | DocProject/Help/*.hhc 174 | DocProject/Help/*.hhk 175 | DocProject/Help/*.hhp 176 | DocProject/Help/Html2 177 | DocProject/Help/html 178 | 179 | # Click-Once directory 180 | publish/ 181 | 182 | # Publish Web Output 183 | *.[Pp]ublish.xml 184 | *.azurePubxml 185 | # Note: Comment the next line if you want to checkin your web deploy settings, 186 | # but database connection strings (with potential passwords) will be unencrypted 187 | *.pubxml 188 | *.publishproj 189 | 190 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 191 | # checkin your Azure Web App publish settings, but sensitive information contained 192 | # in these scripts will be unencrypted 193 | PublishScripts/ 194 | 195 | # NuGet Packages 196 | *.nupkg 197 | # NuGet Symbol Packages 198 | *.snupkg 199 | # The packages folder can be ignored because of Package Restore 200 | **/[Pp]ackages/* 201 | # except build/, which is used as an MSBuild target. 202 | !**/[Pp]ackages/build/ 203 | # Uncomment if necessary however generally it will be regenerated when needed 204 | #!**/[Pp]ackages/repositories.config 205 | # NuGet v3's project.json files produces more ignorable files 206 | *.nuget.props 207 | *.nuget.targets 208 | 209 | # Microsoft Azure Build Output 210 | csx/ 211 | *.build.csdef 212 | 213 | # Microsoft Azure Emulator 214 | ecf/ 215 | rcf/ 216 | 217 | # Windows Store app package directories and files 218 | AppPackages/ 219 | BundleArtifacts/ 220 | Package.StoreAssociation.xml 221 | _pkginfo.txt 222 | *.appx 223 | *.appxbundle 224 | *.appxupload 225 | 226 | # Visual Studio cache files 227 | # files ending in .cache can be ignored 228 | *.[Cc]ache 229 | # but keep track of directories ending in .cache 230 | !?*.[Cc]ache/ 231 | 232 | # Others 233 | ClientBin/ 234 | ~$* 235 | *~ 236 | *.dbmdl 237 | *.dbproj.schemaview 238 | *.jfm 239 | *.pfx 240 | *.publishsettings 241 | orleans.codegen.cs 242 | 243 | # Including strong name files can present a security risk 244 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 245 | #*.snk 246 | 247 | # Since there are multiple workflows, uncomment next line to ignore bower_components 248 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 249 | #bower_components/ 250 | 251 | # RIA/Silverlight projects 252 | Generated_Code/ 253 | 254 | # Backup & report files from converting an old project file 255 | # to a newer Visual Studio version. Backup files are not needed, 256 | # because we have git ;-) 257 | _UpgradeReport_Files/ 258 | Backup*/ 259 | UpgradeLog*.XML 260 | UpgradeLog*.htm 261 | ServiceFabricBackup/ 262 | *.rptproj.bak 263 | 264 | # SQL Server files 265 | *.mdf 266 | *.ldf 267 | *.ndf 268 | 269 | # Business Intelligence projects 270 | *.rdl.data 271 | *.bim.layout 272 | *.bim_*.settings 273 | *.rptproj.rsuser 274 | *- [Bb]ackup.rdl 275 | *- [Bb]ackup ([0-9]).rdl 276 | *- [Bb]ackup ([0-9][0-9]).rdl 277 | 278 | # Microsoft Fakes 279 | FakesAssemblies/ 280 | 281 | # GhostDoc plugin setting file 282 | *.GhostDoc.xml 283 | 284 | # Node.js Tools for Visual Studio 285 | .ntvs_analysis.dat 286 | node_modules/ 287 | 288 | # Visual Studio 6 build log 289 | *.plg 290 | 291 | # Visual Studio 6 workspace options file 292 | *.opt 293 | 294 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 295 | *.vbw 296 | 297 | # Visual Studio 6 auto-generated project file (contains which files were open etc.) 298 | *.vbp 299 | 300 | # Visual Studio 6 workspace and project file (working project files containing files to include in project) 301 | *.dsw 302 | *.dsp 303 | 304 | # Visual Studio 6 technical files 305 | *.ncb 306 | *.aps 307 | 308 | # Visual Studio LightSwitch build output 309 | **/*.HTMLClient/GeneratedArtifacts 310 | **/*.DesktopClient/GeneratedArtifacts 311 | **/*.DesktopClient/ModelManifest.xml 312 | **/*.Server/GeneratedArtifacts 313 | **/*.Server/ModelManifest.xml 314 | _Pvt_Extensions 315 | 316 | # Paket dependency manager 317 | .paket/paket.exe 318 | paket-files/ 319 | 320 | # FAKE - F# Make 321 | .fake/ 322 | 323 | # CodeRush personal settings 324 | .cr/personal 325 | 326 | # Python Tools for Visual Studio (PTVS) 327 | __pycache__/ 328 | *.pyc 329 | 330 | # Cake - Uncomment if you are using it 331 | # tools/** 332 | # !tools/packages.config 333 | 334 | # Tabs Studio 335 | *.tss 336 | 337 | # Telerik's JustMock configuration file 338 | *.jmconfig 339 | 340 | # BizTalk build output 341 | *.btp.cs 342 | *.btm.cs 343 | *.odx.cs 344 | *.xsd.cs 345 | 346 | # OpenCover UI analysis results 347 | OpenCover/ 348 | 349 | # Azure Stream Analytics local run output 350 | ASALocalRun/ 351 | 352 | # MSBuild Binary and Structured Log 353 | *.binlog 354 | 355 | # NVidia Nsight GPU debugger configuration file 356 | *.nvuser 357 | 358 | # MFractors (Xamarin productivity tool) working folder 359 | .mfractor/ 360 | 361 | # Local History for Visual Studio 362 | .localhistory/ 363 | 364 | # Visual Studio History (VSHistory) files 365 | .vshistory/ 366 | 367 | # BeatPulse healthcheck temp database 368 | healthchecksdb 369 | 370 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 371 | MigrationBackup/ 372 | 373 | # Ionide (cross platform F# VS Code tools) working folder 374 | .ionide/ 375 | 376 | # Fody - auto-generated XML schema 377 | FodyWeavers.xsd 378 | 379 | # VS Code files for those working on multiple tools 380 | .vscode/* 381 | !.vscode/settings.json 382 | !.vscode/tasks.json 383 | !.vscode/launch.json 384 | !.vscode/extensions.json 385 | *.code-workspace 386 | 387 | # Local History for Visual Studio Code 388 | .history/ 389 | 390 | # Windows Installer files from build outputs 391 | *.cab 392 | *.msi 393 | *.msix 394 | *.msm 395 | *.msp 396 | 397 | # JetBrains Rider 398 | *.sln.iml 399 | 400 | .vs/* 401 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # Support 2 | 3 | ## How to file issues and get help 4 | 5 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 6 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 7 | feature request as a new Issue. 8 | 9 | For help and questions about using this project, please open an issue. 10 | 11 | ## Microsoft Support Policy 12 | 13 | Support for this project is limited to the resources listed above. 14 | -------------------------------------------------------------------------------- /src/DataverseToSql/AzureStorageFileProvider/AzureStorageDirectoryContents.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Microsoft.Extensions.FileProviders; 5 | using System.Collections; 6 | 7 | namespace AzureStorageFileProvider 8 | { 9 | public class AzureStorageDirectoryContents : IDirectoryContents 10 | { 11 | private readonly AzureStorageFileProvider fileProvider; 12 | private readonly string Delimiter; 13 | private readonly string Prefix; 14 | 15 | internal AzureStorageDirectoryContents( 16 | AzureStorageFileProvider fileProvider, 17 | string delimiter, 18 | string prefix) 19 | { 20 | this.fileProvider = fileProvider; 21 | Delimiter = delimiter; 22 | Prefix = prefix.TrimEnd('/') + "/"; 23 | 24 | Exists = false; 25 | var prefixToSearch = Prefix.TrimEnd('/'); 26 | foreach (var item in fileProvider.BlobContainerClient.GetBlobsByHierarchy( 27 | delimiter: Delimiter, 28 | prefix: prefixToSearch 29 | )) 30 | { 31 | if (item.IsPrefix && item.Prefix == prefixToSearch) 32 | { 33 | Exists = true; 34 | } 35 | } 36 | } 37 | 38 | public bool Exists 39 | { 40 | get; 41 | internal set; 42 | } 43 | 44 | public IEnumerator GetEnumerator() 45 | { 46 | if (Exists) 47 | { 48 | foreach (var item in fileProvider.BlobContainerClient.GetBlobsByHierarchy( 49 | delimiter: Delimiter, 50 | prefix: Prefix 51 | )) 52 | { 53 | yield return new AzureStorageFileInfo(fileProvider, item); 54 | } 55 | } 56 | } 57 | 58 | IEnumerator IEnumerable.GetEnumerator() 59 | { 60 | if (Exists) 61 | { 62 | foreach (var item in fileProvider.BlobContainerClient.GetBlobsByHierarchy( 63 | delimiter: Delimiter, 64 | prefix: Prefix 65 | )) 66 | { 67 | yield return new AzureStorageFileInfo(fileProvider, item); 68 | } 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/DataverseToSql/AzureStorageFileProvider/AzureStorageFileInfo.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Storage.Blobs.Models; 5 | using Microsoft.Extensions.FileProviders; 6 | 7 | namespace AzureStorageFileProvider 8 | { 9 | public class AzureStorageFileInfo : IFileInfo 10 | { 11 | private readonly AzureStorageFileProvider fileProvider; 12 | private readonly BlobHierarchyItem? Item; 13 | 14 | internal AzureStorageFileInfo( 15 | AzureStorageFileProvider fileProvider, 16 | BlobHierarchyItem? item) 17 | { 18 | this.fileProvider = fileProvider; 19 | Item = item; 20 | } 21 | 22 | public bool Exists 23 | { 24 | get 25 | { 26 | return Item is not null; 27 | } 28 | } 29 | 30 | public long Length 31 | { 32 | get 33 | { 34 | if (Item?.IsBlob ?? false) 35 | { 36 | return Item?.Blob?.Properties?.ContentLength ?? 0; 37 | } 38 | else 39 | { 40 | return 0; 41 | } 42 | } 43 | } 44 | 45 | public string? PhysicalPath => null; 46 | 47 | public string Name 48 | { 49 | get 50 | { 51 | if (PhysicalPath is not null) 52 | return PhysicalPath.Split("/").Last(); 53 | else 54 | throw new NullReferenceException(); 55 | } 56 | } 57 | 58 | public DateTimeOffset LastModified 59 | { 60 | get 61 | { 62 | return Item?.Blob.Properties.LastAccessedOn ?? default; 63 | } 64 | } 65 | 66 | public bool IsDirectory 67 | { 68 | get 69 | { 70 | return Item?.IsPrefix ?? false; 71 | } 72 | } 73 | 74 | public Stream CreateReadStream() 75 | { 76 | var blobClient = fileProvider.BlobContainerClient.GetBlobClient(Item.Blob.Name); 77 | return blobClient.OpenRead(); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/DataverseToSql/AzureStorageFileProvider/AzureStorageFileProvider.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Core; 5 | using Azure.Storage.Blobs; 6 | using Microsoft.Extensions.FileProviders; 7 | using Microsoft.Extensions.Primitives; 8 | using System.Text; 9 | 10 | namespace AzureStorageFileProvider 11 | { 12 | public class AzureStorageFileProvider : IFileProvider 13 | { 14 | private readonly string RootPath; 15 | 16 | internal readonly Uri RootUri; 17 | 18 | internal BlobContainerClient BlobContainerClient; 19 | 20 | public AzureStorageFileProvider( 21 | BlobContainerClient blobContainerClient, 22 | string rootPath) 23 | { 24 | this.BlobContainerClient = blobContainerClient; 25 | 26 | RootPath = rootPath; 27 | RootUri = new Uri( 28 | rootPath, 29 | UriKind.Relative 30 | ); 31 | } 32 | 33 | public IDirectoryContents GetDirectoryContents(string subpath) 34 | { 35 | return new AzureStorageDirectoryContents( 36 | this, 37 | delimiter: "/", 38 | prefix: JoinUri(RootPath, subpath) 39 | ); 40 | } 41 | 42 | public IFileInfo GetFileInfo(string subpath) 43 | { 44 | var prefixToSearch = JoinUri(RootPath, subpath); 45 | 46 | foreach (var item in BlobContainerClient.GetBlobsByHierarchy( 47 | delimiter: "/", 48 | prefix: prefixToSearch 49 | )) 50 | { 51 | if (item.IsPrefix && item.Prefix == prefixToSearch) 52 | { 53 | return new AzureStorageFileInfo( 54 | this, 55 | item 56 | ); 57 | } 58 | else if (item.IsBlob && item.Blob.Name == prefixToSearch) 59 | { 60 | return new AzureStorageFileInfo( 61 | this, 62 | item 63 | ); 64 | } 65 | } 66 | 67 | return new AzureStorageFileInfo( 68 | this, 69 | null 70 | ); 71 | } 72 | 73 | public IChangeToken Watch(string filter) 74 | { 75 | throw new NotImplementedException(); 76 | } 77 | 78 | internal string JoinUri(string path1, string path2) 79 | { 80 | var sb = new StringBuilder(); 81 | path1 = path1.Trim().TrimEnd('/'); 82 | if (path1 != "") 83 | { 84 | sb.Append(path1.Trim().TrimEnd('/')); 85 | sb.Append("/"); 86 | } 87 | sb.Append(path2.TrimStart('.').TrimStart('/')); 88 | return sb.ToString(); 89 | } 90 | } 91 | } -------------------------------------------------------------------------------- /src/DataverseToSql/AzureStorageFileProvider/AzureStorageFileProvider.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net6.0 5 | enable 6 | enable 7 | 0.1.0 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/DataverseToSql/BlockBlobClientRangeCopyExtension/BlockBlobClientCopyRangeExtension.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure; 5 | using Azure.Storage.Blobs.Specialized; 6 | using System.Text; 7 | 8 | namespace BlockBlobClientCopyRangeExtension 9 | { 10 | public static class BlockBlobClientCopyRangeExtension 11 | { 12 | /// 13 | /// Performs the copy of a specified range of bytes from a source blob 14 | /// identified by its URI. 15 | /// Data replace any existing content in the target blob; they can optionally 16 | /// be appended using the append argument. 17 | /// If the target blob does not exist, it is created. 18 | /// The method relies on the StageBlockFromUriAsync and 19 | /// CommitBlockListAsync methods of the 20 | /// Azure.Storage.Blobs.Specialized.BlockBlobClient class. 21 | /// The copy is performed using one or more blocks, based on the size of the 22 | /// range to copy and the configured block size. 23 | /// New blocks are committed to the blob every maxUncommittedBlocks 24 | /// uncommitted blocks. 25 | /// 26 | /// 27 | /// Specifies the BlockBlocClient representing the block blob data are 28 | /// being copied to. 29 | /// 30 | /// 31 | /// Specifies the URI of the blob data are copied from. 32 | /// 33 | /// 34 | /// Specifies the authentication details for the source blob. 35 | /// The HttpAuthorization class allows to specify a scheme and the authentication 36 | /// informantion. 37 | /// At the moment only AAD authentication with Bearer scheme is supported. 38 | /// Scheme must be set to "Bearer" and Parameter must be set to a valid AAD token 39 | /// issued for the "https://storage.azure.com/" scope. 40 | /// 41 | /// 42 | /// Specifies the zero-based offset within the source blob of the range 43 | /// of bytes to copy. 44 | /// Must be zero or greater. 45 | /// 46 | /// 47 | /// Specifies the length of the range of bytes to copy. 48 | /// Must be greater than zero. 49 | /// 50 | /// 51 | /// Optionally specifies if the data should be appended to the target 52 | /// block blob, or if it should replace the existing content. 53 | /// Default is false. 54 | /// 55 | /// 56 | /// Optionally specifies the size of blocks used during the copy. 57 | /// Valid values range from 1 B to 4000 MiB. 58 | /// Default is 100 MiB. 59 | /// 60 | /// 61 | /// Optionally specifies the maximum number of uncommitted blocks at 62 | /// any given time. 63 | /// Valid values range from 1 to 50000. 64 | /// Default is 50000. 65 | /// 66 | /// 67 | /// Optional CancellationToken to propagate 68 | /// notifications that the operation should be cancelled. 69 | /// 70 | /// An ArgumentOutOfRangeException will be thrown if 71 | /// arguments are out of bounds. 72 | /// 73 | public static async Task CopyRangeFromUriAsync( 74 | this BlockBlobClient targetBlockBlobClient, 75 | Uri sourceBlobUri, 76 | HttpAuthorization sourceAuthentication, 77 | long offset, 78 | long length, 79 | bool append = false, 80 | long maxBlockSize = 100 * 1024 * 1024, 81 | int maxUncommittedBlocks = 50000, 82 | CancellationToken cancellationToken = default) 83 | { 84 | #region Arguments validation 85 | 86 | if (offset < 0) 87 | { 88 | throw new ArgumentOutOfRangeException( 89 | nameof(offset), 90 | $"{nameof(offset)} cannot be negative." 91 | ); 92 | } 93 | 94 | if (length <= 0) 95 | { 96 | throw new ArgumentOutOfRangeException( 97 | nameof(length), 98 | $"{nameof(length)} must be positive." 99 | ); 100 | } 101 | 102 | const long MIN_MAXBLOCKSIZE = 1; 103 | const long MAX_MAXBLOCKSIZE = 4000L * 1024 * 1024; 104 | 105 | if (maxBlockSize < MIN_MAXBLOCKSIZE 106 | || maxBlockSize > MAX_MAXBLOCKSIZE) 107 | { 108 | throw new ArgumentOutOfRangeException( 109 | nameof(maxBlockSize), 110 | $"{nameof(maxBlockSize)} must be between {MIN_MAXBLOCKSIZE} and {MAX_MAXBLOCKSIZE}." 111 | ); 112 | } 113 | 114 | const int MIN_MAXUNCOMMITEDBLOCKS = 1; 115 | const int MAX_MAXUNCOMMITEDBLOCKS = 50000; 116 | 117 | if (maxUncommittedBlocks < MIN_MAXUNCOMMITEDBLOCKS 118 | || maxUncommittedBlocks > MAX_MAXUNCOMMITEDBLOCKS) 119 | { 120 | throw new ArgumentOutOfRangeException( 121 | nameof(maxUncommittedBlocks), 122 | $"{nameof(maxUncommittedBlocks)} must be between {MIN_MAXUNCOMMITEDBLOCKS} and {MAX_MAXUNCOMMITEDBLOCKS}." 123 | ); 124 | } 125 | 126 | #endregion Arguments validation 127 | 128 | List blockIds; 129 | 130 | // If the append argument is true, retrieve 131 | // the list of committed blocks IDs to preserve them when 132 | // staging the blocks (Put Block List). 133 | // If append is false (default) initialize an empty list 134 | // of block IDs; when the list will be later staged, any 135 | // existing committed block will be removed. 136 | 137 | if (append 138 | && (await targetBlockBlobClient.ExistsAsync(cancellationToken: cancellationToken)).Value) 139 | { 140 | blockIds = (await targetBlockBlobClient 141 | .GetBlockListAsync(cancellationToken: cancellationToken)) 142 | .Value 143 | .CommittedBlocks 144 | .Select(block => block.Name) 145 | .ToList(); 146 | } 147 | else 148 | { 149 | blockIds = new(); 150 | } 151 | 152 | long target = offset + length; 153 | int uncommittedBlocks = 0; 154 | 155 | // Copy the original blob content using one or more blocks, depending 156 | // on its size. 157 | // The size of each block is controlled by the maxBlockSize argument. 158 | // Blocks are assigned a sequential numeric ID starting with 0000000000. 159 | while (offset < target) 160 | { 161 | var blockLength = target - offset; 162 | blockLength = blockLength > maxBlockSize ? maxBlockSize : blockLength; 163 | 164 | var blockId = Convert.ToBase64String( 165 | Encoding.UTF8.GetBytes($"{blockIds.Count:0000000000}")); 166 | 167 | await targetBlockBlobClient.StageBlockFromUriAsync( 168 | sourceBlobUri, 169 | blockId, 170 | new() 171 | { 172 | SourceRange = new(offset, blockLength), 173 | SourceAuthentication = sourceAuthentication 174 | }, 175 | cancellationToken: cancellationToken); 176 | 177 | blockIds.Add(blockId); 178 | uncommittedBlocks++; 179 | 180 | if (uncommittedBlocks == maxUncommittedBlocks) 181 | { 182 | await targetBlockBlobClient.CommitBlockListAsync( 183 | blockIds, 184 | cancellationToken: cancellationToken); 185 | uncommittedBlocks = 0; 186 | } 187 | 188 | offset += blockLength; 189 | } 190 | 191 | if (uncommittedBlocks > 0) 192 | { 193 | await targetBlockBlobClient.CommitBlockListAsync( 194 | blockIds, 195 | cancellationToken: cancellationToken); 196 | } 197 | } 198 | 199 | /// 200 | /// Performs the copy of a specified range of bytes from a source blob 201 | /// identified by its URI. 202 | /// Data replace any existing content in the target blob; they can optionally 203 | /// be appended using the append argument. 204 | /// If the target blob does not exist, it is created. 205 | /// The method relies on the StageBlockFromUri and 206 | /// CommitBlockList methods of the 207 | /// Azure.Storage.Blobs.Specialized.BlockBlobClient class. 208 | /// The copy is performed using one or more blocks, based on the size of the 209 | /// range to copy and the configured block size. 210 | /// New blocks are committed to the blob every maxUncommittedBlocks 211 | /// uncommitted blocks. 212 | /// 213 | /// 214 | /// Specifies the BlockBlocClient representing the block blob data are 215 | /// being copied to. 216 | /// 217 | /// 218 | /// Specifies the URI of the blob data are copied from. 219 | /// 220 | /// 221 | /// Specifies the authentication details for the source blob. 222 | /// The HttpAuthorization class allows to specify a scheme and the authentication 223 | /// informantion. 224 | /// At the moment only AAD authentication with Bearer scheme is supported. 225 | /// Scheme must be set to "Bearer" and Parameter must be set to a valid AAD token 226 | /// issued for the "https://storage.azure.com/" scope. 227 | /// 228 | /// 229 | /// Specifies the zero-based offset within the source blob of the range 230 | /// of bytes to copy. 231 | /// Must be zero or greater. 232 | /// 233 | /// 234 | /// Specifies the length of the range of bytes to copy. 235 | /// Must be greater than zero. 236 | /// 237 | /// 238 | /// Optionally specifies if the data should be appended to the target 239 | /// block blob, or if it should replace the existing content. 240 | /// Default is false. 241 | /// 242 | /// 243 | /// Optionally specifies the size of blocks used during the copy. 244 | /// Valid values range from 1 B to 4000 MiB. 245 | /// Default is 100 MiB. 246 | /// 247 | /// 248 | /// Optionally specifies the maximum number of uncommitted blocks at 249 | /// any given time. 250 | /// Valid values range from 1 to 50000. 251 | /// Default is 50000. 252 | /// 253 | /// 254 | /// Optional CancellationToken to propagate 255 | /// notifications that the operation should be cancelled. 256 | /// 257 | /// An ArgumentOutOfRangeException will be thrown if 258 | /// arguments are out of bounds. 259 | /// 260 | public static void CopyRangeFromUri( 261 | this BlockBlobClient targetBlockBlobClient, 262 | Uri sourceBlobUri, 263 | HttpAuthorization sourceAuthentication, 264 | long offset, 265 | long length, 266 | bool append = false, 267 | long maxBlockSize = 100 * 1024 * 1024, 268 | int maxUncommittedBlocks = 50000, 269 | CancellationToken cancellationToken = default) 270 | { 271 | #region Arguments validation 272 | 273 | if (offset < 0) 274 | { 275 | throw new ArgumentOutOfRangeException( 276 | nameof(offset), 277 | $"{nameof(offset)} cannot be negative." 278 | ); 279 | } 280 | 281 | if (length <= 0) 282 | { 283 | throw new ArgumentOutOfRangeException( 284 | nameof(length), 285 | $"{nameof(length)} must be positive." 286 | ); 287 | } 288 | 289 | const long MIN_MAXBLOCKSIZE = 1; 290 | const long MAX_MAXBLOCKSIZE = 4000L * 1024 * 1024; 291 | 292 | if (maxBlockSize < MIN_MAXBLOCKSIZE 293 | || maxBlockSize > MAX_MAXBLOCKSIZE) 294 | { 295 | throw new ArgumentOutOfRangeException( 296 | nameof(maxBlockSize), 297 | $"{nameof(maxBlockSize)} must be between {MIN_MAXBLOCKSIZE} and {MAX_MAXBLOCKSIZE}." 298 | ); 299 | } 300 | 301 | const int MIN_MAXUNCOMMITEDBLOCKS = 1; 302 | const int MAX_MAXUNCOMMITEDBLOCKS = 50000; 303 | 304 | if (maxUncommittedBlocks < MIN_MAXUNCOMMITEDBLOCKS 305 | || maxUncommittedBlocks > MAX_MAXUNCOMMITEDBLOCKS) 306 | { 307 | throw new ArgumentOutOfRangeException( 308 | nameof(maxUncommittedBlocks), 309 | $"{nameof(maxUncommittedBlocks)} must be between {MIN_MAXUNCOMMITEDBLOCKS} and {MAX_MAXUNCOMMITEDBLOCKS}." 310 | ); 311 | } 312 | 313 | #endregion Arguments validation 314 | 315 | List blockIds; 316 | 317 | // If the append argument is true, retrieve 318 | // the list of committed blocks IDs to preserve them when 319 | // staging the blocks (Put Block List). 320 | // If append is false (default) initialize an empty list 321 | // of block IDs; when the list will be later staged, any 322 | // existing committed block will be removed. 323 | 324 | if (append && targetBlockBlobClient.Exists(cancellationToken: cancellationToken).Value) 325 | { 326 | blockIds = targetBlockBlobClient 327 | .GetBlockList(cancellationToken: cancellationToken).Value 328 | .CommittedBlocks 329 | .Select(block => block.Name) 330 | .ToList(); 331 | } 332 | else 333 | { 334 | blockIds = new(); 335 | } 336 | 337 | long target = offset + length; 338 | int uncommittedBlocks = 0; 339 | 340 | // Copy the original blob content using one or more blocks, depending 341 | // on its size. 342 | // The size of each block is controlled by the maxBlockSize argument. 343 | // Blocks are assigned a sequential numeric ID starting with 0000000000. 344 | while (offset < target) 345 | { 346 | var blockLength = target - offset; 347 | blockLength = blockLength > maxBlockSize ? maxBlockSize : blockLength; 348 | 349 | var blockId = Convert.ToBase64String( 350 | Encoding.UTF8.GetBytes($"{blockIds.Count:0000000000}")); 351 | 352 | targetBlockBlobClient.StageBlockFromUri( 353 | sourceBlobUri, 354 | blockId, 355 | new() 356 | { 357 | SourceRange = new(offset, blockLength), 358 | SourceAuthentication = sourceAuthentication 359 | }, 360 | cancellationToken: cancellationToken); 361 | 362 | blockIds.Add(blockId); 363 | uncommittedBlocks++; 364 | 365 | if (uncommittedBlocks == maxUncommittedBlocks) 366 | { 367 | targetBlockBlobClient.CommitBlockList( 368 | blockIds, 369 | cancellationToken: cancellationToken); 370 | uncommittedBlocks = 0; 371 | } 372 | 373 | offset += blockLength; 374 | } 375 | 376 | if (uncommittedBlocks > 0) 377 | { 378 | targetBlockBlobClient.CommitBlockList( 379 | blockIds, 380 | cancellationToken: cancellationToken); 381 | } 382 | } 383 | } 384 | } -------------------------------------------------------------------------------- /src/DataverseToSql/BlockBlobClientRangeCopyExtension/BlockBlobClientRangeCopyExtension.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net6.0 5 | enable 6 | enable 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Cli/Commands/Deploy.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Identity; 5 | using DataverseToSql.Core.Jobs; 6 | using Microsoft.Extensions.Logging; 7 | using System.CommandLine; 8 | 9 | namespace DataverseToSql.Cli.Commands 10 | { 11 | internal class Deploy 12 | { 13 | internal static Command GetCommandLineCommand(Option pathOption) 14 | { 15 | var command = new Command( 16 | "deploy", 17 | "Deploy the environment." 18 | ); 19 | 20 | command.SetHandler( 21 | Handler, 22 | pathOption 23 | ); 24 | 25 | return command; 26 | } 27 | 28 | internal static void Handler(string path) 29 | { 30 | var logger = Program.loggerFactory.CreateLogger("Deploy"); 31 | 32 | try 33 | { 34 | var environment = new Core.LocalEnvironment( 35 | logger, 36 | new DefaultAzureCredential(), 37 | path 38 | ); 39 | 40 | new DeployJob(logger, environment).Run(); 41 | } 42 | catch (Exception ex) 43 | { 44 | logger.LogError(ex, "{message}", ex.Message); 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Cli/Commands/EntityAdd.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Identity; 5 | using DataverseToSql.Core.Jobs; 6 | using Microsoft.Extensions.Logging; 7 | using System.CommandLine; 8 | 9 | namespace DataverseToSql.Cli.Commands 10 | { 11 | internal class EntityAdd 12 | { 13 | internal static Command GetCommandLineCommand(Option pathOption) 14 | { 15 | var command = new Command( 16 | "add", 17 | "Add entities to the environment." 18 | ); 19 | 20 | var nameOption = new Option( 21 | name: "--name", 22 | description: "Name of the entity to add." 23 | ); 24 | nameOption.AddAlias("-n"); 25 | command.Add(nameOption); 26 | 27 | var allOption = new Option( 28 | name: "--all", 29 | description: "Add all available entities." 30 | ); 31 | allOption.AddAlias("-a"); 32 | command.Add(allOption); 33 | 34 | command.SetHandler( 35 | Handler, 36 | pathOption, 37 | nameOption, 38 | allOption 39 | ); 40 | 41 | command.AddValidator(result => 42 | { 43 | if (result.GetValueForOption(nameOption) is null 44 | && !result.GetValueForOption(allOption)) 45 | { 46 | result.ErrorMessage = "Must specify either --name or --all."; 47 | } 48 | }); 49 | 50 | return command; 51 | } 52 | 53 | internal static async void Handler(string path, string name, bool all) 54 | { 55 | var logger = Program.loggerFactory.CreateLogger("Add"); 56 | 57 | try 58 | { 59 | var environment = new Core.LocalEnvironment( 60 | logger, 61 | new DefaultAzureCredential(), 62 | path 63 | ); 64 | 65 | if (all) 66 | { 67 | var job = new AddAllEntitiesJob(logger, environment); 68 | job.RunAsync().Wait(); 69 | } 70 | else 71 | { 72 | var job = new AddEntityJob(logger, environment, name); 73 | job.RunAsync().Wait(); 74 | } 75 | } 76 | catch (Exception ex) 77 | { 78 | logger.LogError(ex, "{message}", ex.Message); 79 | } 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Cli/Commands/Init.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Microsoft.Extensions.Logging; 5 | using System.CommandLine; 6 | 7 | namespace DataverseToSql.Cli.Commands 8 | { 9 | internal class Init 10 | { 11 | internal static Command GetCommandLineCommand(Option pathOption) 12 | { 13 | var command = new Command( 14 | "init", 15 | "Initialize an environment." 16 | ); 17 | 18 | command.SetHandler( 19 | Handler, 20 | pathOption 21 | ); 22 | 23 | return command; 24 | } 25 | 26 | internal static void Handler(string path) 27 | { 28 | var logger = Program.loggerFactory.CreateLogger("Init"); 29 | 30 | try 31 | { 32 | Core.LocalEnvironment.Init(logger, path); 33 | } 34 | catch (Exception ex) 35 | { 36 | logger.LogError(ex, "{message}", ex.Message); 37 | } 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Cli/DataverseToSql.Cli.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net6.0 6 | enable 7 | enable 8 | dv2sql 9 | 0.1.0 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Cli/Program.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Microsoft.Extensions.Logging; 5 | using System.CommandLine; 6 | using System.Reflection; 7 | 8 | namespace DataverseToSql.Cli 9 | { 10 | class Program 11 | { 12 | internal static ILoggerFactory loggerFactory; 13 | 14 | static Program() 15 | { 16 | loggerFactory = LoggerFactory.Create(builder => 17 | { 18 | builder.AddSimpleConsole(options => 19 | { 20 | options.IncludeScopes = true; 21 | options.SingleLine = true; 22 | options.UseUtcTimestamp = false; 23 | options.TimestampFormat = "yyyy-MM-dd HH:mm:ss.fffzz "; 24 | }); 25 | }); 26 | } 27 | 28 | static int Main(string[] args) 29 | { 30 | var logger = loggerFactory.CreateLogger("Main"); 31 | logger.LogInformation( 32 | "Version: {version}", 33 | Assembly.GetExecutingAssembly().GetName().Version 34 | ); 35 | 36 | try 37 | { 38 | RootCommand rootCommand = InitCommands(); 39 | return rootCommand.Invoke(args); 40 | } 41 | catch (Exception ex) 42 | { 43 | logger.LogError(ex, "{Message}", ex.Message); 44 | return -1; 45 | } 46 | } 47 | 48 | private static RootCommand InitCommands() 49 | { 50 | var rootCommand = new RootCommand("DataverseToSql CLI"); 51 | 52 | var pathOption = new Option( 53 | name: "--path", 54 | description: "Path of the environment.", 55 | getDefaultValue: () => "." 56 | ); 57 | pathOption.AddAlias("-p"); 58 | pathOption.IsRequired = false; 59 | rootCommand.AddGlobalOption(pathOption); 60 | 61 | rootCommand.Add(Commands.Init.GetCommandLineCommand(pathOption)); 62 | rootCommand.Add(Commands.Deploy.GetCommandLineCommand(pathOption)); 63 | rootCommand.Add(Commands.EntityAdd.GetCommandLineCommand(pathOption)); 64 | 65 | return rootCommand; 66 | } 67 | } 68 | } -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Cli/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "profiles": { 3 | "DataverseToSql.Cli": { 4 | "commandName": "Project" 5 | }, 6 | "init": { 7 | "commandName": "Project", 8 | "commandLineArgs": "init -p ./test" 9 | }, 10 | "deploy": { 11 | "commandName": "Project", 12 | "commandLineArgs": "deploy -p ./test" 13 | }, 14 | "add-all": { 15 | "commandName": "Project", 16 | "commandLineArgs": "add --all -p ./test" 17 | }, 18 | "add-non-existing-entity": { 19 | "commandName": "Project", 20 | "commandLineArgs": "add --name non-existing-entity -p ./test" 21 | }, 22 | "add account": { 23 | "commandName": "Project", 24 | "commandLineArgs": "add --name account -p ./test" 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/Auth/DacAuthProvider.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Core; 5 | using Microsoft.SqlServer.Dac; 6 | 7 | namespace DataverseToSql.Core.Auth 8 | { 9 | /// 10 | /// Provides access tokens for DacFx to connect to Azure SQL Database. 11 | /// 12 | internal class DacAuthProvider : IUniversalAuthProvider 13 | { 14 | private readonly TokenCredential _credential; 15 | public DacAuthProvider(TokenCredential credential) 16 | { 17 | _credential = credential; 18 | } 19 | 20 | public string GetValidAccessToken() 21 | { 22 | return _credential.GetToken( 23 | new(new[] { "https://database.windows.net/.default" }) 24 | , default 25 | ).Token; 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/BlobLock.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Storage.Blobs; 5 | using Azure.Storage.Blobs.Specialized; 6 | 7 | namespace DataverseToSql.Core 8 | { 9 | /// 10 | /// Provides a lock based on Azure Blob Storage. 11 | /// The lock is acquired and held via a timer. 12 | /// 13 | internal class BlobLock 14 | { 15 | private BlobLeaseClient blobLeaseClient; 16 | private TimeSpan leaseDuration; 17 | private System.Timers.Timer timer; 18 | 19 | internal BlobLock( 20 | BlobClient blobClient, 21 | TimeSpan? leaseDuration = null 22 | ) 23 | { 24 | if (!blobClient.Exists()) 25 | { 26 | blobClient.Upload(new BinaryData("lock")); 27 | } 28 | 29 | blobLeaseClient = blobClient.GetBlobLeaseClient(); 30 | 31 | this.leaseDuration = leaseDuration ?? TimeSpan.FromSeconds(30); 32 | 33 | if (this.leaseDuration.TotalSeconds < 15.0 34 | || this.leaseDuration.TotalSeconds > 60.0) 35 | { 36 | throw new ArgumentOutOfRangeException( 37 | "leaseDuration", 38 | "leaseDuration must be between 15 and 60 seconds." 39 | ); 40 | } 41 | 42 | timer = new(); 43 | timer.Interval = this.leaseDuration.TotalMilliseconds / 2; 44 | timer.AutoReset = true; 45 | timer.Elapsed += (sender, args) => 46 | { 47 | blobLeaseClient.Renew(); 48 | }; 49 | } 50 | 51 | internal bool TryAcquire() 52 | { 53 | try 54 | { 55 | blobLeaseClient.Acquire(leaseDuration); 56 | timer.Start(); 57 | return true; 58 | } 59 | catch (Azure.RequestFailedException ex) 60 | { 61 | if (ex.ErrorCode == "LeaseAlreadyPresent") 62 | { 63 | return false; 64 | } 65 | throw; 66 | } 67 | } 68 | 69 | internal void Release() 70 | { 71 | timer.Stop(); 72 | blobLeaseClient.Release(); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/CdmModel/CdmAttribute.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using System.Collections; 5 | 6 | namespace DataverseToSql.Core.CdmModel 7 | { 8 | /// 9 | /// Represents a CDM entity attribute as deserialized from a manifest. 10 | /// The class is limited to the fields needed for the project. 11 | /// 12 | internal class CdmAttribute 13 | { 14 | internal readonly string Name; 15 | internal readonly string DataType; 16 | internal readonly int MaxLength; 17 | internal readonly int Precision = -1; 18 | internal readonly int Scale = -1; 19 | internal string? CustomSqlDatatype = null; 20 | 21 | internal CdmAttribute(dynamic attribute) 22 | { 23 | Name = attribute.name; 24 | DataType = attribute.dataType; 25 | MaxLength = attribute.maxLength; 26 | 27 | if (DataType == "decimal") 28 | { 29 | var arguments = ((IEnumerable)attribute["cdm:traits"]) 30 | .Cast() 31 | .Where(a => a.traitReference == "is.dataFormat.numeric.shaped") 32 | .First().arguments; 33 | 34 | var argdict = ((IEnumerable)arguments) 35 | .Cast() 36 | .ToDictionary(a => (string)a.name, a => (int)a.value); 37 | 38 | Precision = argdict["precision"]; 39 | Scale = argdict["scale"]; 40 | } 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/CdmModel/CdmEntity.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using System.Collections; 5 | 6 | namespace DataverseToSql.Core.CdmModel 7 | { 8 | /// 9 | /// Represents a CDM entity as deserialized from a manifest. 10 | /// The class is limited to the fields needed for the project. 11 | /// 12 | internal class CdmEntity 13 | { 14 | internal readonly string Name; 15 | internal readonly IList Attributes; 16 | internal readonly IList Partitions; 17 | internal readonly string PartitionGranularity; 18 | internal readonly string InitialSyncState; 19 | 20 | private IList? _primaryKeyAttributes = null; 21 | 22 | internal CdmEntity(dynamic entity) 23 | { 24 | Name = entity.name; 25 | Attributes = ((IEnumerable)entity.attributes).Cast().Select(a => new CdmAttribute(a)).ToList(); 26 | Partitions = ((IEnumerable)entity.partitions).Cast().Select(a => new CdmPartition(a)).ToList(); 27 | 28 | PartitionGranularity = ((IEnumerable)entity.annotations) 29 | .Cast() 30 | .Where(a => a.name == "Athena:PartitionGranularity") 31 | .Select(a => a.value).FirstOrDefault("Unknown"); 32 | 33 | InitialSyncState = ((IEnumerable)entity.annotations) 34 | .Cast() 35 | .Where(a => a.name == "Athena:InitialSyncState") 36 | .Select(a => a.value).FirstOrDefault("Unknown"); 37 | } 38 | 39 | // Return the set of fields composing the primary key of the entity. 40 | // At the moment it returns Id column only. 41 | internal IList PrimaryKeyAttributes 42 | { 43 | get 44 | { 45 | if (_primaryKeyAttributes is null) 46 | { 47 | CdmAttribute? idAttr = null; 48 | 49 | foreach (var attr in Attributes) 50 | { 51 | if (attr.Name.ToLower() == "id") 52 | { 53 | idAttr = attr; 54 | } 55 | } 56 | 57 | if (idAttr is not null) 58 | { 59 | _primaryKeyAttributes = new List() { idAttr }; 60 | } 61 | else 62 | { 63 | throw new Exception($"Entity {Name} contains no primary key or Id attribute."); 64 | } 65 | } 66 | 67 | return _primaryKeyAttributes; 68 | } 69 | } 70 | 71 | // Return set of partitions grouped by name. 72 | // Partitions belonging to the same period are grouped together. 73 | // E.g. "2018" and "2018_001" are grouped together under 2018. 74 | internal IList PartitionGroups 75 | { 76 | get 77 | { 78 | return Partitions 79 | .GroupBy(p => p.Name.Split('_')[0]) 80 | .Select(g => new CdmPartitionGroup( 81 | name: g.Key, 82 | partitions: g.ToList())) 83 | .ToList(); 84 | } 85 | } 86 | 87 | // Determines if the entity has a primary key. 88 | internal bool HasPrimaryKey { get => PrimaryKeyAttributes.Count > 0; } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/CdmModel/CdmManifest.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using System.Collections; 5 | 6 | namespace DataverseToSql.Core.CdmModel 7 | { 8 | /// 9 | /// Represents a CDM manifest as deserialized from a manifest JSON file. 10 | /// The class is limited to the fields needed for the project. 11 | /// 12 | internal class CdmManifest 13 | { 14 | internal readonly IList Entities; 15 | 16 | internal CdmManifest(dynamic model) 17 | { 18 | try 19 | { 20 | if (!(model.name == "cdm" && model.version == "1.0")) 21 | { 22 | throw new Exception("Unsupported model format."); 23 | } 24 | 25 | Entities = ((IEnumerable)model.entities).Cast().Select(e => new CdmEntity(e)).ToList(); 26 | } 27 | catch (Exception ex) 28 | { 29 | throw new Exception("Failed to load model.", ex); 30 | } 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/CdmModel/CdmPartition.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | namespace DataverseToSql.Core.CdmModel 5 | { 6 | /// 7 | /// Represents a CDM entity partition as deserialized from a manifest. 8 | /// The class is limited to the fields needed for the project. 9 | /// 10 | internal class CdmPartition 11 | { 12 | internal readonly string Name; 13 | internal readonly string Location; 14 | 15 | internal CdmPartition(dynamic partition) 16 | { 17 | Name = partition.name; 18 | Location = partition.location; 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/CdmModel/CdmPartitionGroup.cs: -------------------------------------------------------------------------------- 1 | namespace DataverseToSql.Core.CdmModel 2 | { 3 | internal class CdmPartitionGroup 4 | { 5 | internal readonly string Name; 6 | internal readonly IList Partitions; 7 | 8 | internal CdmPartitionGroup(string name, IList partitions) 9 | { 10 | Name = name; 11 | Partitions = partitions; 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/CdmSqlExtensions.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using DataverseToSql.Core.CdmModel; 5 | using System.Text; 6 | 7 | namespace DataverseToSql.Core 8 | { 9 | /// 10 | /// Provides extension methods to CDM objects for generating SQL code. 11 | /// 12 | internal static class CdmSqlExtensions 13 | { 14 | private const string SQL_CODE_INDENT = " "; 15 | private const string DV_SCHEMA = "DataverseToSql"; 16 | 17 | // Return the scripts of the SQL objects representing the CDM entity. 18 | // The scripts include: 19 | // - The CREATE TABLE statement of the target table. 20 | // - The CREATE TYPE statement for the creation of the table type required 21 | // by the merge stored procedure. 22 | // - The CREATE PROCEDURE statement of the merge stored procedure invoked by 23 | // the Copy activity. 24 | internal static string SqlScripts(this CdmEntity cdmEntity, string schema, bool skipIsDeleteColumn) 25 | { 26 | var sb = new StringBuilder(); 27 | sb.AppendLine(cdmEntity.TableScript(schema, 28 | attributeFilter: a => !skipIsDeleteColumn || a.Name.ToLower() != "isdelete")); 29 | sb.AppendLine(cdmEntity.TableTypeScript( 30 | attributeFilter: a => !skipIsDeleteColumn || a.Name.ToLower() != "isdelete")); 31 | sb.AppendLine(cdmEntity.MergeProcedureScript(schema, 32 | attributeFilter: a => !skipIsDeleteColumn || a.Name.ToLower() != "isdelete")); 33 | return sb.ToString(); 34 | } 35 | 36 | // Return the CREATE TABLE statement of the target table. 37 | internal static string TableScript(this CdmEntity entity, string schema, Func attributeFilter) 38 | { 39 | var sb = new StringBuilder(); 40 | 41 | sb.AppendLine( 42 | $"CREATE TABLE [{schema}].[{entity.Name}] ("); 43 | 44 | sb.Append(SQL_CODE_INDENT); 45 | sb.AppendJoin( 46 | $",\n{SQL_CODE_INDENT}", 47 | entity.Attributes 48 | .Where(attributeFilter) 49 | .Select(attr => attr.SqlColumnDef()) 50 | ); ; 51 | 52 | 53 | if (entity.HasPrimaryKey) 54 | { 55 | var primaryKeyCols = string.Join( 56 | ",", 57 | entity.PrimaryKeyAttributes.Select(attr => attr.SqlColumnName())); 58 | 59 | sb.Append( 60 | $",\n{SQL_CODE_INDENT}CONSTRAINT [PK_{entity.Name}] PRIMARY KEY({primaryKeyCols})"); 61 | } 62 | 63 | sb.AppendLine(); 64 | sb.AppendLine(");"); 65 | sb.AppendLine("GO"); 66 | 67 | return sb.ToString(); 68 | } 69 | 70 | // Return he CREATE TYPE statement for the creation of the table type required 71 | // by the merge stored procedure. 72 | internal static string TableTypeScript(this CdmEntity entity, Func attributeFilter) 73 | { 74 | var sb = new StringBuilder(); 75 | 76 | sb.AppendLine($"CREATE TYPE [{DV_SCHEMA}].{entity.TableTypeName()} AS TABLE ("); 77 | 78 | sb.AppendJoin($",\n", entity.Attributes 79 | //.Where(attributeFilter) 80 | .Select(attr => $"{attr.SqlColumnDef()} NULL")); 81 | 82 | sb.AppendLine(");"); 83 | sb.AppendLine("GO"); 84 | 85 | return sb.ToString(); 86 | } 87 | 88 | // Return the CREATE PROCEDURE statement of the merge stored procedure invoked by 89 | // the Copy activity. 90 | internal static string MergeProcedureScript(this CdmEntity entity, string schema, Func attributeFilter) 91 | { 92 | var sb = new StringBuilder(); 93 | 94 | var primaryKeyCols = entity.PrimaryKeyAttributes.Select(a => a.SqlColumnName()).ToList(); 95 | 96 | var updateExpressions = new List(); 97 | var insertTargetColumns = new List(); 98 | var insertSourceColumns = new List(); 99 | 100 | foreach (var attr in entity.Attributes.Where(attributeFilter)) 101 | { 102 | if (!entity.PrimaryKeyAttributes.Contains(attr)) 103 | { 104 | updateExpressions.Add($"{attr.SqlColumnName()} = source.{attr.SqlColumnName()}"); 105 | } 106 | insertTargetColumns.Add(attr.SqlColumnName()); 107 | insertSourceColumns.Add($"source.{attr.SqlColumnName()}"); 108 | } 109 | 110 | sb.AppendLine($@" 111 | CREATE PROCEDURE [{DV_SCHEMA}].{entity.MergeProcName()} 112 | @entity [{DV_SCHEMA}].{entity.TableTypeName()} READONLY 113 | AS 114 | MERGE [{schema}].[{entity.Name}] AS target 115 | USING @entity AS source 116 | ON {string.Join(" AND ", primaryKeyCols.Select(c => $"source.{c} = target.{c}"))} 117 | WHEN MATCHED AND source.IsDelete = 1 THEN 118 | DELETE 119 | WHEN MATCHED AND ISNULL(source.IsDelete, 0) = 0 AND source.[SinkModifiedOn] > target.[SinkModifiedOn] THEN 120 | UPDATE SET {string.Join(",", updateExpressions)} 121 | WHEN NOT MATCHED THEN 122 | INSERT ({string.Join(",", insertTargetColumns)}) 123 | VALUES ({string.Join(",", insertSourceColumns)}); 124 | GO"); 125 | 126 | return sb.ToString(); 127 | } 128 | 129 | public static string GetServerlessInnerQuery( 130 | this CdmEntity entity, 131 | IList targetColumns) 132 | { 133 | var sourceColumns = string.Join(",", entity.Attributes.Select(attr => attr.SqlColumnDef(serverless: true))); 134 | var primaryKeyCols = entity.PrimaryKeyAttributes.Select(a => a.SqlColumnName()).ToList(); 135 | var primaryKeyString = string.Join(",", primaryKeyCols); 136 | var primaryKeyJoinPredicates = string.Join(" AND ", primaryKeyCols.Select(c => $"s.{c} = r.{c}")); 137 | var targetColumnList = string.Join(",", targetColumns.Select(c => $"[{c}]")); 138 | 139 | var innerQuery = $@" 140 | WITH cte_openrowset AS ( 141 | SELECT * 142 | FROM OPENROWSET(BULK ( N'<<>>' ), 143 | FORMAT = 'csv', FIELDTERMINATOR = ',', FIELDQUOTE = '""') 144 | WITH ({sourceColumns}) AS T1 145 | WHERE T1.filepath(1) >= '<<>>' 146 | AND T1.filepath(1) <= '<<>>' 147 | ), 148 | cte_rownumber AS ( 149 | SELECT ROW_NUMBER() OVER (PARTITION BY {primaryKeyString} ORDER BY [SinkModifiedOn] DESC, [versionnumber] DESC) [DvRowNumber], 150 | * 151 | FROM cte_openrowset 152 | ) 153 | SELECT DISTINCT 154 | {targetColumnList} 155 | FROM 156 | cte_rownumber 157 | WHERE 158 | DvRowNumber = 1 159 | "; 160 | 161 | return innerQuery; 162 | } 163 | 164 | // Return the SQL column definition of the CDM attribute 165 | // in the format [] [] 166 | internal static string SqlColumnDef(this CdmAttribute attr, bool serverless = false) 167 | => $"{attr.SqlColumnName()} {attr.SqlDataType(serverless)}"; 168 | 169 | // Return the formated SQL column name of the CDM attribute 170 | // in the format [] 171 | internal static string SqlColumnName(this CdmAttribute attr) => $"[{attr.Name}]"; 172 | 173 | // Return the SQL data type of the CDM attribute 174 | internal static string SqlDataType(this CdmAttribute attr, bool serverless = false) 175 | { 176 | if (!serverless && attr.CustomSqlDatatype is not null) 177 | return attr.CustomSqlDatatype; 178 | 179 | return attr.DataType.ToLower() switch 180 | { 181 | "binary" => "varbinary(max)", 182 | "boolean" => "bit", 183 | "byte" => "tinyint", 184 | "char" when attr.MaxLength == -1 => "nchar(100)", 185 | "char" when attr.MaxLength <= 4000 => $"nchar({attr.MaxLength})", 186 | "char" => "nvarchar(max)", 187 | "date" => "date", 188 | "datetime" => "datetime2", 189 | "datetimeoffset" => "datetimeoffset", 190 | "decimal" => $"decimal({attr.Precision},{attr.Scale})", 191 | "double" => "float", 192 | "float" => "float", 193 | "guid" => "uniqueidentifier", 194 | "int16" => "smallint", 195 | "int32" => "int", 196 | "int64" => "bigint", 197 | "integer" => "int", 198 | "json" => "nvarchar(max)", 199 | "long" => "bigint", 200 | "short" => "smallint", 201 | "string" when attr.MaxLength == -1 || attr.MaxLength > 4000 => "nvarchar(max)", 202 | "string" => $"nvarchar({attr.MaxLength})", 203 | "time" => "time", 204 | "timestamp" => "datetime2", 205 | _ => "nvarchar(max)" 206 | }; 207 | } 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/CloudEnvironment.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Core; 5 | using Azure.Storage.Blobs; 6 | using Microsoft.Extensions.Logging; 7 | 8 | namespace DataverseToSql.Core 9 | { 10 | /// 11 | /// Derived class of EnvironmentBase that works with configuration from an Azure Storage Account. 12 | /// 13 | public class CloudEnvironment : EnvironmentBase 14 | { 15 | public CloudEnvironment( 16 | ILogger log, 17 | TokenCredential credential, 18 | BlobContainerClient blobContainerClient) 19 | : base(log, credential, ReadConfig(log, blobContainerClient)) 20 | { } 21 | 22 | private static EnvironmentConfiguration ReadConfig( 23 | ILogger log, 24 | BlobContainerClient blobContainerClient) 25 | { 26 | log.LogInformation( 27 | "Loading environment from {containerUri}", 28 | blobContainerClient.Uri 29 | ); 30 | 31 | try 32 | { 33 | AzureStorageFileProvider.AzureStorageFileProvider afp = new( 34 | blobContainerClient, 35 | ""); 36 | 37 | return new(CONFIG_FILE, afp); 38 | } 39 | catch (Exception ex) 40 | { 41 | throw new Exception( 42 | $"Cannot load configuration: {ex.Message}", ex); 43 | } 44 | } 45 | 46 | internal override async Task GetCustomDatatypeMapReader(CancellationToken cancellationToken) 47 | { 48 | var blobUriBuilder = new BlobUriBuilder(Config.ConfigurationStorage.ContainerUri()) 49 | { 50 | BlobName = CUSTOM_DATATYPE_MAP 51 | }; 52 | 53 | var blobClient = new BlobClient(blobUriBuilder.ToUri(), Credential); 54 | 55 | if (!await blobClient.ExistsAsync(cancellationToken)) 56 | return null; 57 | 58 | log.LogInformation("Loading custom data type map from {path}.", blobUriBuilder.ToUri()); 59 | 60 | return new StreamReader(await blobClient.OpenReadAsync(default, cancellationToken)); 61 | } 62 | 63 | internal override async Task> InitCustomScriptsAsync(CancellationToken cancellationToken) 64 | { 65 | List<(string name, string script)> customScripts = new(); 66 | 67 | var containerClient = new BlobContainerClient( 68 | Config.ConfigurationStorage.ContainerUri(), 69 | Credential); 70 | 71 | await foreach (var item in containerClient.GetBlobsByHierarchyAsync( 72 | prefix: $"{CUSTOM_SQL_OBJECTS_FOLDER}/", 73 | cancellationToken: cancellationToken)) 74 | { 75 | if (item.IsBlob && item.Blob.Properties.ContentLength > 0) 76 | { 77 | var blobClient = containerClient.GetBlobClient(item.Blob.Name); 78 | var stream = await blobClient.OpenReadAsync(cancellationToken: cancellationToken); 79 | var streamReader = new StreamReader(stream); 80 | var script = await streamReader.ReadToEndAsync(); 81 | customScripts.Add((item.Blob.Name, script)); 82 | } 83 | } 84 | 85 | return customScripts; 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/DataverseToSql.Core.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net6.0 5 | enable 6 | enable 7 | 0.1.0 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | True 38 | True 39 | Notebooks.resx 40 | 41 | 42 | True 43 | True 44 | SqlObjects.resx 45 | 46 | 47 | 48 | 49 | 50 | ResXFileCodeGenerator 51 | Notebooks.Designer.cs 52 | 53 | 54 | ResXFileCodeGenerator 55 | SqlObjects.Designer.cs 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/EnvironmentBase.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Core; 5 | using Azure.Storage.Blobs; 6 | using DataverseToSql.Core.CdmModel; 7 | using DataverseToSql.Core.Model; 8 | using Microsoft.Extensions.Logging; 9 | using Newtonsoft.Json.Linq; 10 | using System.Collections.Concurrent; 11 | 12 | namespace DataverseToSql.Core 13 | { 14 | /// 15 | /// Provides: 16 | /// - configuration information 17 | /// - access to the CDM repository 18 | /// - access to the managed entities 19 | /// - access to the underlying database 20 | /// 21 | public abstract class EnvironmentBase 22 | { 23 | internal const string CONFIG_FILE = "DataverseToSql.json"; 24 | internal const string CUSTOM_SQL_OBJECTS_FOLDER = "CustomSqlObjects"; 25 | internal const string CUSTOM_DATATYPE_MAP = "CustomDatatypeMap.tsv"; 26 | 27 | protected readonly ILogger log; 28 | internal readonly TokenCredential Credential; 29 | internal readonly EnvironmentConfiguration Config; 30 | 31 | internal readonly Database database; 32 | 33 | private IDictionary? _cdmEntityDict = null; 34 | private IDictionary? _managedEntityDict = null; 35 | private IList<(string name, string script)>? _customScripts = null; 36 | private Dictionary? _managedCustomScriptsDict = null; 37 | private Dictionary>? _optionSets; 38 | private Dictionary>? _customDatatypeMap = null; 39 | 40 | public EnvironmentBase( 41 | ILogger log, 42 | TokenCredential credential, 43 | EnvironmentConfiguration config) 44 | { 45 | this.log = log; 46 | Credential = credential; 47 | Config = config; 48 | 49 | database = new Database( 50 | this.log, 51 | Config.Database, 52 | Credential); 53 | } 54 | 55 | // Return an interable over the blob names under Microsoft.Athena.TrickleFeedService 56 | // in the storage container containing Dataverse data. 57 | // Return all blobs whose name ends with "-model.json". 58 | // The *-model.json blobs contain the manifest of individual entities. 59 | internal async Task> GetCdmEntityModelBlobNamesAsync(CancellationToken cancellationToken) 60 | { 61 | List result = new(); 62 | 63 | var containerClient = new BlobContainerClient( 64 | Config.DataverseStorage.ContainerUri(), 65 | Credential); 66 | 67 | await foreach (var item in containerClient.GetBlobsByHierarchyAsync( 68 | delimiter: "/", 69 | prefix: "/Microsoft.Athena.TrickleFeedService/", 70 | cancellationToken: cancellationToken)) 71 | { 72 | if (item.IsBlob && item.Blob.Name.EndsWith("-model.json")) 73 | { 74 | result.Add(item.Blob.Name); 75 | } 76 | } 77 | 78 | return result; 79 | } 80 | 81 | // Dictionary of available CDM entities indexed by name (lowercase). 82 | internal async Task> GetCdmEntityDictAsync(CancellationToken cancellationToken) 83 | => (_cdmEntityDict ??= await InitCdmEntitiesDictAsync(cancellationToken)); 84 | 85 | // Enumerable of available CDM entities. 86 | internal async Task> GetCdmEntitiesAsync(CancellationToken cancellationToken) 87 | => (await GetCdmEntityDictAsync(cancellationToken)).Values; 88 | 89 | // Initialize the dictionary of CDM entities. 90 | // The dictionary contains only entities with 91 | // "Completed" InitialSyncState. 92 | internal async Task> InitCdmEntitiesDictAsync(CancellationToken cancellationToken) 93 | { 94 | log.LogInformation("Loading CDM entities."); 95 | 96 | var dict = new ConcurrentDictionary(); 97 | 98 | // Iterate over the *-model.json blobs under Microsoft.Athena.TrickleFeedService 99 | // in the container with Dataverse data. 100 | 101 | if (!int.TryParse(Environment.GetEnvironmentVariable("ASYNC_PARALLELISM"), out int parallelism)) 102 | parallelism = 16; 103 | 104 | var parallelOpts = new ParallelOptions() 105 | { 106 | MaxDegreeOfParallelism = parallelism, 107 | CancellationToken = cancellationToken 108 | }; 109 | 110 | // force loading optionset data 111 | await GetOptionSetAsync(cancellationToken); 112 | 113 | await LoadCustomDatatypeMap(cancellationToken); 114 | 115 | await Parallel.ForEachAsync(await GetCdmEntityModelBlobNamesAsync(cancellationToken), 116 | parallelOpts, 117 | async (blobName, ct) => 118 | { 119 | var uriBuilder = new BlobUriBuilder(Config.DataverseStorage.ContainerUri()) 120 | { 121 | BlobName = blobName 122 | }; 123 | 124 | var blobUri = uriBuilder.ToUri(); 125 | 126 | var blobClient = new BlobClient( 127 | blobUri, 128 | Credential); 129 | 130 | // Deserialize the manifest 131 | try 132 | { 133 | var reader = new StreamReader(await blobClient.OpenReadAsync(default, ct)); 134 | var cdmManifest = new CdmManifest(JObject.Parse(await reader.ReadToEndAsync())); 135 | 136 | foreach (var cdmEntity in cdmManifest.Entities) 137 | { 138 | // Add only entities whose InitialSyncState is "Completed" 139 | if (cdmEntity.InitialSyncState == "Completed") 140 | { 141 | await OverrideDatatypes(cdmEntity, cancellationToken); 142 | 143 | dict[cdmEntity.Name.ToLower()] = cdmEntity; 144 | } 145 | } 146 | } 147 | catch (Exception ex) 148 | { 149 | log.LogWarning( 150 | "Failed to deserialized manifest {manifestUri} with error {message} " + 151 | "The ingestion of the entity may be still in progress; " + 152 | "the manifest may require up to one hour to be updated.", 153 | blobUri, 154 | ex.Message); 155 | } 156 | }); 157 | 158 | return dict; 159 | } 160 | 161 | private async Task LoadCustomDatatypeMap(CancellationToken cancellationToken) 162 | { 163 | var reader = await GetCustomDatatypeMapReader(cancellationToken); 164 | 165 | if (reader is null) 166 | { 167 | log.LogInformation("Custom data type map file is not present."); 168 | return; 169 | } 170 | 171 | try 172 | { 173 | _customDatatypeMap = new(); 174 | 175 | string? line; 176 | int lineCount = 0; 177 | while ((line = await reader.ReadLineAsync()) != null) 178 | { 179 | lineCount++; 180 | var vals = line.Split('\t').Select(v => v.Trim().ToLower()).ToList(); 181 | 182 | if (vals.Count != 3) 183 | throw new Exception($"Unexpected number of fields in the custom data type map file on line {lineCount}"); 184 | 185 | for (int i = 0; i < 3; i++) 186 | { 187 | if (vals[i] == "") 188 | throw new Exception($"Empty field {i + 1} on line {lineCount} in the custom data type map file"); 189 | } 190 | 191 | var table = vals[0]; 192 | var column = vals[1]; 193 | var datatype = vals[2]; 194 | 195 | try 196 | { 197 | SqlBuiltinDataTypeValidator.Validate(datatype); 198 | } 199 | catch (Exception ex) 200 | { 201 | throw new Exception( 202 | $"Invalid data type on line {lineCount} in the custom data type map file: {ex.Message}", 203 | ex); 204 | } 205 | 206 | if (!_customDatatypeMap.ContainsKey(table)) 207 | _customDatatypeMap[table] = new(); 208 | 209 | if (_customDatatypeMap[table].ContainsKey(column)) 210 | throw new Exception($"Duplicate column {column} in table {table} in the custom data type map file"); 211 | 212 | _customDatatypeMap[table][column] = datatype; 213 | } 214 | } 215 | catch (Exception) 216 | { 217 | throw; 218 | } 219 | finally 220 | { 221 | reader.Close(); 222 | } 223 | } 224 | 225 | internal abstract Task GetCustomDatatypeMapReader(CancellationToken cancellationToken); 226 | 227 | // Override the datatype of an entity attributes if necessary 228 | // e.g. when wanting to use int instead of bigint for optionsets. 229 | private async Task OverrideDatatypes(CdmEntity cdmEntity, CancellationToken cancellationToken) 230 | { 231 | 232 | // if OptionSetInt32 is true, override the SQL data type 233 | // of int64 optionset fields to int, instead of bigint 234 | if (Config.SchemaHandling.OptionSetInt32 || _customDatatypeMap is not null) 235 | { 236 | var entityName = cdmEntity.Name.ToLower(); 237 | var optionSets = await GetOptionSetAsync(cancellationToken); 238 | 239 | foreach (var attribute in cdmEntity.Attributes) 240 | { 241 | var attributeName = attribute.Name.ToLower(); 242 | 243 | // override data type if OptionSetInt32 option is set 244 | if (Config.SchemaHandling.OptionSetInt32 245 | && (attributeName == "statecode" 246 | || attributeName == "statuscode" 247 | || (optionSets.ContainsKey(entityName) && optionSets[entityName].Contains(attributeName))) 248 | && attribute.DataType.ToLower() == "int64") 249 | { 250 | attribute.CustomSqlDatatype = "int"; 251 | } 252 | 253 | // override data type if there is a match in the custom data type map 254 | if (_customDatatypeMap is not null 255 | && _customDatatypeMap.ContainsKey(entityName) 256 | && _customDatatypeMap[entityName].ContainsKey(attributeName)) 257 | { 258 | attribute.CustomSqlDatatype = _customDatatypeMap[entityName][attributeName]; 259 | } 260 | } 261 | } 262 | } 263 | 264 | private async Task>> GetOptionSetAsync(CancellationToken cancellationToken) 265 | { 266 | if (_optionSets is null) 267 | { 268 | _optionSets = new(); 269 | 270 | var uriBuilder = new BlobUriBuilder(Config.DataverseStorage.ContainerUri()) 271 | { 272 | BlobName = "OptionsetMetadata/GlobalOptionsetMetadata.csv" 273 | }; 274 | 275 | var blobUri = uriBuilder.ToUri(); 276 | 277 | var blobClient = new BlobClient( 278 | blobUri, 279 | Credential); 280 | 281 | using var globalOptionSetMetadatareader = new StreamReader(await blobClient.OpenReadAsync(default, cancellationToken)); 282 | while (true) 283 | { 284 | var line = await globalOptionSetMetadatareader.ReadLineAsync(); 285 | if (line is null) break; 286 | 287 | var globalOptionSetFields = line.Split(','); 288 | var table = globalOptionSetFields[6].ToLower(); 289 | var field = globalOptionSetFields[0].ToLower(); 290 | 291 | if (!_optionSets.ContainsKey(table)) 292 | { 293 | _optionSets[table] = new(); 294 | } 295 | 296 | _optionSets[table].Add(field); 297 | } 298 | 299 | uriBuilder = new BlobUriBuilder(Config.DataverseStorage.ContainerUri()) 300 | { 301 | BlobName = "OptionsetMetadata/OptionsetMetadata.csv" 302 | }; 303 | 304 | blobUri = uriBuilder.ToUri(); 305 | 306 | blobClient = new BlobClient( 307 | blobUri, 308 | Credential); 309 | 310 | using var optionSetMetadatareader = new StreamReader(await blobClient.OpenReadAsync(default, cancellationToken)); 311 | while (true) 312 | { 313 | var line = await optionSetMetadatareader.ReadLineAsync(); 314 | if (line is null) break; 315 | 316 | var globalOptionSetFields = line.Split(','); 317 | var table = globalOptionSetFields[0].ToLower(); 318 | var field = globalOptionSetFields[1].ToLower(); 319 | 320 | if (!_optionSets.ContainsKey(table)) 321 | { 322 | _optionSets[table] = new(); 323 | } 324 | 325 | _optionSets[table].Add(field); 326 | } 327 | } 328 | 329 | return _optionSets; 330 | } 331 | 332 | // Dictionary of managed entities indexed by name (lowercase). 333 | internal async Task> GetManagedEntityDictAsync(CancellationToken cancellationToken) 334 | { 335 | return _managedEntityDict ??= 336 | (await database.GetManagedEntitiesAsync(cancellationToken)) 337 | .ToDictionary(e => e.Name.ToLower(), e => e); 338 | } 339 | 340 | // Enumerable of managed entities. 341 | internal async Task> GetManagedEntitiesAsync(CancellationToken cancellationToken) 342 | => (await GetManagedEntityDictAsync(cancellationToken)).Values; 343 | 344 | // Try to retrieve a CDM entity based on its name 345 | internal async Task<(bool found, CdmEntity? cdmEntity)> TryGetCdmEntityAsync(string name, CancellationToken cancellationToken) 346 | { 347 | if ((await GetCdmEntityDictAsync(cancellationToken)).TryGetValue(name.ToLower(), out var cdmEntity)) 348 | { 349 | return (true, cdmEntity); 350 | } 351 | return (false, null); 352 | } 353 | 354 | // Try to retrieve a CDM entity based on the name of a given managed entity 355 | internal async Task<(bool found, CdmEntity? cdmEntity)> TryGetCdmEntityAsync(ManagedEntity managedEntity, CancellationToken cancellationToken) 356 | => await TryGetCdmEntityAsync(managedEntity.Name, cancellationToken); 357 | 358 | // Determines if the given entity (by name) is managed or not. 359 | internal async Task IsManagedEntityAsync(string name, CancellationToken cancellationToken) 360 | => (await GetManagedEntityDictAsync(cancellationToken)).ContainsKey(name.ToLower()); 361 | 362 | // Determines if the given entity (based on the name of a given CDM entity) is managed or not. 363 | internal async Task IsManagedEntityAsync(CdmEntity cdmEntity, CancellationToken cancellationToken) 364 | => await IsManagedEntityAsync(cdmEntity.Name, cancellationToken); 365 | 366 | // Return all custom scripts contained in the environment. 367 | internal async Task> GetCustomScriptsAsync(CancellationToken cancellationToken) 368 | { 369 | return _customScripts ??= (await InitCustomScriptsAsync(cancellationToken)); 370 | } 371 | 372 | // Initialize list of custom scripts from storage. 373 | internal abstract Task> InitCustomScriptsAsync(CancellationToken cancellationToken); 374 | 375 | // Return a managed custom script by name. Return null if not found. 376 | internal async Task GetManagedCustomScript(string name, CancellationToken cancellationToken) 377 | { 378 | (await GetManagedCustomScriptDict(cancellationToken)).TryGetValue(name.ToLower(), out ManagedCustomScript? result); 379 | return result; 380 | } 381 | 382 | // Return the dictionary of managed custom script, indexed by name. 383 | internal async Task> GetManagedCustomScriptDict(CancellationToken cancellationToken) 384 | { 385 | if (_managedCustomScriptsDict is null) 386 | { 387 | var managedCustomScripts = await database.GetManagedCustomScriptsAsync(cancellationToken); 388 | _managedCustomScriptsDict = managedCustomScripts.ToDictionary(k => k.Name.ToLower(), v => v); 389 | } 390 | 391 | return _managedCustomScriptsDict; 392 | } 393 | } 394 | } 395 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/EnvironmentConfiguration.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Analytics.Synapse.Artifacts.Models; 5 | using Azure.Storage.Blobs; 6 | using Azure.Storage.Files.DataLake; 7 | using Microsoft.Extensions.Configuration; 8 | using Microsoft.Extensions.FileProviders; 9 | using Newtonsoft.Json; 10 | using System.Text.RegularExpressions; 11 | 12 | namespace DataverseToSql.Core 13 | { 14 | /// 15 | /// Configuration class for EnvironmentBase. 16 | /// The class if used when deserializing the JSON configuration file. 17 | /// 18 | public class EnvironmentConfiguration 19 | { 20 | // This constrcutor initializes configuration with placeholder values. 21 | // Useful to generate a template configuration file. 22 | public EnvironmentConfiguration() 23 | { 24 | ConfigFilePath = ""; 25 | } 26 | 27 | // Loads the configuration from the specified filepath. 28 | public EnvironmentConfiguration(string filepath) 29 | { 30 | new ConfigurationBuilder() 31 | .AddJsonFile(filepath, optional: false) 32 | .Build() 33 | .Bind( 34 | this, 35 | options => 36 | { 37 | options.ErrorOnUnknownConfiguration = true; 38 | } 39 | ); 40 | 41 | ConfigFilePath = filepath; 42 | } 43 | 44 | // Loads the configuration from the specified filepath, retrieving 45 | // the file using the specified IFileProvider implementation. 46 | // Currently used for ConfigurationBuilder to load the configuration 47 | // from a storage accoount. 48 | public EnvironmentConfiguration(string filepath, IFileProvider fileProvider) 49 | { 50 | new ConfigurationBuilder() 51 | .SetFileProvider(fileProvider) 52 | .AddJsonFile(filepath, optional: false) 53 | .Build() 54 | .Bind( 55 | this, 56 | options => 57 | { 58 | options.ErrorOnUnknownConfiguration = true; 59 | } 60 | ); 61 | 62 | ConfigFilePath = filepath; 63 | } 64 | 65 | // Path of the file the configuration was read from 66 | [JsonIgnore] 67 | public readonly string ConfigFilePath; 68 | 69 | // Configuration of the storage container hosting 70 | // Dataverse data 71 | public StorageConfiguration DataverseStorage { get; set; } = new(); 72 | 73 | // Configuration of the storage container where 74 | // Incremental data must be copied to 75 | public StorageConfiguration IncrementalStorage { get; set; } = new(); 76 | 77 | // Configuration of the storage container where 78 | // configuration information is stored 79 | public StorageConfiguration ConfigurationStorage { get; set; } = new(); 80 | 81 | // Configuration of the target Azure SQL Database 82 | public DatabaseConfiguration Database { get; set; } = new(); 83 | 84 | // Configuration of the Synapse workspace 85 | public SynapseWorkspaceConfiguration SynapseWorkspace { get; set; } = new(); 86 | 87 | // Configuration of the ingestion process 88 | public IngestionConfiguration Ingestion { get; set; } = new(); 89 | 90 | public SchemaHandlingConfiguration SchemaHandling { get; set; } = new(); 91 | 92 | public SparkPoolConfiguration Spark { get; set; } = new(); 93 | 94 | // Fill the configuration with placeholder values. 95 | // Useful to generate a template. 96 | internal void FillTemplateValues() 97 | { 98 | DataverseStorage.FillTemplateValues(); 99 | IncrementalStorage.FillTemplateValues(); 100 | ConfigurationStorage.FillTemplateValues(); 101 | Database.FillTemplateValues(); 102 | SynapseWorkspace.FillTemplateValues(); 103 | SchemaHandling.FillTemplateValues(); 104 | } 105 | } 106 | 107 | public class StorageConfiguration 108 | { 109 | // Name of the storage account without FQDN 110 | public string StorageAccount { get; set; } = ""; 111 | // Name of the container 112 | public string Container { get; set; } = ""; 113 | 114 | // URI of the container 115 | public Uri ContainerUri() 116 | { 117 | return new BlobUriBuilder(new Uri(StorageAccount)) 118 | { 119 | BlobContainerName = Container 120 | }.ToUri(); 121 | } 122 | 123 | private static readonly Regex blobUriRegex = new("^\\s*https://([a-zA-Z0-9]{3,24})\\.blob\\.core\\.windows\\.net/?"); 124 | 125 | // Datalake URI of the container 126 | public Uri DatalakeUri() 127 | { 128 | var match = blobUriRegex.Match(StorageAccount); 129 | 130 | if (!match.Success) 131 | { 132 | throw new ArgumentException($"Invalid storage URI: {StorageAccount}"); 133 | } 134 | 135 | var datalakeUri = $"https://{match.Groups[1].Value}.dfs.core.windows.net/"; 136 | 137 | return new Uri(datalakeUri); 138 | } 139 | 140 | // Account name 141 | public string AccountName() 142 | { 143 | var bub = new BlobUriBuilder(new Uri(StorageAccount)); 144 | return bub.AccountName; 145 | } 146 | 147 | // Fill the configuration with placeholder values. 148 | internal void FillTemplateValues() 149 | { 150 | StorageAccount = ""; 151 | Container = ""; 152 | } 153 | } 154 | 155 | public class DatabaseConfiguration 156 | { 157 | // FQDN of the Azure SQL logical server 158 | public string Server { get; set; } = ""; 159 | 160 | // Name of the database 161 | public string Database { get; set; } = ""; 162 | 163 | // Default schema for the tables containing CDM entities 164 | public string Schema { get; set; } = ""; 165 | 166 | // Fill the configuration with placeholder values. 167 | internal void FillTemplateValues() 168 | { 169 | Server = ""; 170 | Database = ""; 171 | Schema = ""; 172 | } 173 | } 174 | 175 | public class IngestionConfiguration 176 | { 177 | // Level of parallelism of the Foreach activity of the ingestion pipeline 178 | public int Parallelism { get; set; } = 1; 179 | } 180 | 181 | public class SynapseWorkspaceConfiguration 182 | { 183 | // Subscription ID of the Synapse Workspace 184 | public string SubscriptionId { get; set; } = ""; 185 | 186 | // Resource group of the Synapse Workspace 187 | public string ResourceGroup { get; set; } = ""; 188 | 189 | // Name of the Synapse Workspace 190 | public string Workspace { get; set; } = ""; 191 | 192 | // Return the URI of the Dev endpoint of the Synapse workspace. 193 | public string DevEndpoint() 194 | { 195 | return $"https://{Workspace}.dev.azuresynapse.net"; 196 | } 197 | 198 | // Return the FQDN of the Serverless endpoint of the Synapse workspace. 199 | public string ServerlessEndpoint() 200 | { 201 | return $"{Workspace}-ondemand.sql.azuresynapse.net"; 202 | } 203 | 204 | // Fill the configuration with placeholder values. 205 | internal void FillTemplateValues() 206 | { 207 | SubscriptionId = ""; 208 | ResourceGroup = ""; 209 | Workspace = ""; 210 | } 211 | } 212 | 213 | public class SchemaHandlingConfiguration 214 | { 215 | public bool EnableSchemaUpgradeForExistingTables { get; set; } = true; 216 | public bool OptionSetInt32 { get; set; } = false; 217 | 218 | public bool SkipIsDeleteColumn { get; set; } = false; 219 | 220 | internal void FillTemplateValues() 221 | { 222 | EnableSchemaUpgradeForExistingTables = true; 223 | } 224 | } 225 | 226 | public class SparkPoolConfiguration 227 | { 228 | public string SparkPool { get; set; } = ""; 229 | public int EntityConcurrency { get; set; } = 4; 230 | 231 | internal void FillTemplateValues() 232 | { 233 | SparkPool = ""; 234 | } 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/Jobs/AddAllEntitiesJob.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Microsoft.Extensions.Logging; 5 | 6 | namespace DataverseToSql.Core.Jobs 7 | { 8 | /// 9 | /// Adds all available CDM entities as managed entities. 10 | /// 11 | public class AddAllEntitiesJob 12 | { 13 | private readonly ILogger log; 14 | private readonly LocalEnvironment environment; 15 | 16 | public AddAllEntitiesJob(ILogger log, LocalEnvironment environment) 17 | { 18 | this.log = log; 19 | this.environment = environment; 20 | } 21 | 22 | public async Task RunAsync(CancellationToken cancellationToken=default) 23 | { 24 | var entityCount = 0; 25 | 26 | foreach (var cdmEntity in await environment.GetCdmEntitiesAsync(cancellationToken)) 27 | { 28 | if (! await environment.IsManagedEntityAsync(cdmEntity, cancellationToken)) 29 | { 30 | entityCount++; 31 | var job = new AddEntityJob(log, environment, cdmEntity.Name); 32 | await job.RunAsync(cancellationToken); 33 | } 34 | } 35 | 36 | if (entityCount == 0) 37 | { 38 | log.LogInformation("No entities were added."); 39 | } 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/Jobs/AddEntityJob.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using DataverseToSql.Core.Model; 5 | using Microsoft.Extensions.Logging; 6 | using System.Security.AccessControl; 7 | 8 | namespace DataverseToSql.Core.Jobs 9 | { 10 | /// 11 | /// Adds the specified CDM entity as a managed entity. 12 | /// 13 | public class AddEntityJob 14 | { 15 | private readonly ILogger log; 16 | private readonly LocalEnvironment environment; 17 | private readonly string name; 18 | 19 | public AddEntityJob(ILogger log, LocalEnvironment environment, string name) 20 | { 21 | this.log = log; 22 | this.environment = environment; 23 | this.name = name; 24 | } 25 | 26 | public async Task RunAsync(CancellationToken cancellationToken = default) 27 | { 28 | if (await environment.IsManagedEntityAsync(name, cancellationToken)) 29 | { 30 | log.LogWarning("Entity {name} is already present in the environment.", name); 31 | return; 32 | } 33 | 34 | var (found, cdmEntity) = await environment.TryGetCdmEntityAsync(name, cancellationToken); 35 | if (!found || cdmEntity is null) 36 | { 37 | throw new Exception($"Entity {name} was not found."); 38 | } 39 | 40 | log.LogInformation("Adding entity {EntityName}.", cdmEntity.Name); 41 | 42 | await environment.database.UpsertAsync(new ManagedEntity 43 | { 44 | Name = cdmEntity.Name, 45 | TargetSchema = environment.Config.Database.Schema, 46 | TargetTable = cdmEntity.Name 47 | }, 48 | cancellationToken); 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/LocalEnvironment.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Core; 5 | using Azure.Storage.Blobs; 6 | using Microsoft.Extensions.Logging; 7 | using Microsoft.SqlServer.TransactSql.ScriptDom; 8 | using Newtonsoft.Json; 9 | 10 | namespace DataverseToSql.Core 11 | { 12 | /// 13 | /// Derived class of EnvironmentBase that works with configuration from local files. 14 | /// Provides the ability to initialize a local environment from scratch. 15 | /// 16 | public class LocalEnvironment : EnvironmentBase 17 | { 18 | public readonly string LocalPath; 19 | 20 | public LocalEnvironment(ILogger log, TokenCredential credential, string path) 21 | : base(log, credential, ReadConfig(log, path)) 22 | { 23 | LocalPath = path; 24 | } 25 | 26 | // Loads configuration from the specified path. 27 | private static EnvironmentConfiguration ReadConfig(ILogger log, string path) 28 | { 29 | var environmentPath = Path.GetFullPath(path); 30 | var ConfigFilePath = Path.Join(environmentPath, CONFIG_FILE); 31 | 32 | log.LogInformation( 33 | "Loading environment from path {EnvironmentPath}", 34 | environmentPath 35 | ); 36 | 37 | log.LogInformation( 38 | "Loading configuration file {ConfigFilePath}", 39 | ConfigFilePath 40 | ); 41 | 42 | try 43 | { 44 | return new(ConfigFilePath); 45 | } 46 | catch (Exception ex) 47 | { 48 | throw new Exception( 49 | $"Cannot load configuration from {ConfigFilePath}", ex); 50 | } 51 | } 52 | 53 | // Initialize an environment on the local file system, at the specified path. 54 | // The configuration file is initialized with placeholder values. 55 | public static void Init(ILogger log, string path) 56 | { 57 | // Create environment root folder, if it does not exist 58 | 59 | var environmentPath = Path.GetFullPath(path); 60 | log.LogInformation( 61 | "Initializing environment in path {path}", 62 | environmentPath 63 | ); 64 | 65 | try 66 | { 67 | if (!Directory.Exists(environmentPath)) 68 | { 69 | Directory.CreateDirectory(environmentPath); 70 | } 71 | } 72 | catch (Exception ex) 73 | { 74 | throw new Exception("Error creating environment directory.", ex); 75 | } 76 | 77 | // Create a the configuration file. 78 | 79 | var configFile = Path.Join(environmentPath, CONFIG_FILE); 80 | 81 | if (File.Exists(configFile)) 82 | { 83 | throw new Exception($"Configuration file {configFile} already exists."); 84 | } 85 | 86 | try 87 | { 88 | var emptyConfig = new EnvironmentConfiguration(); 89 | emptyConfig.FillTemplateValues(); 90 | 91 | File.WriteAllText( 92 | configFile, 93 | JsonConvert.SerializeObject(emptyConfig, Formatting.Indented)); 94 | 95 | } 96 | catch (Exception ex) 97 | { 98 | throw new Exception("Error creating configuration file.", ex); 99 | } 100 | 101 | // Create custom SQL objects folder 102 | 103 | var customSqlObjectsFolderName = Path.Join(environmentPath, CUSTOM_SQL_OBJECTS_FOLDER); 104 | 105 | if (!Directory.Exists(customSqlObjectsFolderName)) 106 | { 107 | try 108 | { 109 | log.LogInformation("Creating custom SQL objects folder {customSqlObjectsFolderName}.", 110 | customSqlObjectsFolderName); 111 | Directory.CreateDirectory(customSqlObjectsFolderName); 112 | } 113 | catch (Exception ex) 114 | { 115 | throw new Exception("Error creating custom SQL objects folder.", ex); 116 | } 117 | } 118 | 119 | log.LogInformation("Successfully initialized the environment."); 120 | } 121 | 122 | internal override Task> InitCustomScriptsAsync(CancellationToken cancellationToken) 123 | { 124 | throw new NotImplementedException(); 125 | } 126 | 127 | internal override async Task GetCustomDatatypeMapReader(CancellationToken cancellationToken) 128 | { 129 | var customDatatypeMapPath = Path.Join(LocalPath, CUSTOM_DATATYPE_MAP); 130 | 131 | if (!File.Exists(customDatatypeMapPath)) 132 | return null; 133 | 134 | log.LogInformation("Loading custom data type map from {path}.", customDatatypeMapPath); 135 | 136 | return new StreamReader(new FileStream(customDatatypeMapPath, FileMode.Open)); 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/Model/BlobToIngest.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | namespace DataverseToSql.Core.Model 5 | { 6 | /// 7 | /// Represents a blob to be ingested by the Synapse ingestion pipeline. 8 | /// Data are stored in the BlobsToIngest table in Azure SQL Database. 9 | /// 10 | internal class BlobToIngest 11 | { 12 | internal BlobToIngest( 13 | ManagedEntity entity, 14 | string name, 15 | string basePath, 16 | string timestamp, 17 | string partition, 18 | LoadType loadType 19 | ) 20 | { 21 | Entity = entity; 22 | Name = name; 23 | BasePath = basePath; 24 | Timestamp = timestamp; 25 | Partition = partition; 26 | LoadType = loadType; 27 | } 28 | 29 | // Reference to the managed entity the blob belongs to 30 | public ManagedEntity Entity { get; } 31 | // Name of the blob, as a full path in the container 32 | public string Name { get; } 33 | // Base path of the blob up to the entity level 34 | public string BasePath { get; } 35 | // Timestamp of the blob 36 | public string Timestamp { get; } 37 | // Name of the partition the blob belongs to 38 | public string Partition { get; } 39 | // Type of load operation, either full or incremenetal 40 | public LoadType LoadType { get; } 41 | } 42 | 43 | internal enum LoadType 44 | { 45 | Full = 0, 46 | Incremental = 1 47 | } 48 | } -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/Model/ManagedBlob.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | namespace DataverseToSql.Core.Model 5 | { 6 | /// 7 | /// Represents a blob that is beign tracked for incremental ingestion. 8 | /// The main goal is to keep track of the offset the incremental ingestion has reached so far. 9 | /// Data are stored in the ManagedBlob table. 10 | /// 11 | internal class ManagedBlob 12 | { 13 | internal ManagedBlob( 14 | ManagedEntity entity, 15 | string name, 16 | long offset 17 | ) 18 | { 19 | Entity = entity; 20 | Name = name; 21 | Offset = offset; 22 | } 23 | 24 | // Reference to the managed entity the blob belongs to 25 | public ManagedEntity Entity { get; } 26 | // Name of the blob, as a file basename (e.g. 2022.csv) 27 | public string Name { get; } 28 | // Offset inside the source blob that ingestion has reached so far 29 | public long Offset { get; set; } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/Model/ManagedCustomScript.cs: -------------------------------------------------------------------------------- 1 | namespace DataverseToSql.Core.Model 2 | { 3 | internal class ManagedCustomScript 4 | { 5 | internal ManagedCustomScript( 6 | string name, 7 | string hash) 8 | { 9 | Name = name; 10 | Hash = hash; 11 | } 12 | 13 | public string Name { get; } 14 | public string Hash { get; } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/Model/ManagedEntity.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | namespace DataverseToSql.Core.Model 5 | { 6 | /// 7 | /// Represents an entity tracked for incremental ingestion. 8 | /// 9 | public class ManagedEntity 10 | { 11 | // Name of the entity 12 | public string Name { get; set; } = ""; 13 | // State of the entity inside the ingestion lifecycle 14 | public ManagedEntityState State { get; set; } 15 | = ManagedEntityState.New; 16 | // BASE64 encoded SHA1 of the latest SQL schema deployed for the entity 17 | public string? SchemaHash { get; set; } = null; 18 | // Name of the schema containing the table in the target SQL database 19 | public string TargetSchema { get; set; } = ""; 20 | // Name of the the table in the target SQL database 21 | public string TargetTable { get; set; } = ""; 22 | // Serverless query for deduplication of full loads 23 | public string? FullLoadInnerQuery { get; set; } = null; 24 | // Serverless query for deduplication of incremental loads 25 | public string? IncrementalInnerQuery { get; set; } = null; 26 | // Serverless OPENROWSET query 27 | } 28 | 29 | public enum ManagedEntityState 30 | { 31 | // New = the entity has been added but its schema has not yet been created 32 | New = 0, 33 | 34 | // PendingInitialIngestion = the schema of the entity has been created, 35 | // and the entity is waiting its initial load 36 | PendingInitialIngestion = 1, 37 | 38 | // IngestionInProgress = the intial ingestion of the entity is in progress 39 | IngestionInProgress = 2, 40 | 41 | // Ready = the initial load of the entity is complete and the entity is 42 | // ready for incremental loads 43 | Ready = 3 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/Naming.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using DataverseToSql.Core.CdmModel; 5 | 6 | namespace DataverseToSql.Core 7 | { 8 | /// 9 | /// Provides naming for Synapse artifacts and some SQL objects. 10 | /// 11 | internal static class Naming 12 | { 13 | // Pipelines 14 | internal static string PipelineFolderName() => "DataverseToSql"; 15 | internal static string IncrementalLoadPipelineName() => $"DataverseToSql_IncrementalLoad"; 16 | internal static string FullLoadPipelineName() => $"DataverseToSql_FullLoad"; 17 | 18 | // Datasets 19 | internal static string DatasetFolder() => "DataverseToSql"; 20 | internal static string MetadataDatasetName() => "DataverseToSql_Metadata"; 21 | internal static string ServerlessDatasetName() => "DataverseToSql_Severless"; 22 | internal static string AzureSqlDatasetName() => "DataverseToSql_AzureSQL"; 23 | 24 | // Linked services 25 | internal static string AzureSqlLinkedServiceName() => "DataverseToSql_AzureSQL"; 26 | internal static string ServerlessPoolLinkedServiceName() => "DataverseToSql_Serverless"; 27 | 28 | // SQL objects 29 | // Note: changing the names below requires changing the Copy activity code in DeployJob 30 | // to reflect the new naming in the expressions referencing stored procedure and table 31 | // type 32 | internal static string MergeProcName(this CdmEntity entity) => $"[Merge_{entity.Name}]"; 33 | internal static string TableTypeName(this CdmEntity entity) => $"[{entity.Name}_TableType]"; 34 | 35 | // Spark 36 | internal static string RootNotebookName() => "DataverseToSql_RootNotebook"; 37 | internal static string ProcessEntityNotebookName() => "DataverseToSql_ProcessEntity"; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/Notebooks.Designer.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // This code was generated by a tool. 4 | // Runtime Version:4.0.30319.42000 5 | // 6 | // Changes to this file may cause incorrect behavior and will be lost if 7 | // the code is regenerated. 8 | // 9 | //------------------------------------------------------------------------------ 10 | 11 | namespace DataverseToSql.Core { 12 | using System; 13 | 14 | 15 | /// 16 | /// A strongly-typed resource class, for looking up localized strings, etc. 17 | /// 18 | // This class was auto-generated by the StronglyTypedResourceBuilder 19 | // class via a tool like ResGen or Visual Studio. 20 | // To add or remove a member, edit your .ResX file then rerun ResGen 21 | // with the /str option, or rebuild your VS project. 22 | [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] 23 | [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] 24 | [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] 25 | internal class Notebooks { 26 | 27 | private static global::System.Resources.ResourceManager resourceMan; 28 | 29 | private static global::System.Globalization.CultureInfo resourceCulture; 30 | 31 | [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] 32 | internal Notebooks() { 33 | } 34 | 35 | /// 36 | /// Returns the cached ResourceManager instance used by this class. 37 | /// 38 | [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] 39 | internal static global::System.Resources.ResourceManager ResourceManager { 40 | get { 41 | if (object.ReferenceEquals(resourceMan, null)) { 42 | global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("DataverseToSql.Core.Notebooks", typeof(Notebooks).Assembly); 43 | resourceMan = temp; 44 | } 45 | return resourceMan; 46 | } 47 | } 48 | 49 | /// 50 | /// Overrides the current thread's CurrentUICulture property for all 51 | /// resource lookups using this strongly typed resource class. 52 | /// 53 | [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] 54 | internal static global::System.Globalization.CultureInfo Culture { 55 | get { 56 | return resourceCulture; 57 | } 58 | set { 59 | resourceCulture = value; 60 | } 61 | } 62 | 63 | /// 64 | /// Looks up a localized resource of type System.Byte[]. 65 | /// 66 | internal static byte[] DataverseToSql_ProcessEntity { 67 | get { 68 | object obj = ResourceManager.GetObject("DataverseToSql_ProcessEntity", resourceCulture); 69 | return ((byte[])(obj)); 70 | } 71 | } 72 | 73 | /// 74 | /// Looks up a localized resource of type System.Byte[]. 75 | /// 76 | internal static byte[] DataverseToSql_RootNotebook { 77 | get { 78 | object obj = ResourceManager.GetObject("DataverseToSql_RootNotebook", resourceCulture); 79 | return ((byte[])(obj)); 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/Notebooks.resx: -------------------------------------------------------------------------------- 1 |  2 | 3 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | text/microsoft-resx 110 | 111 | 112 | 2.0 113 | 114 | 115 | System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 116 | 117 | 118 | System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 119 | 120 | 121 | 122 | Notebooks\DataverseToSql_ProcessEntity.ipynb;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 123 | 124 | 125 | Notebooks\DataverseToSql_RootNotebook.ipynb;System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 126 | 127 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/Notebooks/DataverseToSql_ProcessEntity.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "source": [ 6 | "entity = '' # Name of the entity to process\r\n", 7 | "target_schema = '' # Schema of the target table\r\n", 8 | "\r\n", 9 | "storage_account = '' # Storage account name (no FQDN)\r\n", 10 | "container = '' # Container for Synapse Link for Dataverse\r\n", 11 | "servername = '' # Azure SQL Database server name\r\n", 12 | "dbname = '' # Azure SQL Database name" 13 | ], 14 | "outputs": [], 15 | "execution_count": 1, 16 | "metadata": { 17 | "jupyter": { 18 | "source_hidden": false, 19 | "outputs_hidden": false 20 | }, 21 | "nteract": { 22 | "transient": { 23 | "deleting": false 24 | } 25 | }, 26 | "tags": [ 27 | "parameters" 28 | ] 29 | } 30 | }, 31 | { 32 | "cell_type": "code", 33 | "source": [ 34 | "from pyspark.sql.functions import to_timestamp, col, dense_rank, desc, rank,row_number, coalesce\r\n", 35 | "from pyspark.sql.window import Window\r\n", 36 | "import json\r\n", 37 | "import pyspark.sql.types as types\r\n", 38 | "import pyodbc\r\n", 39 | "import re\r\n", 40 | "import struct\r\n", 41 | "\r\n", 42 | "spark.conf.set(\"spark.sql.parquet.int96RebaseModeInWrite\", \"CORRECTED\")" 43 | ], 44 | "outputs": [], 45 | "execution_count": 2, 46 | "metadata": { 47 | "jupyter": { 48 | "source_hidden": false, 49 | "outputs_hidden": false 50 | }, 51 | "nteract": { 52 | "transient": { 53 | "deleting": false 54 | } 55 | } 56 | } 57 | }, 58 | { 59 | "cell_type": "code", 60 | "source": [ 61 | "# Functions\r\n", 62 | "\r\n", 63 | "# Returns the spark datatype based on the CDM model datatype\r\n", 64 | "def get_attribute_spark_datatype(attribute):\r\n", 65 | " match attribute['dataType']:\r\n", 66 | " case 'boolean':\r\n", 67 | " return types.BooleanType()\r\n", 68 | " case 'dateTime':\r\n", 69 | " return types.StringType()\r\n", 70 | " case 'decimal':\r\n", 71 | " numeric_trait = [t for t in attribute[\"cdm:traits\"] if t[\"traitReference\"] == \"is.dataFormat.numeric.shaped\"][0]\r\n", 72 | " precision = int([a for a in numeric_trait[\"arguments\"] if a[\"name\"]==\"precision\"][0][\"value\"])\r\n", 73 | " scale = int([a for a in numeric_trait[\"arguments\"] if a[\"name\"]==\"scale\"][0][\"value\"])\r\n", 74 | " return types.DecimalType(precision, scale)\r\n", 75 | " case 'double':\r\n", 76 | " return types.DoubleType()\r\n", 77 | " case 'guid':\r\n", 78 | " return types.StringType()\r\n", 79 | " case 'int64':\r\n", 80 | " return types.LongType()\r\n", 81 | " case 'string':\r\n", 82 | " return types.StringType()\r\n", 83 | " case _:\r\n", 84 | " raise Exception(f\"Unsupported CDM data type: {attribute['dataType']}\")\r\n", 85 | "\r\n", 86 | "# Read the content of a file\r\n", 87 | "def get_file_content(path):\r\n", 88 | " for file in mssparkutils.fs.ls(model_path):\r\n", 89 | " pass \r\n", 90 | "\r\n", 91 | " if not file.isFile:\r\n", 92 | " raise ValueError(f\"The specified path is not a file: {path}\")\r\n", 93 | "\r\n", 94 | " return mssparkutils.fs.head(model_path, file.size)\r\n", 95 | "\r\n", 96 | "\r\n", 97 | "# Get the AAD token to access Azure SQL Database\r\n", 98 | "def get_sql_token():\r\n", 99 | " return mssparkutils.credentials.getToken('DW')\r\n", 100 | "\r\n", 101 | "# Open a new pyodbc SQL Server connection \r\n", 102 | "def get_pyodbc_sql_conn(server, database):\r\n", 103 | " sql_token = get_sql_token()\r\n", 104 | " token_bytes = sql_token.encode(\"UTF-16-LE\")\r\n", 105 | " token_struct = struct.pack(f' (dtName[0] == 'n' ? 4000 : 8000)) // dtName[0] == 'n' identifies Unicode data types 32 | { 33 | throw new Exception($"Invalid length: {datatype}"); 34 | } 35 | break; 36 | default: 37 | throw new Exception($"Invalid data type parameter: {datatype}"); 38 | } 39 | } 40 | else if (dt.Parameters.Count != 0) 41 | throw new Exception($"Invalid data type: {datatype}"); 42 | break; 43 | case "datetime2": 44 | case "datetimeoffset": 45 | case "time": 46 | if (dt.Parameters.Count == 1 47 | && dt.Parameters[0] is Literal timeScaleLiteral 48 | && timeScaleLiteral.LiteralType == LiteralType.Integer) 49 | { 50 | var timeScale = int.Parse(timeScaleLiteral.Value); 51 | if (timeScale < 0 || timeScale > 7) 52 | throw new Exception($"Invalid scale: {datatype}"); 53 | } 54 | else if (dt.Parameters.Count != 0) 55 | throw new Exception($"Unexpected data type parameters: {datatype}"); 56 | break; 57 | case "dec": 58 | case "decimal": 59 | case "numeric": 60 | if (dt.Parameters.Count >= 1 61 | && dt.Parameters.Count <= 2 62 | && dt.Parameters[0] is Literal precisionLiteral1 63 | && precisionLiteral1.LiteralType == LiteralType.Integer) 64 | { 65 | var precision = int.Parse(precisionLiteral1.Value); 66 | if (precision < 1 || precision > 38) 67 | throw new Exception($"Invalid precision: {datatype}"); 68 | 69 | if (dt.Parameters.Count == 2) 70 | { 71 | if (dt.Parameters[1] is Literal scaleLiteral 72 | && scaleLiteral.LiteralType == LiteralType.Integer) 73 | { 74 | var scale = int.Parse(scaleLiteral.Value); 75 | 76 | if (scale < 0) 77 | throw new Exception($"Invalid scale: {datatype}"); 78 | 79 | if (scale > precision) 80 | throw new Exception($"Scale cannot be larger than precision: {datatype}"); 81 | } 82 | else 83 | throw new Exception($"Invalid scale: {datatype}"); 84 | } 85 | } 86 | else if (dt.Parameters.Count != 0) 87 | throw new Exception($"Unexpected data type parameters: {datatype}"); 88 | break; 89 | case "float": 90 | if (dt.Parameters.Count == 1 91 | && dt.Parameters[0] is Literal precisionLiteral2 92 | && precisionLiteral2.LiteralType == LiteralType.Integer) 93 | { 94 | var precision = int.Parse(precisionLiteral2.Value); 95 | if (precision < 1 || precision > 53) 96 | throw new Exception($"Invalid precision: {datatype}"); 97 | } 98 | else if (dt.Parameters.Count != 0) 99 | throw new Exception($"Unexpected data type parameters: {datatype}"); 100 | break; 101 | case "int": 102 | case "bigint": 103 | case "bit": 104 | case "date": 105 | case "datetime": 106 | case "image": 107 | case "money": 108 | case "ntext": 109 | case "real": 110 | case "smalldatetime": 111 | case "smallint": 112 | case "smallmoney": 113 | case "sql_variant": 114 | case "sysname": 115 | case "text": 116 | case "timestamp": 117 | case "tinyint": 118 | case "uniqueidentifier": 119 | if (dt.Parameters.Count != 0) 120 | throw new Exception($"Unexpected data type parameters: {datatype}"); 121 | break; 122 | default: 123 | throw new Exception($"Unexpected data type: {datatype}"); 124 | } 125 | } 126 | 127 | private static ParameterizedDataTypeReference Parse(string datatype) 128 | { 129 | var parser = new TSql160Parser(false); 130 | var stringReader = new StringReader(datatype); 131 | 132 | var dt = parser.ParseScalarDataType(stringReader, out var errors); 133 | 134 | if (errors.Count > 0 || dt is null) 135 | throw new AggregateException( 136 | $"Error parsing data type: {datatype}", 137 | errors.Select(e => new Exception(e.Message))); 138 | 139 | if (dt is SqlDataTypeReference sqlDataTypeReference) 140 | return sqlDataTypeReference; 141 | else if (dt is UserDataTypeReference udt 142 | && udt.Name.Identifiers.Count == 1 143 | && udt.Name.BaseIdentifier.Value.ToLower() == "sysname") 144 | return udt; 145 | else if (dt is XmlDataTypeReference) 146 | throw new Exception($"Data type is XML: {datatype}"); 147 | else 148 | throw new Exception($"Data type is not built-in: {datatype}"); 149 | } 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects.resx: -------------------------------------------------------------------------------- 1 |  2 | 3 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | text/microsoft-resx 110 | 111 | 112 | 2.0 113 | 114 | 115 | System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 116 | 117 | 118 | System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 119 | 120 | 121 | 122 | sqlobjects\dataversetosql.blobstoingest_insert_proc.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 123 | 124 | 125 | sqlobjects\dataversetosql.blobstoingest_table.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 126 | 127 | 128 | SqlObjects\DataverseToSql.IngestionJobs_Complete_Proc.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;Windows-1252 129 | 130 | 131 | SqlObjects\DataverseToSql.IngestionJobs_Get_Proc.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 132 | 133 | 134 | sqlobjects\dataversetosql.managedblobs_table.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 135 | 136 | 137 | sqlobjects\dataversetosql.managedblobs_upsert_proc.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 138 | 139 | 140 | sqlobjects\dataversetosql.managedcustomscripts_table.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 141 | 142 | 143 | sqlobjects\dataversetosql.managedcustomscripts_upsert_proc.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 144 | 145 | 146 | sqlobjects\dataversetosql.managedentities_table.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 147 | 148 | 149 | sqlobjects\dataversetosql.managedentities_upsert_proc.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 150 | 151 | 152 | sqlobjects\dataversetosql_schema.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 153 | 154 | 155 | sqlobjects\dataversetosql.types.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 156 | 157 | 158 | sqlobjects\optionsets_attributemetadata.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 159 | 160 | 161 | sqlobjects\optionsets_globaloptionsetmetadata.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 162 | 163 | 164 | sqlobjects\optionsets_optionsetmetadata.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 165 | 166 | 167 | sqlobjects\optionsets_statemetadata.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 168 | 169 | 170 | sqlobjects\optionsets_statusmetadata.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 171 | 172 | 173 | sqlobjects\optionsets_targetmetadata.sql;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8 174 | 175 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/DataverseToSql.BlobsToIngest_Insert_Proc.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE PROCEDURE [DataverseToSql].[BlobsToIngest_Insert] 5 | @EntityName [DataverseToSql].[EntityType], 6 | @BlobName [DataverseToSql].[BlobNameType], 7 | @BasePath [DataverseToSql].[BlobNameType], 8 | @Timestamp [DataverseToSql].[TimestampType], 9 | @Partition [DataverseToSql].[BlobPartitionType], 10 | @LoadType INT 11 | AS 12 | IF NOT EXISTS ( 13 | SELECT * FROM [DataverseToSql].[BlobsToIngest] 14 | WHERE 15 | [EntityName] = @EntityName 16 | AND [BlobName] = @BlobName 17 | ) 18 | BEGIN 19 | INSERT INTO [DataverseToSql].[BlobsToIngest]( 20 | [EntityName], 21 | [BlobName], 22 | [BasePath], 23 | [Timestamp], 24 | [Partition], 25 | [LoadType] 26 | ) 27 | VALUES ( 28 | @EntityName, 29 | @BlobName, 30 | @BasePath, 31 | @Timestamp, 32 | @Partition, 33 | @LoadType 34 | ) 35 | END -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/DataverseToSql.BlobsToIngest_Table.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE TABLE [DataverseToSql].[BlobsToIngest] ( 5 | [Id] BIGINT IDENTITY PRIMARY KEY, 6 | [EntityName] [DataverseToSql].[EntityType] NOT NULL, 7 | [BlobName] [DataverseToSql].[BlobNameType] NOT NULL, 8 | [BasePath] [DataverseToSql].[BlobNameType] NOT NULL, 9 | [Timestamp] [DataverseToSql].[TimestampType] NOT NULL, 10 | [Partition] [DataverseToSql].[BlobPartitionType] NOT NULL, 11 | [LoadType] INT NOT NULL, -- Full=0, Incremental=1 (from LoadType enum) 12 | [Complete] INT NOT NULL DEFAULT 0, 13 | CONSTRAINT UQ_BlobsToIngest UNIQUE ([EntityName], [BlobName]), 14 | CONSTRAINT FK_BlobsToIngest_Entities 15 | FOREIGN KEY ([EntityName]) REFERENCES [DataverseToSql].[ManagedEntities]([EntityName]) 16 | ); 17 | GO 18 | 19 | CREATE INDEX [IX_BlobsToIngest_EntityName] 20 | ON [DataverseToSql].[BlobsToIngest]([EntityName]) -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/DataverseToSql.IngestionJobs_Complete_Proc.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE PROCEDURE [DataverseToSql].[IngestionJobs_Complete] 5 | @JobId [DataverseToSql].[JobIdType] 6 | AS 7 | 8 | DECLARE @CompletedBlobs TABLE ( 9 | [EntityName] [DataverseToSql].[EntityType], 10 | [LoadType] INT 11 | ) 12 | 13 | BEGIN TRAN 14 | 15 | DECLARE @Entity [DataverseToSql].[EntityType] 16 | DECLARE @LoadType INT 17 | 18 | SELECT 19 | @Entity=[EntityName], 20 | @LoadType=[LoadType] 21 | FROM 22 | [DataverseToSql].[BlobsToIngest] b 23 | WHERE 24 | b.[Id] = @JobId 25 | 26 | IF @LoadType = 1 -- Incremental load 27 | BEGIN 28 | UPDATE a 29 | SET 30 | [Complete] = 1 31 | OUTPUT inserted.[EntityName], inserted.[LoadType] 32 | INTO @CompletedBlobs 33 | FROM 34 | [DataverseToSql].[BlobsToIngest] a 35 | INNER JOIN [DataverseToSql].[BlobsToIngest] b 36 | ON a.[Id] <= b.Id 37 | AND a.[EntityName] = b.[EntityName] 38 | AND a.[LoadType] = b.[LoadType] 39 | and a.[Partition] = b.[Partition] 40 | WHERE 41 | b.[Id] = @JobId 42 | END 43 | ELSE -- Full Load 44 | BEGIN 45 | -- Check if the entity is not partitioned 46 | -- The condition is that there is a partition named "1" 47 | IF EXISTS ( 48 | SELECT * 49 | FROM [DataverseToSql].[BlobsToIngest] 50 | WHERE 51 | [Partition] = '1' 52 | AND [EntityName] = @Entity 53 | ) 54 | BEGIN -- Entity is not partitioned 55 | UPDATE a 56 | SET 57 | [Complete] = 1 58 | OUTPUT inserted.[EntityName], inserted.[LoadType] 59 | INTO @CompletedBlobs 60 | FROM 61 | [DataverseToSql].[BlobsToIngest] a 62 | INNER JOIN [DataverseToSql].[BlobsToIngest] b 63 | ON a.[Id] <= b.Id 64 | AND a.[EntityName] = b.[EntityName] 65 | AND a.[LoadType] = b.[LoadType] 66 | WHERE 67 | b.[Id] = @JobId 68 | END 69 | ELSE -- Entity is partitioned by Year 70 | BEGIN 71 | UPDATE a 72 | SET 73 | [Complete] = 1 74 | OUTPUT inserted.[EntityName], inserted.[LoadType] 75 | INTO @CompletedBlobs 76 | FROM 77 | [DataverseToSql].[BlobsToIngest] a 78 | INNER JOIN [DataverseToSql].[BlobsToIngest] b 79 | ON a.[Id] <= b.Id 80 | AND a.[EntityName] = b.[EntityName] 81 | AND a.[LoadType] = b.[LoadType] 82 | and a.[Partition] = b.[Partition] 83 | WHERE 84 | b.[Id] = @JobId 85 | END 86 | 87 | -- For full loads, set entity state to Ready if the initial 88 | -- load of all blobs of the entity is complete 89 | IF NOT EXISTS ( 90 | SELECT * 91 | FROM [DataverseToSql].[BlobsToIngest] 92 | WHERE 93 | [EntityName] = @Entity 94 | AND [LoadType] IN (0, 2) -- Full load 95 | AND [Complete] = 0) 96 | BEGIN 97 | UPDATE [DataverseToSql].[ManagedEntities] 98 | SET [State] = 3 -- Ready 99 | WHERE 100 | [State] = 2 -- IngestionInProgress 101 | AND [EntityName] = @Entity 102 | END 103 | END 104 | 105 | COMMIT -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/DataverseToSql.IngestionJobs_Get_Proc.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE PROCEDURE [DataverseToSql].[IngestionJobs_Get] 5 | AS 6 | 7 | WITH 8 | jobs AS ( 9 | SELECT 10 | MAX([Id]) AS [JobId], 11 | [EntityName], 12 | [BasePath], 13 | MIN([Timestamp]) AS [TimestampFrom], 14 | MAX([Timestamp]) AS [TimestampTo], 15 | [Partition], 16 | [LoadType] 17 | FROM 18 | [DataverseToSql].[BlobsToIngest] 19 | WHERE 20 | [Complete] = 0 21 | AND [LoadType] = 1 22 | GROUP BY 23 | [EntityName], 24 | [BasePath], 25 | [Partition], 26 | [LoadType] 27 | ) 28 | SELECT 29 | j.[JobId], 30 | e.[EntityName], 31 | e.[TargetSchema], 32 | e.[TargetTable], 33 | -- OPENROWSET query 34 | REPLACE(REPLACE(REPLACE( 35 | CASE j.[LoadType] 36 | -- Full load 37 | WHEN 0 THEN e.[FullLoadInnerQuery] 38 | -- Incremental load 39 | WHEN 1 THEN e.[IncrementalInnerQuery] 40 | END, 41 | '<<>>', 42 | j.[TimestampFrom]), 43 | '<<>>', 44 | j.[TimestampTo]), 45 | '<<>>', 46 | TRIM('/' FROM j.[BasePath]) + '/*/' + CASE j.[LoadType] 47 | -- Full load 48 | WHEN 0 THEN j.[Partition] + '*.csv' 49 | -- Incremental load 50 | WHEN 1 THEN '*.csv' 51 | END) 52 | AS [ServerlessQuery], 53 | j.[LoadType] 54 | FROM 55 | jobs j 56 | INNER JOIN [DataverseToSql].[ManagedEntities] e 57 | ON j.[EntityName] = e.[EntityName] -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/DataverseToSql.ManagedBlobs_Table.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE TABLE [DataverseToSql].[ManagedBlobs] ( 5 | [EntityName] [DataverseToSql].[EntityType] NOT NULL, 6 | [BlobName] [DataverseToSql].[BlobNameType] NOT NULL, 7 | [FileOffset] BIGINT NOT NULL, 8 | CONSTRAINT PK_ManagedBlobs PRIMARY KEY ([EntityName], [BlobName]), 9 | CONSTRAINT FK_ManagedBlobs_Entities FOREIGN KEY ([EntityName]) REFERENCES [DataverseToSql].[ManagedEntities]([EntityName]) 10 | ); -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/DataverseToSql.ManagedBlobs_Upsert_Proc.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE PROCEDURE [DataverseToSql].[ManagedBlobs_Upsert] 5 | @EntityName [DataverseToSql].[EntityType], 6 | @BlobName [DataverseToSql].[BlobNameType], 7 | @FileOffset BIGINT 8 | AS 9 | IF NOT EXISTS ( 10 | SELECT * FROM [DataverseToSql].[ManagedBlobs] 11 | WHERE 12 | [EntityName] = @EntityName 13 | AND [BlobName] = @BlobName 14 | ) 15 | BEGIN 16 | INSERT INTO [DataverseToSql].[ManagedBlobs]( 17 | [EntityName], 18 | [BlobName], 19 | [FileOffset] 20 | ) 21 | VALUES ( 22 | @EntityName, 23 | @BlobName, 24 | @FileOffset 25 | ) 26 | END 27 | ELSE 28 | BEGIN 29 | UPDATE [DataverseToSql].[ManagedBlobs] 30 | SET 31 | [FileOffset] = @FileOffset 32 | WHERE 33 | [EntityName] = @EntityName 34 | AND [BlobName] = @BlobName 35 | END -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/DataverseToSql.ManagedCustomScripts_Table.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE TABLE [DataverseToSql].[ManagedCustomScripts] ( 5 | [ScriptName] [DataverseToSql].[CustomScriptNameType] NOT NULL PRIMARY KEY, 6 | [Hash] NVARCHAR(128) NULL 7 | ) -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/DataverseToSql.ManagedCustomScripts_Upsert_Proc.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE PROCEDURE [DataverseToSql].[ManagedCustomScripts_Upsert] 5 | @ScriptName [DataverseToSql].[CustomScriptNameType], 6 | @Hash NVARCHAR(128) 7 | AS 8 | IF NOT EXISTS ( 9 | SELECT * FROM [DataverseToSql].[ManagedCustomScripts] 10 | WHERE 11 | [ScriptName] = @ScriptName 12 | ) 13 | BEGIN 14 | INSERT INTO [DataverseToSql].[ManagedCustomScripts]( 15 | [ScriptName], 16 | [Hash] 17 | ) 18 | VALUES ( 19 | @ScriptName, 20 | @Hash 21 | ) 22 | END 23 | ELSE 24 | BEGIN 25 | UPDATE [DataverseToSql].[ManagedCustomScripts] 26 | SET 27 | [Hash] = @Hash 28 | WHERE 29 | [ScriptName] = @ScriptName 30 | END -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/DataverseToSql.ManagedEntities_Table.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE TABLE [DataverseToSql].[ManagedEntities] ( 5 | [EntityName] [DataverseToSql].[EntityType] NOT NULL PRIMARY KEY, 6 | [State] INT NOT NULL, 7 | [SchemaHash] NVARCHAR(128) NULL, 8 | [TargetSchema] SYSNAME NOT NULL, 9 | [TargetTable] SYSNAME NOT NULL, 10 | [FullLoadInnerQuery] NVARCHAR(MAX) NULL, 11 | [IncrementalInnerQuery] NVARCHAR(MAX) NULL 12 | ) -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/DataverseToSql.ManagedEntities_Upsert_Proc.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE PROCEDURE [DataverseToSql].[ManagedEntities_Upsert] 5 | @EntityName [DataverseToSql].[EntityType], 6 | @TargetSchema SYSNAME, 7 | @TargetTable SYSNAME, 8 | @State INT = NULL, 9 | @SchemaHash NVARCHAR(128) = NULL, 10 | @FullLoadInnerQuery NVARCHAR(MAX) = NULL, 11 | @IncrementalInnerQuery NVARCHAR(MAX) = NULL 12 | AS 13 | IF NOT EXISTS ( 14 | SELECT * FROM [DataverseToSql].[ManagedEntities] 15 | WHERE [EntityName] = @EntityName 16 | ) 17 | BEGIN 18 | INSERT INTO [DataverseToSql].[ManagedEntities]( 19 | [EntityName], 20 | [State], 21 | [SchemaHash], 22 | [TargetSchema], 23 | [TargetTable], 24 | [FullLoadInnerQuery], 25 | [IncrementalInnerQuery] 26 | ) 27 | VALUES ( 28 | @EntityName, 29 | @State, 30 | @SchemaHash, 31 | @TargetSchema, 32 | @TargetTable, 33 | @FullLoadInnerQuery, 34 | @IncrementalInnerQuery 35 | ) 36 | END 37 | ELSE 38 | BEGIN 39 | UPDATE [DataverseToSql].[ManagedEntities] 40 | SET 41 | [State] = ISNULL(@State, [State]), 42 | [SchemaHash] = ISNULL(@SchemaHash, [SchemaHash]), 43 | [FullLoadInnerQuery] = ISNULL(@FullLoadInnerQuery, [FullLoadInnerQuery]), 44 | [IncrementalInnerQuery] = ISNULL(@IncrementalInnerQuery, [IncrementalInnerQuery]) 45 | WHERE 46 | [EntityName] = @EntityName 47 | END -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/DataverseToSql.Types.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE TYPE [DataverseToSql].[EntityType] 5 | FROM NVARCHAR(128); 6 | GO 7 | 8 | CREATE TYPE [DataverseToSql].[BlobNameType] 9 | FROM NVARCHAR(722); 10 | GO 11 | 12 | CREATE TYPE [DataverseToSql].[CustomScriptNameType] 13 | FROM NVARCHAR(512); 14 | GO 15 | 16 | CREATE TYPE [DataverseToSql].[BlobPartitionType] 17 | FROM NVARCHAR(512); 18 | GO 19 | 20 | CREATE TYPE [DataverseToSql].[JobIdType] 21 | FROM BIGINT; 22 | GO 23 | 24 | CREATE TYPE [DataverseToSql].[TimestampType] 25 | FROM NVARCHAR(14); 26 | GO -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/DataverseToSql_Schema.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE SCHEMA [DataverseToSql]; -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/Optionsets_AttributeMetadata.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE TABLE [$$SCHEMA$$].[AttributeMetadata] ( 5 | [Id] [bigint] NULL, 6 | [EntityName] [nvarchar](64) NULL, 7 | [AttributeName] [nvarchar](64) NULL, 8 | [AttributeType] [nvarchar](64) NULL, 9 | [AttributeTypeCode] [int] NULL, 10 | [Version] [bigint] NULL, 11 | [Timestamp] [datetime] NULL, 12 | [MetadataId] [nvarchar](64) NULL, 13 | [Precision] [int] NULL 14 | ); 15 | GO 16 | 17 | CREATE TYPE [DataverseToSql].[AttributeMetadata_TableType] AS TABLE ( 18 | [Id] [bigint] NULL, 19 | [EntityName] [nvarchar](64) NULL, 20 | [AttributeName] [nvarchar](64) NULL, 21 | [AttributeType] [nvarchar](64) NULL, 22 | [AttributeTypeCode] [int] NULL, 23 | [Version] [bigint] NULL, 24 | [Timestamp] [datetime] NULL, 25 | [MetadataId] [nvarchar](64) NULL, 26 | [Precision] [int] NULL 27 | ); 28 | GO 29 | 30 | CREATE PROCEDURE [DataverseToSql].[Merge_AttributeMetadata] 31 | @source [DataverseToSql].[AttributeMetadata_TableType] READONLY 32 | AS 33 | BEGIN TRAN 34 | 35 | TRUNCATE TABLE [$$SCHEMA$$].[AttributeMetadata] 36 | 37 | INSERT [$$SCHEMA$$].[AttributeMetadata] 38 | SELECT * FROM @source 39 | 40 | COMMIT -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/Optionsets_GlobalOptionsetMetadata.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE TABLE [$$SCHEMA$$].[GlobalOptionsetMetadata] ( 5 | [OptionSetName] [nvarchar](64) NULL, 6 | [Option] [int] NULL, 7 | [IsUserLocalizedLabel] [bit] NULL, 8 | [LocalizedLabelLanguageCode] [int] NULL, 9 | [LocalizedLabel] [nvarchar](350) NULL, 10 | [GlobalOptionSetName] [nvarchar](64) NULL, 11 | [EntityName] [nvarchar](64) NULL 12 | ); 13 | GO 14 | 15 | CREATE TYPE [DataverseToSql].[GlobalOptionsetMetadata_TableType] AS TABLE ( 16 | [OptionSetName] [nvarchar](64) NULL, 17 | [Option] [int] NULL, 18 | [IsUserLocalizedLabel] [bit] NULL, 19 | [LocalizedLabelLanguageCode] [int] NULL, 20 | [LocalizedLabel] [nvarchar](350) NULL, 21 | [GlobalOptionSetName] [nvarchar](64) NULL, 22 | [EntityName] [nvarchar](64) NULL 23 | ); 24 | GO 25 | 26 | CREATE PROCEDURE [DataverseToSql].[Merge_GlobalOptionsetMetadata] 27 | @source [DataverseToSql].[GlobalOptionsetMetadata_TableType] READONLY 28 | AS 29 | BEGIN TRAN 30 | 31 | TRUNCATE TABLE [$$SCHEMA$$].[GlobalOptionsetMetadata] 32 | 33 | INSERT [$$SCHEMA$$].[GlobalOptionsetMetadata] 34 | SELECT * FROM @source 35 | 36 | COMMIT -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/Optionsets_OptionsetMetadata.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE TABLE [$$SCHEMA$$].[OptionsetMetadata] ( 5 | [EntityName] [nvarchar](64) NULL, 6 | [OptionSetName] [nvarchar](64) NULL, 7 | [Option] [int] NULL, 8 | [IsUserLocalizedLabel] [bit] NULL, 9 | [LocalizedLabelLanguageCode] [int] NULL, 10 | [LocalizedLabel] [nvarchar](350) NULL 11 | ); 12 | GO 13 | 14 | CREATE TYPE [DataverseToSql].[OptionsetMetadata_TableType] AS TABLE ( 15 | [EntityName] [nvarchar](64) NULL, 16 | [OptionSetName] [nvarchar](64) NULL, 17 | [Option] [int] NULL, 18 | [IsUserLocalizedLabel] [bit] NULL, 19 | [LocalizedLabelLanguageCode] [int] NULL, 20 | [LocalizedLabel] [nvarchar](350) NULL 21 | ); 22 | GO 23 | 24 | CREATE PROCEDURE [DataverseToSql].[Merge_OptionsetMetadata] 25 | @source [DataverseToSql].[OptionsetMetadata_TableType] READONLY 26 | AS 27 | BEGIN TRAN 28 | 29 | TRUNCATE TABLE [$$SCHEMA$$].[OptionsetMetadata] 30 | 31 | INSERT [$$SCHEMA$$].[OptionsetMetadata] 32 | SELECT * FROM @source 33 | 34 | COMMIT -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/Optionsets_StateMetadata.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE TABLE [$$SCHEMA$$].[StateMetadata] ( 5 | [EntityName] [nvarchar](64) NULL, 6 | [State] [int] NULL, 7 | [IsUserLocalizedLabel] [bit] NULL, 8 | [LocalizedLabelLanguageCode] [int] NULL, 9 | [LocalizedLabel] [nvarchar](350) NULL 10 | ); 11 | GO 12 | 13 | CREATE TYPE [DataverseToSql].[StateMetadata_TableType] AS TABLE ( 14 | [EntityName] [nvarchar](64) NULL, 15 | [State] [int] NULL, 16 | [IsUserLocalizedLabel] [bit] NULL, 17 | [LocalizedLabelLanguageCode] [int] NULL, 18 | [LocalizedLabel] [nvarchar](350) NULL 19 | ); 20 | GO 21 | 22 | CREATE PROCEDURE [DataverseToSql].[Merge_StateMetadata] 23 | @source [DataverseToSql].[StateMetadata_TableType] READONLY 24 | AS 25 | BEGIN TRAN 26 | 27 | TRUNCATE TABLE [$$SCHEMA$$].[StateMetadata] 28 | 29 | INSERT [$$SCHEMA$$].[StateMetadata] 30 | SELECT * FROM @source 31 | 32 | COMMIT -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/Optionsets_StatusMetadata.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE TABLE [$$SCHEMA$$].[StatusMetadata] ( 5 | [EntityName] [nvarchar](64) NULL, 6 | [State] [int] NULL, 7 | [Status] [int] NULL, 8 | [IsUserLocalizedLabel] [bit] NULL, 9 | [LocalizedLabelLanguageCode] [int] NULL, 10 | [LocalizedLabel] [nvarchar](350) NULL 11 | ); 12 | GO 13 | 14 | CREATE TYPE [DataverseToSql].[StatusMetadata_TableType] AS TABLE ( 15 | [EntityName] [nvarchar](64) NULL, 16 | [State] [int] NULL, 17 | [Status] [int] NULL, 18 | [IsUserLocalizedLabel] [bit] NULL, 19 | [LocalizedLabelLanguageCode] [int] NULL, 20 | [LocalizedLabel] [nvarchar](350) NULL 21 | ); 22 | GO 23 | 24 | CREATE PROCEDURE [DataverseToSql].[Merge_StatusMetadata] 25 | @source [DataverseToSql].[StatusMetadata_TableType] READONLY 26 | AS 27 | BEGIN TRAN 28 | 29 | TRUNCATE TABLE [$$SCHEMA$$].[StatusMetadata] 30 | 31 | INSERT [$$SCHEMA$$].[StatusMetadata] 32 | SELECT * FROM @source 33 | 34 | COMMIT -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/SqlObjects/Optionsets_TargetMetadata.sql: -------------------------------------------------------------------------------- 1 | -- Copyright (c) Microsoft Corporation. 2 | -- Licensed under the MIT License. 3 | 4 | CREATE TABLE [$$SCHEMA$$].[TargetMetadata] ( 5 | [EntityName] [nvarchar](64) NULL, 6 | [AttributeName] [nvarchar](64) NULL, 7 | [ReferencedEntity] [nvarchar](64) NULL, 8 | [ReferencedAttribute] [nvarchar](64) NULL 9 | ); 10 | GO 11 | 12 | CREATE TYPE [DataverseToSql].[TargetMetadata_TableType] AS TABLE ( 13 | [EntityName] [nvarchar](64) NULL, 14 | [AttributeName] [nvarchar](64) NULL, 15 | [ReferencedEntity] [nvarchar](64) NULL, 16 | [ReferencedAttribute] [nvarchar](64) NULL 17 | ); 18 | GO 19 | 20 | CREATE PROCEDURE [DataverseToSql].[Merge_TargetMetadata] 21 | @source [DataverseToSql].[TargetMetadata_TableType] READONLY 22 | AS 23 | BEGIN TRAN 24 | 25 | TRUNCATE TABLE [$$SCHEMA$$].[TargetMetadata] 26 | 27 | INSERT [$$SCHEMA$$].[TargetMetadata] 28 | SELECT * FROM @source 29 | 30 | COMMIT -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/StringHashExtension.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using System.Security.Cryptography; 5 | using System.Text; 6 | 7 | namespace DataverseToSql.Core 8 | { 9 | /// 10 | /// Provides an extension method for strings to compute SHA1. 11 | /// 12 | internal static class StringHashExtension 13 | { 14 | internal static string Sha1(this string sourceString) 15 | { 16 | var sha1 = SHA1.Create(); 17 | var hash = sha1.ComputeHash(Encoding.UTF8.GetBytes(sourceString)); 18 | return Convert.ToBase64String(hash); 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Core/TokenCredentialExtensions.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Core; 5 | 6 | namespace DataverseToSql.Core 7 | { 8 | /// 9 | /// Provide an extension method to TokenCredential to retrieve a token for 10 | /// some spefied scopes. 11 | /// 12 | internal static class TokenCredentialExtensions 13 | { 14 | internal static string GetToken(this TokenCredential credential, string[] scopes) 15 | { 16 | return credential.GetToken( 17 | new(scopes), 18 | default 19 | ).Token; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Function/.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # Azure Functions localsettings file 5 | local.settings.json 6 | 7 | # User-specific files 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Build results 17 | [Dd]ebug/ 18 | [Dd]ebugPublic/ 19 | [Rr]elease/ 20 | [Rr]eleases/ 21 | x64/ 22 | x86/ 23 | bld/ 24 | [Bb]in/ 25 | [Oo]bj/ 26 | [Ll]og/ 27 | 28 | # Visual Studio 2015 cache/options directory 29 | .vs/ 30 | # Uncomment if you have tasks that create the project's static files in wwwroot 31 | #wwwroot/ 32 | 33 | # MSTest test Results 34 | [Tt]est[Rr]esult*/ 35 | [Bb]uild[Ll]og.* 36 | 37 | # NUNIT 38 | *.VisualState.xml 39 | TestResult.xml 40 | 41 | # Build Results of an ATL Project 42 | [Dd]ebugPS/ 43 | [Rr]eleasePS/ 44 | dlldata.c 45 | 46 | # DNX 47 | project.lock.json 48 | project.fragment.lock.json 49 | artifacts/ 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # NCrunch 117 | _NCrunch_* 118 | .*crunch*.local.xml 119 | nCrunchTemp_* 120 | 121 | # MightyMoose 122 | *.mm.* 123 | AutoTest.Net/ 124 | 125 | # Web workbench (sass) 126 | .sass-cache/ 127 | 128 | # Installshield output folder 129 | [Ee]xpress/ 130 | 131 | # DocProject is a documentation generator add-in 132 | DocProject/buildhelp/ 133 | DocProject/Help/*.HxT 134 | DocProject/Help/*.HxC 135 | DocProject/Help/*.hhc 136 | DocProject/Help/*.hhk 137 | DocProject/Help/*.hhp 138 | DocProject/Help/Html2 139 | DocProject/Help/html 140 | 141 | # Click-Once directory 142 | publish/ 143 | 144 | # Publish Web Output 145 | *.[Pp]ublish.xml 146 | *.azurePubxml 147 | # TODO: Comment the next line if you want to checkin your web deploy settings 148 | # but database connection strings (with potential passwords) will be unencrypted 149 | #*.pubxml 150 | *.publishproj 151 | 152 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 153 | # checkin your Azure Web App publish settings, but sensitive information contained 154 | # in these scripts will be unencrypted 155 | PublishScripts/ 156 | 157 | # NuGet Packages 158 | *.nupkg 159 | # The packages folder can be ignored because of Package Restore 160 | **/packages/* 161 | # except build/, which is used as an MSBuild target. 162 | !**/packages/build/ 163 | # Uncomment if necessary however generally it will be regenerated when needed 164 | #!**/packages/repositories.config 165 | # NuGet v3's project.json files produces more ignoreable files 166 | *.nuget.props 167 | *.nuget.targets 168 | 169 | # Microsoft Azure Build Output 170 | csx/ 171 | *.build.csdef 172 | 173 | # Microsoft Azure Emulator 174 | ecf/ 175 | rcf/ 176 | 177 | # Windows Store app package directories and files 178 | AppPackages/ 179 | BundleArtifacts/ 180 | Package.StoreAssociation.xml 181 | _pkginfo.txt 182 | 183 | # Visual Studio cache files 184 | # files ending in .cache can be ignored 185 | *.[Cc]ache 186 | # but keep track of directories ending in .cache 187 | !*.[Cc]ache/ 188 | 189 | # Others 190 | ClientBin/ 191 | ~$* 192 | *~ 193 | *.dbmdl 194 | *.dbproj.schemaview 195 | *.jfm 196 | *.pfx 197 | *.publishsettings 198 | node_modules/ 199 | orleans.codegen.cs 200 | 201 | # Since there are multiple workflows, uncomment next line to ignore bower_components 202 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 203 | #bower_components/ 204 | 205 | # RIA/Silverlight projects 206 | Generated_Code/ 207 | 208 | # Backup & report files from converting an old project file 209 | # to a newer Visual Studio version. Backup files are not needed, 210 | # because we have git ;-) 211 | _UpgradeReport_Files/ 212 | Backup*/ 213 | UpgradeLog*.XML 214 | UpgradeLog*.htm 215 | 216 | # SQL Server files 217 | *.mdf 218 | *.ldf 219 | 220 | # Business Intelligence projects 221 | *.rdl.data 222 | *.bim.layout 223 | *.bim_*.settings 224 | 225 | # Microsoft Fakes 226 | FakesAssemblies/ 227 | 228 | # GhostDoc plugin setting file 229 | *.GhostDoc.xml 230 | 231 | # Node.js Tools for Visual Studio 232 | .ntvs_analysis.dat 233 | 234 | # Visual Studio 6 build log 235 | *.plg 236 | 237 | # Visual Studio 6 workspace options file 238 | *.opt 239 | 240 | # Visual Studio LightSwitch build output 241 | **/*.HTMLClient/GeneratedArtifacts 242 | **/*.DesktopClient/GeneratedArtifacts 243 | **/*.DesktopClient/ModelManifest.xml 244 | **/*.Server/GeneratedArtifacts 245 | **/*.Server/ModelManifest.xml 246 | _Pvt_Extensions 247 | 248 | # Paket dependency manager 249 | .paket/paket.exe 250 | paket-files/ 251 | 252 | # FAKE - F# Make 253 | .fake/ 254 | 255 | # JetBrains Rider 256 | .idea/ 257 | *.sln.iml 258 | 259 | # CodeRush 260 | .cr/ 261 | 262 | # Python Tools for Visual Studio (PTVS) 263 | __pycache__/ 264 | *.pyc -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Function/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-azuretools.vscode-azurefunctions", 4 | "ms-dotnettools.csharp" 5 | ] 6 | } -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Function/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Attach to .NET Functions", 6 | "type": "coreclr", 7 | "request": "attach", 8 | "processId": "${command:azureFunctions.pickProcess}" 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Function/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "azureFunctions.deploySubpath": "bin/Release/net6.0/publish", 3 | "azureFunctions.projectLanguage": "C#", 4 | "azureFunctions.projectRuntime": "~4", 5 | "debug.internalConsoleOptions": "neverOpen", 6 | "azureFunctions.preDeployTask": "publish (functions)" 7 | } -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Function/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "clean (functions)", 6 | "command": "dotnet", 7 | "args": [ 8 | "clean", 9 | "/property:GenerateFullPaths=true", 10 | "/consoleloggerparameters:NoSummary" 11 | ], 12 | "type": "process", 13 | "problemMatcher": "$msCompile" 14 | }, 15 | { 16 | "label": "build (functions)", 17 | "command": "dotnet", 18 | "args": [ 19 | "build", 20 | "/property:GenerateFullPaths=true", 21 | "/consoleloggerparameters:NoSummary" 22 | ], 23 | "type": "process", 24 | "dependsOn": "clean (functions)", 25 | "group": { 26 | "kind": "build", 27 | "isDefault": true 28 | }, 29 | "problemMatcher": "$msCompile" 30 | }, 31 | { 32 | "label": "clean release (functions)", 33 | "command": "dotnet", 34 | "args": [ 35 | "clean", 36 | "--configuration", 37 | "Release", 38 | "/property:GenerateFullPaths=true", 39 | "/consoleloggerparameters:NoSummary" 40 | ], 41 | "type": "process", 42 | "problemMatcher": "$msCompile" 43 | }, 44 | { 45 | "label": "publish (functions)", 46 | "command": "dotnet", 47 | "args": [ 48 | "publish", 49 | "--configuration", 50 | "Release", 51 | "/property:GenerateFullPaths=true", 52 | "/consoleloggerparameters:NoSummary" 53 | ], 54 | "type": "process", 55 | "dependsOn": "clean release (functions)", 56 | "problemMatcher": "$msCompile" 57 | }, 58 | { 59 | "type": "func", 60 | "dependsOn": "build (functions)", 61 | "options": { 62 | "cwd": "${workspaceFolder}/bin/Debug/net6.0" 63 | }, 64 | "command": "host start", 65 | "isBackground": true, 66 | "problemMatcher": "$func-dotnet-watch" 67 | } 68 | ] 69 | } -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Function/DataverseToSql.Function.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | net6.0 4 | v4 5 | 0.1.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | PreserveNewest 16 | 17 | 18 | PreserveNewest 19 | Never 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Function/Ingestion.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | using Azure.Core; 5 | using Azure.Identity; 6 | using Azure.Storage.Blobs; 7 | using DataverseToSql.Core; 8 | using DataverseToSql.Core.Jobs; 9 | using Microsoft.Azure.WebJobs; 10 | using Microsoft.Extensions.Logging; 11 | using System; 12 | using System.Threading.Tasks; 13 | 14 | namespace DataverseToSql.Function 15 | { 16 | public class Ingestion 17 | { 18 | [FunctionName("Ingestion")] 19 | public async Task Run([TimerTrigger("%TIMER_SCHEDULE%")] TimerInfo myTimer, ILogger log) 20 | { 21 | var cred = new DefaultAzureCredential(); 22 | 23 | // Instantiate the BlobContainerClient to read configuration. 24 | var blobContainerClient = new BlobContainerClient( 25 | new Uri(Environment.GetEnvironmentVariable("CONFIGURATION_CONTAINER")), 26 | cred, 27 | default); 28 | 29 | // Instantiate the CloudEnvironment from the specified container. 30 | var env = new CloudEnvironment(log, cred, blobContainerClient); 31 | 32 | // Run the ingestion job 33 | var job = new IngestionJob(log, env); 34 | await job.RunAsync(); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Function/Properties/serviceDependencies.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "appInsights1": { 4 | "type": "appInsights" 5 | }, 6 | "storage1": { 7 | "type": "storage", 8 | "connectionId": "AzureWebJobsStorage" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Function/Properties/serviceDependencies.local.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "appInsights1": { 4 | "type": "appInsights.sdk" 5 | }, 6 | "storage1": { 7 | "type": "storage.emulator", 8 | "connectionId": "AzureWebJobsStorage" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.Function/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "logging": { 4 | "applicationInsights": { 5 | "samplingSettings": { 6 | "isEnabled": true, 7 | "excludedTypes": "Request" 8 | } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /src/DataverseToSql/DataverseToSql.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.3.32811.315 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataverseToSql.Core", "DataverseToSql.Core\DataverseToSql.Core.csproj", "{3F0B5376-B376-4993-9A69-9F349F5EA4FD}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataverseToSql.Cli", "DataverseToSql.Cli\DataverseToSql.Cli.csproj", "{C94625AA-E9B4-4440-A42F-9BCFAE8903A9}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataverseToSql.Function", "DataverseToSql.Function\DataverseToSql.Function.csproj", "{18944B2C-E048-4AEC-970D-706399699AA7}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureStorageFileProvider", "AzureStorageFileProvider\AzureStorageFileProvider.csproj", "{D8540958-8097-4870-ACE8-C2EDD98EFA08}" 13 | EndProject 14 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BlockBlobClientRangeCopyExtension", "BlockBlobClientRangeCopyExtension\BlockBlobClientRangeCopyExtension.csproj", "{05513D19-00D2-40E5-9134-D8646BD97F8A}" 15 | EndProject 16 | Global 17 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 18 | Debug|Any CPU = Debug|Any CPU 19 | Release|Any CPU = Release|Any CPU 20 | EndGlobalSection 21 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 22 | {3F0B5376-B376-4993-9A69-9F349F5EA4FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 23 | {3F0B5376-B376-4993-9A69-9F349F5EA4FD}.Debug|Any CPU.Build.0 = Debug|Any CPU 24 | {3F0B5376-B376-4993-9A69-9F349F5EA4FD}.Release|Any CPU.ActiveCfg = Release|Any CPU 25 | {3F0B5376-B376-4993-9A69-9F349F5EA4FD}.Release|Any CPU.Build.0 = Release|Any CPU 26 | {C94625AA-E9B4-4440-A42F-9BCFAE8903A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 27 | {C94625AA-E9B4-4440-A42F-9BCFAE8903A9}.Debug|Any CPU.Build.0 = Debug|Any CPU 28 | {C94625AA-E9B4-4440-A42F-9BCFAE8903A9}.Release|Any CPU.ActiveCfg = Release|Any CPU 29 | {C94625AA-E9B4-4440-A42F-9BCFAE8903A9}.Release|Any CPU.Build.0 = Release|Any CPU 30 | {18944B2C-E048-4AEC-970D-706399699AA7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 31 | {18944B2C-E048-4AEC-970D-706399699AA7}.Debug|Any CPU.Build.0 = Debug|Any CPU 32 | {18944B2C-E048-4AEC-970D-706399699AA7}.Release|Any CPU.ActiveCfg = Release|Any CPU 33 | {18944B2C-E048-4AEC-970D-706399699AA7}.Release|Any CPU.Build.0 = Release|Any CPU 34 | {D8540958-8097-4870-ACE8-C2EDD98EFA08}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 35 | {D8540958-8097-4870-ACE8-C2EDD98EFA08}.Debug|Any CPU.Build.0 = Debug|Any CPU 36 | {D8540958-8097-4870-ACE8-C2EDD98EFA08}.Release|Any CPU.ActiveCfg = Release|Any CPU 37 | {D8540958-8097-4870-ACE8-C2EDD98EFA08}.Release|Any CPU.Build.0 = Release|Any CPU 38 | {05513D19-00D2-40E5-9134-D8646BD97F8A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 39 | {05513D19-00D2-40E5-9134-D8646BD97F8A}.Debug|Any CPU.Build.0 = Debug|Any CPU 40 | {05513D19-00D2-40E5-9134-D8646BD97F8A}.Release|Any CPU.ActiveCfg = Release|Any CPU 41 | {05513D19-00D2-40E5-9134-D8646BD97F8A}.Release|Any CPU.Build.0 = Release|Any CPU 42 | EndGlobalSection 43 | GlobalSection(SolutionProperties) = preSolution 44 | HideSolutionNode = FALSE 45 | EndGlobalSection 46 | GlobalSection(ExtensibilityGlobals) = postSolution 47 | SolutionGuid = {F6F44F42-1D38-4A8D-9694-52F4861F3653} 48 | EndGlobalSection 49 | EndGlobal 50 | --------------------------------------------------------------------------------