├── README.md
├── IaaS
└── AzVmPricing
│ ├── Install-Modules.ps1
│ ├── azVMs.pbix
│ ├── SQL Licensing.xlsx
│ └── Download-AzVmPricing.ps1
├── AzFunctions
├── Minimal
│ └── C#
│ │ ├── .vs
│ │ ├── ExtractApp
│ │ │ ├── FileContentIndex
│ │ │ │ ├── read.lock
│ │ │ │ ├── 528830b3-7b47-4811-a194-a6bc36bcf3a6.vsidx
│ │ │ │ ├── 73de7e46-7966-4da1-92e1-b85a85c85f38.vsidx
│ │ │ │ └── b1ac9f6f-af7c-42b1-a1fe-db55918a3419.vsidx
│ │ │ ├── v17
│ │ │ │ ├── .suo
│ │ │ │ ├── .futdcache.v1
│ │ │ │ └── TestStore
│ │ │ │ │ └── 0
│ │ │ │ │ ├── 000.testlog
│ │ │ │ │ └── testlog.manifest
│ │ │ └── DesignTimeBuild
│ │ │ │ └── .dtbcache.v2
│ │ └── ProjectEvaluation
│ │ │ ├── extractapp.metadata.v2
│ │ │ └── extractapp.projects.v2
│ │ ├── TestProject1
│ │ ├── Usings.cs
│ │ ├── TestProject1.csproj
│ │ └── UnitTest1.cs
│ │ ├── ExtractApp
│ │ ├── Properties
│ │ │ ├── launchSettings.json
│ │ │ ├── serviceDependencies.json
│ │ │ ├── serviceDependencies.local.json
│ │ │ ├── PublishProfiles
│ │ │ │ └── AzExtractApp - Zip Deploy.pubxml
│ │ │ └── ServiceDependencies
│ │ │ │ └── AzExtractApp - Zip Deploy
│ │ │ │ └── profile.arm.json
│ │ ├── host.json
│ │ ├── ExtractApp.sln.DotSettings
│ │ ├── ExtractApp.csproj
│ │ ├── HelloWorld.cs
│ │ ├── .gitignore
│ │ └── ExtractCSV.cs
│ │ ├── ExtractApp.sln.DotSettings
│ │ ├── CreateTable.sql
│ │ └── ExtractApp.sln
├── ADF.PNG
├── UnitTest.PNG
├── Performance.PNG
└── README.md
├── PBI
└── CRM
│ ├── DAX
│ ├── 2. Rank.dax.queryBuilder
│ ├── q. LENE.dax.queryBuilder
│ ├── 2. Rank.dax
│ ├── q. LENE.dax
│ └── q. LENE.dax.serverTimings
│ ├── Demo
│ ├── Hubspot.bak
│ ├── test-Pipeline.csv
│ ├── test-companies.csv
│ └── test-deals.csv
│ ├── Sales Orders.pbix
│ ├── CRM-Hubspot-v1.pbix
│ ├── CRM-Hubspot-v2.pbix
│ └── Modellign Workflow and CRM.pptx
└── .gitignore
/README.md:
--------------------------------------------------------------------------------
1 | # Demos
2 | Prodata Demo Scripts
3 |
--------------------------------------------------------------------------------
/IaaS/AzVmPricing/Install-Modules.ps1:
--------------------------------------------------------------------------------
1 | Install-Module Az
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/.vs/ExtractApp/FileContentIndex/read.lock:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/PBI/CRM/DAX/2. Rank.dax.queryBuilder:
--------------------------------------------------------------------------------
1 | {"Columns":[],"Filters":{"Items":[]}}
--------------------------------------------------------------------------------
/PBI/CRM/DAX/q. LENE.dax.queryBuilder:
--------------------------------------------------------------------------------
1 | {"Columns":[],"Filters":{"Items":[]}}
--------------------------------------------------------------------------------
/AzFunctions/ADF.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/ADF.PNG
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/TestProject1/Usings.cs:
--------------------------------------------------------------------------------
1 | global using Microsoft.VisualStudio.TestTools.UnitTesting;
--------------------------------------------------------------------------------
/AzFunctions/UnitTest.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/UnitTest.PNG
--------------------------------------------------------------------------------
/PBI/CRM/Demo/Hubspot.bak:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/PBI/CRM/Demo/Hubspot.bak
--------------------------------------------------------------------------------
/PBI/CRM/Sales Orders.pbix:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/PBI/CRM/Sales Orders.pbix
--------------------------------------------------------------------------------
/AzFunctions/Performance.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/Performance.PNG
--------------------------------------------------------------------------------
/IaaS/AzVmPricing/azVMs.pbix:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/IaaS/AzVmPricing/azVMs.pbix
--------------------------------------------------------------------------------
/PBI/CRM/CRM-Hubspot-v1.pbix:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/PBI/CRM/CRM-Hubspot-v1.pbix
--------------------------------------------------------------------------------
/PBI/CRM/CRM-Hubspot-v2.pbix:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/PBI/CRM/CRM-Hubspot-v2.pbix
--------------------------------------------------------------------------------
/IaaS/AzVmPricing/SQL Licensing.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/IaaS/AzVmPricing/SQL Licensing.xlsx
--------------------------------------------------------------------------------
/PBI/CRM/Modellign Workflow and CRM.pptx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/PBI/CRM/Modellign Workflow and CRM.pptx
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/.vs/ExtractApp/v17/.suo:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/Minimal/C#/.vs/ExtractApp/v17/.suo
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/.vs/ExtractApp/v17/.futdcache.v1:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/Minimal/C#/.vs/ExtractApp/v17/.futdcache.v1
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/.vs/ExtractApp/v17/TestStore/0/000.testlog:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/Minimal/C#/.vs/ExtractApp/v17/TestStore/0/000.testlog
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/.vs/ExtractApp/DesignTimeBuild/.dtbcache.v2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/Minimal/C#/.vs/ExtractApp/DesignTimeBuild/.dtbcache.v2
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/.vs/ProjectEvaluation/extractapp.metadata.v2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/Minimal/C#/.vs/ProjectEvaluation/extractapp.metadata.v2
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/.vs/ProjectEvaluation/extractapp.projects.v2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/Minimal/C#/.vs/ProjectEvaluation/extractapp.projects.v2
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/.vs/ExtractApp/v17/TestStore/0/testlog.manifest:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/Minimal/C#/.vs/ExtractApp/v17/TestStore/0/testlog.manifest
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/.vs/ExtractApp/FileContentIndex/528830b3-7b47-4811-a194-a6bc36bcf3a6.vsidx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/Minimal/C#/.vs/ExtractApp/FileContentIndex/528830b3-7b47-4811-a194-a6bc36bcf3a6.vsidx
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/.vs/ExtractApp/FileContentIndex/73de7e46-7966-4da1-92e1-b85a85c85f38.vsidx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/Minimal/C#/.vs/ExtractApp/FileContentIndex/73de7e46-7966-4da1-92e1-b85a85c85f38.vsidx
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/.vs/ExtractApp/FileContentIndex/b1ac9f6f-af7c-42b1-a1fe-db55918a3419.vsidx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ProdataSQL/Demos/main/AzFunctions/Minimal/C#/.vs/ExtractApp/FileContentIndex/b1ac9f6f-af7c-42b1-a1fe-db55918a3419.vsidx
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/ExtractApp/Properties/launchSettings.json:
--------------------------------------------------------------------------------
1 | {
2 | "profiles": {
3 | "ExtractApp": {
4 | "commandName": "Project",
5 | "commandLineArgs": "--port 7227",
6 | "launchBrowser": false
7 | }
8 | }
9 | }
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/ExtractApp/Properties/serviceDependencies.json:
--------------------------------------------------------------------------------
1 | {
2 | "dependencies": {
3 | "appInsights1": {
4 | "type": "appInsights"
5 | },
6 | "storage1": {
7 | "type": "storage",
8 | "connectionId": "AzureWebJobsStorage"
9 | }
10 | }
11 | }
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/ExtractApp/host.json:
--------------------------------------------------------------------------------
1 | {
2 | "version": "2.0",
3 | "logging": {
4 | "applicationInsights": {
5 | "samplingSettings": {
6 | "isEnabled": true,
7 | "excludedTypes": "Request"
8 | }
9 | }
10 | }
11 | }
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/ExtractApp/Properties/serviceDependencies.local.json:
--------------------------------------------------------------------------------
1 | {
2 | "dependencies": {
3 | "appInsights1": {
4 | "type": "appInsights.sdk"
5 | },
6 | "storage1": {
7 | "type": "storage.emulator",
8 | "connectionId": "AzureWebJobsStorage"
9 | }
10 | }
11 | }
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | ################################################################################
2 | # This .gitignore file was automatically created by Microsoft(R) Visual Studio.
3 | ################################################################################
4 |
5 | /AzFunctions/Minimal/C#/TestProject1/bin
6 | /AzFunctions/Minimal/C#/TestProject1/obj
7 |
--------------------------------------------------------------------------------
/PBI/CRM/Demo/test-Pipeline.csv:
--------------------------------------------------------------------------------
1 | Pipeline,Deal_Stage,Probability,StageNo
2 | Sales Pipeline,Appointment scheduled,.30,2
3 | Sales Pipeline,Closed lost,.00,9
4 | Sales Pipeline,Closed won,1.00,9
5 | Sales Pipeline,Contract sent,.80,5
6 | Sales Pipeline,Decision maker bought-In,.70,4
7 | Sales Pipeline,Presentation scheduled,.40,3
8 | Sales Pipeline,Qualified to buy,.20,1
9 |
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/ExtractApp.sln.DotSettings:
--------------------------------------------------------------------------------
1 |
2 | True
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/ExtractApp/ExtractApp.sln.DotSettings:
--------------------------------------------------------------------------------
1 |
2 | True
3 | True
4 | True
5 | True
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/TestProject1/TestProject1.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | net6.0
5 | enable
6 | enable
7 |
8 | false
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/IaaS/AzVmPricing/Download-AzVmPricing.ps1:
--------------------------------------------------------------------------------
1 | $region = "northeurope"
2 | $Path= "C:\temp\AzComputeResourceSku.csv"
3 | (Get-AzComputeResourceSku -Location "northeurope" ) | where {$_.ResourceType -eq "virtualMachines" } | Select-Object @{Name='vmName'; Expression={ $_.Name } ; } -ExpandProperty Capabilities | Export-Csv -Path $Path
4 |
5 |
6 |
7 | $Path= "C:\temp\AzCost.csv"
8 | if (Test-Path $Path) {
9 | Remove-Item $Path
10 | }
11 | $Uri='https://prices.azure.com/api/retail/prices?currencyCode=''EUR''&$filter=serviceName eq ''Virtual Machines'' and priceType eq ''Consumption'' and armRegionName eq ''northeurope'''
12 | Do {
13 | $Uri
14 | $page=Invoke-RestMethod -Uri $Uri
15 | $Count=$page.Count
16 | $Uri=$page.NextPageLink
17 | $page.Items | Select-Object CurrencyCode, armSkuName , productId, skuId, location, retailPrice, type , unitOfMeasure | Export-Csv -Path $Path -Append
18 |
19 | } while ($Uri)
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/ExtractApp/ExtractApp.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 | net6.0
4 | v4
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 | PreserveNewest
15 |
16 |
17 | PreserveNewest
18 | Never
19 |
20 |
21 |
22 |
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/ExtractApp/Properties/PublishProfiles/AzExtractApp - Zip Deploy.pubxml:
--------------------------------------------------------------------------------
1 |
2 |
5 |
6 |
7 | ZipDeploy
8 | AzureWebSite
9 | Release
10 | Any CPU
11 | https://azextractapp.azurewebsites.net
12 | false
13 | /subscriptions/c9d4dcc8-20cb-47b6-8c52-a583abab20c3/resourcegroups/rg-MinimalFunctionApp/providers/Microsoft.Web/sites/AzExtractApp
14 | $AzExtractApp
15 | <_SavePWD>true
16 | true
17 | https://azextractapp.scm.azurewebsites.net/
18 |
19 |
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/CreateTable.sql:
--------------------------------------------------------------------------------
1 | /****** Object: Table [stg].[IMDB] Script Date: 02/04/2023 13:44:06 ******/
2 | IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[stg].[IMDB]') AND type in (N'U'))
3 | DROP TABLE [stg].[IMDB]
4 | GO
5 |
6 | /****** Object: Table [stg].[IMDB] Script Date: 02/04/2023 13:44:06 ******/
7 | SET ANSI_NULLS ON
8 | GO
9 |
10 | SET QUOTED_IDENTIFIER ON
11 | GO
12 |
13 | CREATE TABLE [stg].[IMDB]
14 | (
15 | [movie title] [varchar](4000) NULL,
16 | [Run Time] [varchar](4000) NULL,
17 | [Rating] [varchar](4000) NULL,
18 | [User Rating] [varchar](4000) NULL,
19 | [Generes] [varchar](4000) NULL,
20 | [Overview] [varchar](4000) NULL,
21 | [Plot Kyeword] [varchar](4000) NULL,
22 | [Director] [varchar](4000) NULL,
23 | [Top 5 Casts] [varchar](4000) NULL,
24 | [Writer] [varchar](4000) NULL,
25 | [year] [varchar](4000) NULL,
26 | [path] [varchar](4000) NULL
27 | )
28 | WITH
29 | (
30 | DISTRIBUTION = ROUND_ROBIN,
31 | HEAP
32 | )
33 | GO
34 |
35 |
36 |
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/ExtractApp/HelloWorld.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.IO;
3 | using System.Threading.Tasks;
4 | using Microsoft.AspNetCore.Mvc;
5 | using Microsoft.Azure.WebJobs;
6 | using Microsoft.Azure.WebJobs.Extensions.Http;
7 | using Microsoft.AspNetCore.Http;
8 | using Microsoft.Extensions.Logging;
9 | using Newtonsoft.Json;
10 |
11 | namespace ExtractApp
12 | {
13 | public static class HelloWorld
14 | {
15 | [FunctionName("HelloWorld")]
16 | public static async Task Run(
17 | [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
18 | ILogger log)
19 | {
20 | log.LogInformation("C# HTTP trigger function processed a request.");
21 |
22 | string name = req.Query["name"];
23 |
24 | string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
25 | dynamic data = JsonConvert.DeserializeObject(requestBody);
26 | name = name ?? data?.name;
27 |
28 | string responseMessage = string.IsNullOrEmpty(name)
29 | ? "This HTTP triggered function executed successfully. Pass a name in the query string or in the request body for a personalized response."
30 | : $"Hello, {name}. This HTTP triggered function executed successfully.";
31 |
32 | return new OkObjectResult(responseMessage);
33 | }
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/PBI/CRM/DAX/2. Rank.dax:
--------------------------------------------------------------------------------
1 | // DAX Query
2 | DEFINE
3 | VAR __DS0Core =
4 | SUMMARIZECOLUMNS(
5 | 'Date'[Month],
6 | 'Date'[MonthNo],
7 | ROLLUPADDISSUBTOTAL(
8 | ROLLUPGROUP('Pipeline'[Deal_Stage], 'Pipeline'[StageNo]), "IsGrandTotalColumnTotal"
9 | ),
10 | "LENE_Amount4", 'Deals'[LENE Amount4]
11 | )
12 |
13 | VAR __DS0PrimaryWindowed =
14 | TOPN(
15 | 101,
16 | SUMMARIZE(__DS0Core, 'Date'[Month], 'Date'[MonthNo]),
17 | 'Date'[MonthNo],
18 | 1,
19 | 'Date'[Month],
20 | 1
21 | )
22 |
23 | VAR __DS0SecondaryBase =
24 | SUMMARIZE(__DS0Core, 'Pipeline'[Deal_Stage], 'Pipeline'[StageNo], [IsGrandTotalColumnTotal])
25 |
26 | VAR __DS0Secondary =
27 | TOPN(
28 | 102,
29 | __DS0SecondaryBase,
30 | [IsGrandTotalColumnTotal],
31 | 1,
32 | 'Pipeline'[StageNo],
33 | 1,
34 | 'Pipeline'[Deal_Stage],
35 | 1
36 | )
37 |
38 | VAR __DS0BodyLimited =
39 | NATURALLEFTOUTERJOIN(
40 | __DS0PrimaryWindowed,
41 | SUBSTITUTEWITHINDEX(
42 | __DS0Core,
43 | "ColumnIndex",
44 | __DS0Secondary,
45 | [IsGrandTotalColumnTotal],
46 | ASC,
47 | 'Pipeline'[StageNo],
48 | ASC,
49 | 'Pipeline'[Deal_Stage],
50 | ASC
51 | )
52 | )
53 |
54 | EVALUATE
55 | __DS0Secondary
56 |
57 | ORDER BY
58 | [IsGrandTotalColumnTotal], 'Pipeline'[StageNo], 'Pipeline'[Deal_Stage]
59 |
60 | EVALUATE
61 | __DS0BodyLimited
62 |
63 | ORDER BY
64 | 'Date'[MonthNo], 'Date'[Month], [ColumnIndex]
65 |
--------------------------------------------------------------------------------
/PBI/CRM/DAX/q. LENE.dax:
--------------------------------------------------------------------------------
1 | // DAX Query
2 | DEFINE
3 | VAR __DS0Core =
4 | SUMMARIZECOLUMNS(
5 | 'Date'[Month],
6 | 'Date'[MonthNo],
7 | ROLLUPADDISSUBTOTAL(
8 | ROLLUPGROUP('Pipeline'[Deal_Stage], 'Pipeline'[StageNo]), "IsGrandTotalColumnTotal"
9 | ),
10 | "LENE_Amount3", 'Deals'[LENE Amount3]
11 | )
12 |
13 | VAR __DS0PrimaryWindowed =
14 | TOPN(
15 | 101,
16 | SUMMARIZE(__DS0Core, 'Date'[Month], 'Date'[MonthNo]),
17 | 'Date'[MonthNo],
18 | 1,
19 | 'Date'[Month],
20 | 1
21 | )
22 |
23 | VAR __DS0SecondaryBase =
24 | SUMMARIZE(__DS0Core, 'Pipeline'[Deal_Stage], 'Pipeline'[StageNo], [IsGrandTotalColumnTotal])
25 |
26 | VAR __DS0Secondary =
27 | TOPN(
28 | 102,
29 | __DS0SecondaryBase,
30 | [IsGrandTotalColumnTotal],
31 | 1,
32 | 'Pipeline'[StageNo],
33 | 1,
34 | 'Pipeline'[Deal_Stage],
35 | 1
36 | )
37 |
38 | VAR __DS0BodyLimited =
39 | NATURALLEFTOUTERJOIN(
40 | __DS0PrimaryWindowed,
41 | SUBSTITUTEWITHINDEX(
42 | __DS0Core,
43 | "ColumnIndex",
44 | __DS0Secondary,
45 | [IsGrandTotalColumnTotal],
46 | ASC,
47 | 'Pipeline'[StageNo],
48 | ASC,
49 | 'Pipeline'[Deal_Stage],
50 | ASC
51 | )
52 | )
53 |
54 | EVALUATE
55 | __DS0Secondary
56 |
57 | ORDER BY
58 | [IsGrandTotalColumnTotal], 'Pipeline'[StageNo], 'Pipeline'[Deal_Stage]
59 |
60 | EVALUATE
61 | __DS0BodyLimited
62 |
63 | ORDER BY
64 | 'Date'[MonthNo], 'Date'[Month], [ColumnIndex]
65 |
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/ExtractApp.sln:
--------------------------------------------------------------------------------
1 |
2 | Microsoft Visual Studio Solution File, Format Version 12.00
3 | # Visual Studio Version 17
4 | VisualStudioVersion = 17.2.32616.157
5 | MinimumVisualStudioVersion = 10.0.40219.1
6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ExtractApp", "ExtractApp\ExtractApp.csproj", "{54EA06E0-B2A5-44EC-8371-512BD33C3CB7}"
7 | EndProject
8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestProject1", "TestProject1\TestProject1.csproj", "{82FC1837-F9BD-42B2-B2A8-A309C433A18A}"
9 | EndProject
10 | Global
11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution
12 | Debug|Any CPU = Debug|Any CPU
13 | Release|Any CPU = Release|Any CPU
14 | EndGlobalSection
15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution
16 | {54EA06E0-B2A5-44EC-8371-512BD33C3CB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
17 | {54EA06E0-B2A5-44EC-8371-512BD33C3CB7}.Debug|Any CPU.Build.0 = Debug|Any CPU
18 | {54EA06E0-B2A5-44EC-8371-512BD33C3CB7}.Release|Any CPU.ActiveCfg = Release|Any CPU
19 | {54EA06E0-B2A5-44EC-8371-512BD33C3CB7}.Release|Any CPU.Build.0 = Release|Any CPU
20 | {82FC1837-F9BD-42B2-B2A8-A309C433A18A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
21 | {82FC1837-F9BD-42B2-B2A8-A309C433A18A}.Debug|Any CPU.Build.0 = Debug|Any CPU
22 | {82FC1837-F9BD-42B2-B2A8-A309C433A18A}.Release|Any CPU.ActiveCfg = Release|Any CPU
23 | {82FC1837-F9BD-42B2-B2A8-A309C433A18A}.Release|Any CPU.Build.0 = Release|Any CPU
24 | EndGlobalSection
25 | GlobalSection(SolutionProperties) = preSolution
26 | HideSolutionNode = FALSE
27 | EndGlobalSection
28 | GlobalSection(ExtensibilityGlobals) = postSolution
29 | SolutionGuid = {BF3A7949-187E-410E-BBDC-48346577C25F}
30 | EndGlobalSection
31 | EndGlobal
32 |
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/TestProject1/UnitTest1.cs:
--------------------------------------------------------------------------------
1 | using System.Linq.Expressions;
2 | using System.Text;
3 | using ExtractApp;
4 | using Microsoft.AspNetCore.Http;
5 | using Microsoft.AspNetCore.Http.Internal;
6 | using Microsoft.AspNetCore.Mvc;
7 | using Microsoft.Extensions.Logging.Abstractions;
8 | using Microsoft.Extensions.Primitives;
9 | using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery;
10 | using Newtonsoft.Json;
11 |
12 | namespace TestProject1
13 | {
14 | [TestClass]
15 | public class UnitTest1
16 | {
17 |
18 |
19 | [TestInitialize]
20 | public void Initialize()
21 | {
22 |
23 | }
24 |
25 | [TestMethod]
26 | [TestCategory("CSV")]
27 | [TestCategory("azFunctions")]
28 | [Description("Extract a CSV to SQL ASDL API => SqlBilkCopy")]
29 | public async Task ExtractCsv()
30 | {
31 | var bodyDictionary= new Dictionary()
32 | {
33 | {"storageAccount", "bobawdatalakedev"},
34 | {"container", "raw"},
35 | {"directory", ""},
36 | {"fileName", "25k IMDb movie Dataset.csv"},
37 | {"sqlServer", "swat-dev.database.windows.net"},
38 | {"sqlDatabase", "SwatDW"},
39 | {"sqlTable", "stg.IMDB"}
40 | };
41 |
42 |
43 |
44 | var json = JsonConvert.SerializeObject(bodyDictionary);
45 |
46 | var body = new MemoryStream(Encoding.ASCII.GetBytes(json));
47 |
48 | var req = new DefaultHttpRequest(new DefaultHttpContext())
49 | {
50 | Body = body
51 |
52 | };
53 |
54 | var logger = NullLoggerFactory.Instance.CreateLogger("Null Logger");
55 | var response = await ExtractCSV.Run(req, logger);
56 | var result = (OkObjectResult)response;
57 |
58 | Console.WriteLine(result.Value );
59 |
60 | }
61 | }
62 | }
--------------------------------------------------------------------------------
/AzFunctions/README.md:
--------------------------------------------------------------------------------
1 | # Sample C# Azure Function for Datalake and SQL
2 |
3 | Welcome to the sample c# Azure functions. This contains two sample projects written in Visual Studio 2019:
4 |
5 | - ExtractApp.csproj. A sample Azure function which reads a CSV using the DataLakeClient API and writes to a SQL Server using SqlBulkCopy API
6 | - TestProject1.csproj. A sample C# unit test project that shows how make a unit test for an azure function using a mock HTTP request object.
7 |
8 | ## Key concepts to demo
9 |
10 | - A template for writing c# Azure function that use Datalakes and SQL
11 | - Example of using the latest Azure.Identity for both Managed Identity when run from Azure and Visual Studio Azure Identity when run from Visual Studio.
12 | - Example use of the newer DataLakeServiceClient instead of the more generic BlobServiceClient.
13 | - Example use of SqlBulkCopy API to write a Dataset from Dot.Ney to SQL with a Bulk API.
14 | - Sample Dot.Net Unit Test project showing how to write unit tests to develop and test functions locally so you dont have to mess with debugging as much in Azure Portal.
15 | - Use of a local.settings.json file and Envioronment variables to have different settings in VS Studio versus Azure. Specificaaly the ExcludeManagedIdentityCredential boolean flag.
16 |
17 | ## Setup and Permissions
18 |
19 | To get this example to work you will need a few things:
20 |
21 | 1. An ASDL Storage Account with a container
22 | 2. Upload sampel CSV from https://www.kaggle.com/datasets/utsh0dey/25k-movie-dataset
23 | 3. A SQL DB or Syanpse SQL Pool with table created as per [CreateTable.sql](Minimal/C#/CreateTable.sql)
24 | 4. Azure function published from source code. [ExtractApp.sln](Minimal/C%23/ExtractApp.sln)
25 | 5. Managed Identity Setup on AzFunction and ADF as per [this blog](https://prodata.ie/2022/06/16/enabling-managed-identity-authentication-on-azure-functions-in-data-factory/)
26 | 6. Grant the Function App Identity permissions on the SqlServer. [this article](https://learn.microsoft.com/en-us/azure/azure-functions/functions-identity-access-azure-sql-with-managed-identity)
27 |
28 | This can be cumbersome, but good news its once off setup.
29 |
30 | ## Why use an Azure Function
31 |
32 | Theres a few scenarios where Azure Functions can be useful with data engineering, even if you are using ADF:
33 |
34 | - When the source file needs some high code solution to parse nasty files.
35 | - C# does allow for more streamlined file procesing. At the extepse of development time. This can lead to better performance. More than 2x in this case of a 12MB file.
36 | - When the low code ADF envionment is not flexible enough.
37 | - Azure functions can be considerably cheaper than ADF for copying files if you are not using a self Hosted Integration Run Time.
38 |
39 | ## Performance Testing ADF Copy v c# Azure function
40 |
41 | As a performance test we took a sample 12 MB CSV File which you can download from kaggle here
42 | https://www.kaggle.com/datasets/utsh0dey/25k-movie-dataset
43 |
44 | We then read the file and loaded it into SQL using two scenarios
45 | - The Standard Copy Activity using the SqlPool COPY option.
46 | - This sample c# azure function.
47 |
48 | Average run times are show below. Potentially the c# azure funciton was more than 2x faster then the ADF Copy Activity.
49 | 
50 |
51 | Notes:
52 |
53 | 1. We were using the most basic tier of azure functions on a Linux host. The "consumption" plan is about 1 core and 1.5GB ram so really limited in resources.
54 | 2. We read the file into a DataTable. Its a bit more complex but potentially faster and more scalable to use a streaming interface like iDataReader.
55 |
56 | ## Testing c# Azure Function from VS Unit Test and from ADF
57 |
58 | We can invoke the AzFunction either locally from VS Unit Test or once pulished from ADF. Examples of these two experiences are below
59 |
60 | c# Unit Test Running using Test Explorer
61 | 
62 |
63 | ADF Pipeline runing using Azure Function Activity
64 | 
65 |
66 | ## Going from this sample to Production Quality
67 |
68 | While the API calls are similar, there are some things we do slighty differently in a commerical/production solution. We didn't show them here to keep it as a simple demo:
69 |
70 | - We would not place the code inside the azure function. Better to put the logic into a dedicated Class solution.
71 | - We write unit tests based on the seperate logic classes rather than try and mock up azure function calls. This lets us develop and test code without worrying about any pecularities of azure functions.
72 | - We do not use the APIs like Azure.Identity, or DataLakeClient within the function or logic class. Instead we create our own library which wraps the APIs for easier re-use.
73 | - This example using a DataTable, sometimes we may develop data processing using a more streaming method such as and IDataReader interface, or chunking the DataTable into 1MB chunks.
74 | - Note that we use the full Visual Studio and not Visual Code for c# projects. This is mainly due to use of tools like VS Unit Test and Jet Brains to help development. We tend to use VS code more for scripting languages like Powershell and Python. But many authors do use VS Code for c#.
75 |
76 | ## References
77 |
78 | https://prodata.ie/2022/06/16/enabling-managed-identity-authentication-on-azure-functions-in-data-factory/
79 | https://www.mssqltips.com/sqlservertip/7532/retrieve-file-azure-blob-storage-azure-function/
80 | https://learn.microsoft.com/en-us/azure/azure-functions/functions-identity-access-azure-sql-with-managed-identity
81 |
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/ExtractApp/.gitignore:
--------------------------------------------------------------------------------
1 | ## Ignore Visual Studio temporary files, build results, and
2 | ## files generated by popular Visual Studio add-ons.
3 |
4 | # Azure Functions localsettings file
5 | local.settings.json
6 |
7 | # User-specific files
8 | *.suo
9 | *.user
10 | *.userosscache
11 | *.sln.docstates
12 |
13 | # User-specific files (MonoDevelop/Xamarin Studio)
14 | *.userprefs
15 |
16 | # Build results
17 | [Dd]ebug/
18 | [Dd]ebugPublic/
19 | [Rr]elease/
20 | [Rr]eleases/
21 | x64/
22 | x86/
23 | bld/
24 | [Bb]in/
25 | [Oo]bj/
26 | [Ll]og/
27 |
28 | # Visual Studio 2015 cache/options directory
29 | .vs/
30 | # Uncomment if you have tasks that create the project's static files in wwwroot
31 | #wwwroot/
32 |
33 | # MSTest test Results
34 | [Tt]est[Rr]esult*/
35 | [Bb]uild[Ll]og.*
36 |
37 | # NUNIT
38 | *.VisualState.xml
39 | TestResult.xml
40 |
41 | # Build Results of an ATL Project
42 | [Dd]ebugPS/
43 | [Rr]eleasePS/
44 | dlldata.c
45 |
46 | # DNX
47 | project.lock.json
48 | project.fragment.lock.json
49 | artifacts/
50 |
51 | *_i.c
52 | *_p.c
53 | *_i.h
54 | *.ilk
55 | *.meta
56 | *.obj
57 | *.pch
58 | *.pdb
59 | *.pgc
60 | *.pgd
61 | *.rsp
62 | *.sbr
63 | *.tlb
64 | *.tli
65 | *.tlh
66 | *.tmp
67 | *.tmp_proj
68 | *.log
69 | *.vspscc
70 | *.vssscc
71 | .builds
72 | *.pidb
73 | *.svclog
74 | *.scc
75 |
76 | # Chutzpah Test files
77 | _Chutzpah*
78 |
79 | # Visual C++ cache files
80 | ipch/
81 | *.aps
82 | *.ncb
83 | *.opendb
84 | *.opensdf
85 | *.sdf
86 | *.cachefile
87 | *.VC.db
88 | *.VC.VC.opendb
89 |
90 | # Visual Studio profiler
91 | *.psess
92 | *.vsp
93 | *.vspx
94 | *.sap
95 |
96 | # TFS 2012 Local Workspace
97 | $tf/
98 |
99 | # Guidance Automation Toolkit
100 | *.gpState
101 |
102 | # ReSharper is a .NET coding add-in
103 | _ReSharper*/
104 | *.[Rr]e[Ss]harper
105 | *.DotSettings.user
106 |
107 | # JustCode is a .NET coding add-in
108 | .JustCode
109 |
110 | # TeamCity is a build add-in
111 | _TeamCity*
112 |
113 | # DotCover is a Code Coverage Tool
114 | *.dotCover
115 |
116 | # NCrunch
117 | _NCrunch_*
118 | .*crunch*.local.xml
119 | nCrunchTemp_*
120 |
121 | # MightyMoose
122 | *.mm.*
123 | AutoTest.Net/
124 |
125 | # Web workbench (sass)
126 | .sass-cache/
127 |
128 | # Installshield output folder
129 | [Ee]xpress/
130 |
131 | # DocProject is a documentation generator add-in
132 | DocProject/buildhelp/
133 | DocProject/Help/*.HxT
134 | DocProject/Help/*.HxC
135 | DocProject/Help/*.hhc
136 | DocProject/Help/*.hhk
137 | DocProject/Help/*.hhp
138 | DocProject/Help/Html2
139 | DocProject/Help/html
140 |
141 | # Click-Once directory
142 | publish/
143 |
144 | # Publish Web Output
145 | *.[Pp]ublish.xml
146 | *.azurePubxml
147 | # TODO: Comment the next line if you want to checkin your web deploy settings
148 | # but database connection strings (with potential passwords) will be unencrypted
149 | #*.pubxml
150 | *.publishproj
151 |
152 | # Microsoft Azure Web App publish settings. Comment the next line if you want to
153 | # checkin your Azure Web App publish settings, but sensitive information contained
154 | # in these scripts will be unencrypted
155 | PublishScripts/
156 |
157 | # NuGet Packages
158 | *.nupkg
159 | # The packages folder can be ignored because of Package Restore
160 | **/packages/*
161 | # except build/, which is used as an MSBuild target.
162 | !**/packages/build/
163 | # Uncomment if necessary however generally it will be regenerated when needed
164 | #!**/packages/repositories.config
165 | # NuGet v3's project.json files produces more ignoreable files
166 | *.nuget.props
167 | *.nuget.targets
168 |
169 | # Microsoft Azure Build Output
170 | csx/
171 | *.build.csdef
172 |
173 | # Microsoft Azure Emulator
174 | ecf/
175 | rcf/
176 |
177 | # Windows Store app package directories and files
178 | AppPackages/
179 | BundleArtifacts/
180 | Package.StoreAssociation.xml
181 | _pkginfo.txt
182 |
183 | # Visual Studio cache files
184 | # files ending in .cache can be ignored
185 | *.[Cc]ache
186 | # but keep track of directories ending in .cache
187 | !*.[Cc]ache/
188 |
189 | # Others
190 | ClientBin/
191 | ~$*
192 | *~
193 | *.dbmdl
194 | *.dbproj.schemaview
195 | *.jfm
196 | *.pfx
197 | *.publishsettings
198 | node_modules/
199 | orleans.codegen.cs
200 |
201 | # Since there are multiple workflows, uncomment next line to ignore bower_components
202 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
203 | #bower_components/
204 |
205 | # RIA/Silverlight projects
206 | Generated_Code/
207 |
208 | # Backup & report files from converting an old project file
209 | # to a newer Visual Studio version. Backup files are not needed,
210 | # because we have git ;-)
211 | _UpgradeReport_Files/
212 | Backup*/
213 | UpgradeLog*.XML
214 | UpgradeLog*.htm
215 |
216 | # SQL Server files
217 | *.mdf
218 | *.ldf
219 |
220 | # Business Intelligence projects
221 | *.rdl.data
222 | *.bim.layout
223 | *.bim_*.settings
224 |
225 | # Microsoft Fakes
226 | FakesAssemblies/
227 |
228 | # GhostDoc plugin setting file
229 | *.GhostDoc.xml
230 |
231 | # Node.js Tools for Visual Studio
232 | .ntvs_analysis.dat
233 |
234 | # Visual Studio 6 build log
235 | *.plg
236 |
237 | # Visual Studio 6 workspace options file
238 | *.opt
239 |
240 | # Visual Studio LightSwitch build output
241 | **/*.HTMLClient/GeneratedArtifacts
242 | **/*.DesktopClient/GeneratedArtifacts
243 | **/*.DesktopClient/ModelManifest.xml
244 | **/*.Server/GeneratedArtifacts
245 | **/*.Server/ModelManifest.xml
246 | _Pvt_Extensions
247 |
248 | # Paket dependency manager
249 | .paket/paket.exe
250 | paket-files/
251 |
252 | # FAKE - F# Make
253 | .fake/
254 |
255 | # JetBrains Rider
256 | .idea/
257 | *.sln.iml
258 |
259 | # CodeRush
260 | .cr/
261 |
262 | # Python Tools for Visual Studio (PTVS)
263 | __pycache__/
264 | *.pyc
--------------------------------------------------------------------------------
/AzFunctions/Minimal/C#/ExtractApp/ExtractCSV.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.ComponentModel;
3 | using System.Data;
4 | using System.Data.SqlClient;
5 | using System.Diagnostics;
6 | using System.IO;
7 | using System.Linq;
8 | using System.Text.RegularExpressions;
9 | using System.Threading.Tasks;
10 | using Microsoft.AspNetCore.Mvc;
11 | using Microsoft.Azure.WebJobs;
12 | using Microsoft.Azure.WebJobs.Extensions.Http;
13 | using Microsoft.AspNetCore.Http;
14 | using Microsoft.Extensions.Logging;
15 | using Newtonsoft.Json;
16 | using Azure.Core;
17 | using Azure.Identity;
18 | using Azure.Storage.Files.DataLake;
19 | using Microsoft.Extensions.Configuration;
20 |
21 | namespace ExtractApp
22 | {
23 | public static class ExtractCSV
24 | {
25 |
26 | [FunctionName("ExtractCSV")]
27 | public static async Task Run(
28 | [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
29 | ILogger log)
30 | {
31 | /* Set Parameters */
32 | string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
33 | dynamic data = JsonConvert.DeserializeObject(requestBody);
34 | string storageAccount = data?.storageAccount;
35 | string container = data?.container;
36 | string directory = data?.directory;
37 | string fileName = data?.fileName;
38 | string sqlServer = data?.sqlServer;
39 | string sqlDatabase = data?.sqlDatabase;
40 | string sqlTable = data?.sqlTable;
41 | var dt = new DataTable();
42 | var csvParser = new Regex(",(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))"); //Regex to identify and Split Double Quote
43 |
44 | // read config file if present. we use this for toggle for ExcludeManagedIdentityCredential
45 | var config = new ConfigurationBuilder()
46 | .SetBasePath(Directory.GetCurrentDirectory())
47 | .AddJsonFile("local.settings.json",optional:true, reloadOnChange:true)
48 | .AddEnvironmentVariables()
49 | .Build();
50 |
51 |
52 | /* Authenticate to ASDL. Note that we have a setting to Exclude MSI or not*/
53 | var sw = new Stopwatch();
54 | sw.Start();
55 | var defaultAzureCredentialOptions = new DefaultAzureCredentialOptions()
56 | {
57 | ExcludeAzureCliCredential = true,
58 | ExcludeManagedIdentityCredential =
59 | Convert.ToBoolean(config["ExcludeManagedIdentityCredential"] ?? "false"),
60 | ExcludeSharedTokenCacheCredential = true,
61 | ExcludeVisualStudioCredential = false,
62 | ExcludeAzurePowerShellCredential = true,
63 | ExcludeEnvironmentCredential = true,
64 | ExcludeVisualStudioCodeCredential = true,
65 | ExcludeInteractiveBrowserCredential = true
66 | };
67 | var resource = $"https://{storageAccount}.blob.core.windows.net";
68 | var dataLakeServiceClient = new DataLakeServiceClient(new Uri(resource), new DefaultAzureCredential(defaultAzureCredentialOptions));
69 | var fileSystemClient = dataLakeServiceClient.GetFileSystemClient(container);
70 | var directoryClient = fileSystemClient.GetDirectoryClient(directory);
71 | var fileClient = directoryClient.GetFileClient(fileName);
72 |
73 | Console.WriteLine("ExtractCSV - Sample AzFunction to Extract CSV and write to SQL");
74 | Console.WriteLine("Source: {0}", System.IO.Path.Combine(storageAccount!,container!, directory!, fileName!));
75 | Console.WriteLine("Target: {0}.{1}.{2}", sqlServer,sqlDatabase, sqlTable);
76 |
77 | Console.WriteLine("Connected to ASDL: {0} ms", sw.ElapsedMilliseconds);
78 |
79 | /* Authenticate to SQL. */
80 | var con = new SqlConnection()
81 | {
82 | ConnectionString = $"Server={sqlServer};Database={sqlDatabase}",
83 | AccessToken = (await new DefaultAzureCredential(defaultAzureCredentialOptions).GetTokenAsync(new TokenRequestContext(new string[] { "https://database.windows.net//.default" }))).Token
84 | };
85 | await con.OpenAsync();
86 | Console.WriteLine("Connected to SQL: {0} ms", sw.ElapsedMilliseconds);
87 | sw.Restart();
88 |
89 | // Read CSV into DataTable
90 | var streamReader = new StreamReader(await fileClient.OpenReadAsync());
91 | string[] headers = (await streamReader.ReadLineAsync())!.Split(',');
92 | foreach (string header in headers)
93 | {
94 | dt.Columns.Add(header == "" ? "id" : header);
95 | }
96 |
97 | while (!streamReader.EndOfStream)
98 | {
99 | var line = await streamReader.ReadLineAsync();
100 | object[] columnArray = csvParser.Split(line!).ToArray