├── .gitignore ├── .vscode ├── extensions.json ├── launch.json ├── settings.json └── tasks.json ├── README.md ├── SQL ├── create-app-user.sql ├── create-objects.sql └── enable-bulk-load.sql ├── csv ├── csv.fmt └── test.csv └── function ├── .vscode ├── extensions.json ├── launch.json ├── settings.json └── tasks.json ├── ImportCSV.cs ├── functions-csv-to-sql.csproj ├── host.json └── run-local.txt /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # Azure Functions localsettings file 5 | local.settings.json 6 | 7 | # User-specific files 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Build results 17 | [Dd]ebug/ 18 | [Dd]ebugPublic/ 19 | [Rr]elease/ 20 | [Rr]eleases/ 21 | x64/ 22 | x86/ 23 | bld/ 24 | [Bb]in/ 25 | [Oo]bj/ 26 | [Ll]og/ 27 | 28 | # Visual Studio 2015 cache/options directory 29 | .vs/ 30 | # Uncomment if you have tasks that create the project's static files in wwwroot 31 | #wwwroot/ 32 | 33 | # MSTest test Results 34 | [Tt]est[Rr]esult*/ 35 | [Bb]uild[Ll]og.* 36 | 37 | # NUNIT 38 | *.VisualState.xml 39 | TestResult.xml 40 | 41 | # Build Results of an ATL Project 42 | [Dd]ebugPS/ 43 | [Rr]eleasePS/ 44 | dlldata.c 45 | 46 | # DNX 47 | project.lock.json 48 | project.fragment.lock.json 49 | artifacts/ 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # NCrunch 117 | _NCrunch_* 118 | .*crunch*.local.xml 119 | nCrunchTemp_* 120 | 121 | # MightyMoose 122 | *.mm.* 123 | AutoTest.Net/ 124 | 125 | # Web workbench (sass) 126 | .sass-cache/ 127 | 128 | # Installshield output folder 129 | [Ee]xpress/ 130 | 131 | # DocProject is a documentation generator add-in 132 | DocProject/buildhelp/ 133 | DocProject/Help/*.HxT 134 | DocProject/Help/*.HxC 135 | DocProject/Help/*.hhc 136 | DocProject/Help/*.hhk 137 | DocProject/Help/*.hhp 138 | DocProject/Help/Html2 139 | DocProject/Help/html 140 | 141 | # Click-Once directory 142 | publish/ 143 | 144 | # Publish Web Output 145 | *.[Pp]ublish.xml 146 | *.azurePubxml 147 | # TODO: Comment the next line if you want to checkin your web deploy settings 148 | # but database connection strings (with potential passwords) will be unencrypted 149 | #*.pubxml 150 | *.publishproj 151 | 152 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 153 | # checkin your Azure Web App publish settings, but sensitive information contained 154 | # in these scripts will be unencrypted 155 | PublishScripts/ 156 | 157 | # NuGet Packages 158 | *.nupkg 159 | # The packages folder can be ignored because of Package Restore 160 | **/packages/* 161 | # except build/, which is used as an MSBuild target. 162 | !**/packages/build/ 163 | # Uncomment if necessary however generally it will be regenerated when needed 164 | #!**/packages/repositories.config 165 | # NuGet v3's project.json files produces more ignoreable files 166 | *.nuget.props 167 | *.nuget.targets 168 | 169 | # Microsoft Azure Build Output 170 | csx/ 171 | *.build.csdef 172 | 173 | # Microsoft Azure Emulator 174 | ecf/ 175 | rcf/ 176 | 177 | # Windows Store app package directories and files 178 | AppPackages/ 179 | BundleArtifacts/ 180 | Package.StoreAssociation.xml 181 | _pkginfo.txt 182 | 183 | # Visual Studio cache files 184 | # files ending in .cache can be ignored 185 | *.[Cc]ache 186 | # but keep track of directories ending in .cache 187 | !*.[Cc]ache/ 188 | 189 | # Others 190 | ClientBin/ 191 | ~$* 192 | *~ 193 | *.dbmdl 194 | *.dbproj.schemaview 195 | *.jfm 196 | *.pfx 197 | *.publishsettings 198 | node_modules/ 199 | orleans.codegen.cs 200 | 201 | # Since there are multiple workflows, uncomment next line to ignore bower_components 202 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 203 | #bower_components/ 204 | 205 | # RIA/Silverlight projects 206 | Generated_Code/ 207 | 208 | # Backup & report files from converting an old project file 209 | # to a newer Visual Studio version. Backup files are not needed, 210 | # because we have git ;-) 211 | _UpgradeReport_Files/ 212 | Backup*/ 213 | UpgradeLog*.XML 214 | UpgradeLog*.htm 215 | 216 | # SQL Server files 217 | *.mdf 218 | *.ldf 219 | 220 | # Business Intelligence projects 221 | *.rdl.data 222 | *.bim.layout 223 | *.bim_*.settings 224 | 225 | # Microsoft Fakes 226 | FakesAssemblies/ 227 | 228 | # GhostDoc plugin setting file 229 | *.GhostDoc.xml 230 | 231 | # Node.js Tools for Visual Studio 232 | .ntvs_analysis.dat 233 | 234 | # Visual Studio 6 build log 235 | *.plg 236 | 237 | # Visual Studio 6 workspace options file 238 | *.opt 239 | 240 | # Visual Studio LightSwitch build output 241 | **/*.HTMLClient/GeneratedArtifacts 242 | **/*.DesktopClient/GeneratedArtifacts 243 | **/*.DesktopClient/ModelManifest.xml 244 | **/*.Server/GeneratedArtifacts 245 | **/*.Server/ModelManifest.xml 246 | _Pvt_Extensions 247 | 248 | # Paket dependency manager 249 | .paket/paket.exe 250 | paket-files/ 251 | 252 | # FAKE - F# Make 253 | .fake/ 254 | 255 | # JetBrains Rider 256 | .idea/ 257 | *.sln.iml 258 | 259 | # CodeRush 260 | .cr/ 261 | 262 | # Python Tools for Visual Studio (PTVS) 263 | __pycache__/ 264 | *.pyc -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-azuretools.vscode-azurefunctions", 4 | "ms-vscode.csharp" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Attach to .NET Functions", 6 | "type": "coreclr", 7 | "request": "attach", 8 | "processId": "${command:azureFunctions.pickProcess}" 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "azureFunctions.deploySubpath": "function/bin/Release/netcoreapp2.1/publish", 3 | "azureFunctions.projectLanguage": "C#", 4 | "azureFunctions.projectRuntime": "~2", 5 | "debug.internalConsoleOptions": "neverOpen", 6 | "azureFunctions.preDeployTask": "publish" 7 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "clean", 6 | "command": "dotnet", 7 | "args": [ 8 | "clean", 9 | "/property:GenerateFullPaths=true", 10 | "/consoleloggerparameters:NoSummary" 11 | ], 12 | "type": "process", 13 | "problemMatcher": "$msCompile", 14 | "options": { 15 | "cwd": "${workspaceFolder}/function" 16 | } 17 | }, 18 | { 19 | "label": "build", 20 | "command": "dotnet", 21 | "args": [ 22 | "build", 23 | "/property:GenerateFullPaths=true", 24 | "/consoleloggerparameters:NoSummary" 25 | ], 26 | "type": "process", 27 | "dependsOn": "clean", 28 | "group": { 29 | "kind": "build", 30 | "isDefault": true 31 | }, 32 | "problemMatcher": "$msCompile", 33 | "options": { 34 | "cwd": "${workspaceFolder}/function" 35 | } 36 | }, 37 | { 38 | "label": "clean release", 39 | "command": "dotnet", 40 | "args": [ 41 | "clean", 42 | "--configuration", 43 | "Release", 44 | "/property:GenerateFullPaths=true", 45 | "/consoleloggerparameters:NoSummary" 46 | ], 47 | "type": "process", 48 | "problemMatcher": "$msCompile", 49 | "options": { 50 | "cwd": "${workspaceFolder}/function" 51 | } 52 | }, 53 | { 54 | "label": "publish", 55 | "command": "dotnet", 56 | "args": [ 57 | "publish", 58 | "--configuration", 59 | "Release", 60 | "/property:GenerateFullPaths=true", 61 | "/consoleloggerparameters:NoSummary" 62 | ], 63 | "type": "process", 64 | "dependsOn": "clean release", 65 | "problemMatcher": "$msCompile", 66 | "options": { 67 | "cwd": "${workspaceFolder}/function" 68 | } 69 | }, 70 | { 71 | "type": "func", 72 | "dependsOn": "build", 73 | "options": { 74 | "cwd": "${workspaceFolder}/function/bin/Debug/netcoreapp2.1" 75 | }, 76 | "command": "host start", 77 | "isBackground": true, 78 | "problemMatcher": "$func-watch" 79 | } 80 | ] 81 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Upload CSV to SQL Server via Azure Function 2 | 3 | A sample template that allows to upload the content of a CSV file to a Azure SQL database as soon as the .csv file is dropped in an Azure Blob Store. Full article is here: 4 | 5 | [Automatic import of CSV data using Azure Functions and Azure SQL](https://medium.com/@mauridb/automatic-import-of-csv-data-using-azure-functions-and-azure-sql-63e1070963cf) 6 | 7 | Following there are the instructions you need to follow in order to setup the solution. [Azure CLI 2.0](https://docs.microsoft.com/en-us/cli/azure/get-started-with-azure-cli) commands are also provided as a reference 8 | 9 | ## Authenticate Azure CLI 2.0 10 | 11 | Login with your account 12 | 13 | az login 14 | 15 | and then select the Subscription you want to use 16 | 17 | az account set --subscription 18 | 19 | ## Create a Resource Group 20 | 21 | All resources of this example will be created in a dedicated Resource Group, named "CSVImportDemo" 22 | 23 | az group create --name CSVImportDemo --location westus 24 | 25 | ## Create Blob Storage 26 | 27 | Create the azure storage account used by Azure Function and to drop in our CSV files to be automatically imported 28 | 29 | az storage account create --name csvimportdemo --location westus --resource-group CSVImportDemo --sku Standard_LRS 30 | 31 | Once this is done get the account key and create a container named 'csv': 32 | 33 | az storage container create --account-name csvimportdemo --account-key --name csv 34 | 35 | Generate also Shared Access Signature (SAS) key token and store it for later use. The easies way to do this is to use [Azure Storage Explorer](https://azure.microsoft.com/en-us/features/storage-explorer/). 36 | 37 | Otherwise you can do it via AZ CLI using the `az storage account generate-sas` command. 38 | 39 | ## Create and Deploy the function app 40 | 41 | The easiest way to install, build and deploy the sample Function App, is to use [Visual Studio Code](https://code.visualstudio.com/). It will automatically detect that the `.csproj` is related to a FUnction App, will download the Function App runtime and also recommend you to download the Azure Function extension. 42 | 43 | Once the project is loaded, add the `AzureSQL` configuration to your `local.settings.json` file: 44 | 45 | "AzureSQL": "" 46 | 47 | Also make sure that the Function App is correctly monitoring the Azure Storage Account where you plan to drop you CSV files. If you used Visual Studio code, this should have already been set up for you. If not make sure you have the 'AzureStorage' configuration element in your `local.settings.json` file and that it has the connection string for the Azure Blob Storage account you want to use: 48 | 49 | "AzureStorage": "DefaultEndpointsProtocol=https;AccountName=csvimportdemo;AccountKey=[account-key-here];EndpointSuffix=core.windows.net" 50 | 51 | ## Create Azure SQL Server and Database 52 | 53 | Create an Azure SQL Server: 54 | 55 | az sql server create --name csvimportdemo --resource-group CSVImportDemo --location westus --admin-user csvimportdemo --admin-password csvimportdemoPassw0rd! 56 | 57 | Via the Azure Portal make sure that the firewall is configure to "Allow access to Azure Services". 58 | 59 | Also a small Azure SQL Database: 60 | 61 | az sql db create --name CSVImportDemo --resource-group CSVImportDemo --server csvimportdemo 62 | 63 | ## Upload format file 64 | 65 | Behind the scenes, the solution uses the T-SQL `BULK INSERT` command to import data read from a .csv file. In order to work the command needs a format file named `csv.fmt` in `_auto` folder the `csv` container. 66 | 67 | az storage blob upload --container-name csv --file SQL\csv.fmt --name _auto\csv.fmt --account-name csvimportdemo --account-key 68 | 69 | As mentioned before, the easiest way is to use [Azure Storage Explorer](https://azure.microsoft.com/en-us/features/storage-explorer/). Make sure you create the `_auto` folder manually if you use Azure Storage Explorer. 70 | 71 | ## Configure Bulk Load Security 72 | 73 | Connect to the created Azure SQL database and execute the script to configure access to blob store for Azure SQL. The script is available here 74 | 75 | `sql/enable-bulk-load.sql` 76 | 77 | just customize it with your own info before running it. 78 | 79 | Please not that when you specific the SAS key token, you have to remove the initial question mark (?) that is automatically added 80 | when you create the SAS key token online. 81 | 82 | ## Create Database Objects 83 | 84 | In the Azure SQL database couple of tables and a stored procedures needs also to be created in order to have the sample working correctly. 85 | 86 | Scripts to create the mentioned objects are available in 87 | 88 | `sql/create-objects.sql` 89 | 90 | Just execute it against the Azure SQL database. 91 | 92 | ## Deploy and Run the Function App 93 | 94 | You can now run the Function app on your machine, or you can deploy using Visual Studio Code and its Azure Function extension. Or you can use `az functionapp` to deploy the function manually. 95 | 96 | ## Test the solution 97 | 98 | All the CSV files that will be copied into the `csv` container, `_auto` folder, will be loaded into Azure SQL and specifically into the following tables: 99 | 100 | - File 101 | - FileData 102 | 103 | To test the everything works copy the `test.csv` file to Azure: 104 | 105 | az storage blob upload --container-name csv --file SQL\test.csv --name _auto\test.csv --account-name csvimportdemo --account-key 106 | 107 | or, if you prefer, just use Azure Storage Explorer, and drag on drop the file into the `_auto` folder. If you open Function App Log you will see that the function has been invoked or, if you're running the function locally on your machine, you will see the log directly on the console. You will find the content of the `test.csv` file into Azure SQL. 108 | -------------------------------------------------------------------------------- /SQL/create-app-user.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Create User used by Azure Functions 3 | */ 4 | CREATE USER [demo] WITH PASSWORD = 'AVery?STRONGPa22W0rd!'; 5 | ALTER ROLE [db_owner] ADD MEMBER [demo] 6 | GO 7 | -------------------------------------------------------------------------------- /SQL/create-objects.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS dbo.[FileData] 2 | GO 3 | 4 | IF EXISTS(SELECT * FROM sys.tables WHERE [object_id] = OBJECT_ID('dbo.[File]') AND temporal_type = 2) BEGIN 5 | ALTER TABLE dbo.[File] SET (SYSTEM_VERSIONING = OFF) 6 | END 7 | GO 8 | 9 | DROP TABLE IF EXISTS dbo.[FileHistory] 10 | GO 11 | 12 | DROP TABLE IF EXISTS dbo.[File] 13 | GO 14 | 15 | CREATE TABLE dbo.[File] 16 | ( 17 | Id INT IDENTITY NOT NULL PRIMARY KEY, 18 | [Name] NVARCHAR(128) UNIQUE NOT NULL, 19 | [ValidFrom] DATETIME2 GENERATED ALWAYS AS ROW START NOT NULL, 20 | [ValidTo] DATETIME2 GENERATED ALWAYS AS ROW END NOT NULL, 21 | PERIOD FOR SYSTEM_TIME ([ValidFrom], [ValidTo]) 22 | ) WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE = dbo.[FileHistory])) 23 | GO 24 | 25 | CREATE TABLE dbo.[FileData] 26 | ( 27 | FileId INT NOT NULL REFERENCES dbo.[File](Id), 28 | [FirstName] NVARCHAR(100) NOT NULL, 29 | [LastName] NVARCHAR(100) NOT NULL, 30 | [TwitterHandle] NVARCHAR(100) NOT NULL 31 | ) 32 | GO 33 | 34 | CREATE OR ALTER PROCEDURE dbo.SetFileMetadata 35 | @fileName NVARCHAR(128) 36 | AS 37 | SET XACT_ABORT ON 38 | BEGIN TRAN 39 | DELETE d FROM dbo.[FileData] d INNER JOIN dbo.[File] f ON d.FileId = f.Id WHERE f.[Name] = @fileName; 40 | DELETE d FROM dbo.[File] d WHERE d.Name = @fileName; 41 | 42 | INSERT INTO dbo.[File](Name) OUTPUT inserted.Id VALUES (@fileName); 43 | COMMIT TRAN 44 | GO 45 | 46 | CREATE OR ALTER PROCEDURE [dbo].[BulkLoadFromAzure] 47 | @sourceFileName NVARCHAR(100) 48 | AS 49 | DECLARE @fid INT; 50 | DECLARE @fileName NVARCHAR(MAX) = REPLACE(@sourceFileName, '.csv', ''); 51 | DECLARE @bulkFile NVARCHAR(MAX) = 'csv/' + @sourceFileName; 52 | 53 | DROP TABLE IF EXISTS #Result; 54 | 55 | CREATE TABLE #Result(FileID INT); 56 | 57 | INSERT INTO #Result EXEC [dbo].[SetFileMetadata] @fileName; 58 | 59 | SELECT TOP 1 @fid = FileID FROM #Result; 60 | 61 | DECLARE @sql NVARCHAR(MAX); 62 | SET @sql = N' 63 | INSERT INTO dbo.FileData 64 | ( 65 | FileId, 66 | FirstName, 67 | LastName, 68 | TwitterHandle 69 | ) 70 | SELECT 71 | FileId = ' + CAST(@fid AS NVARCHAR(9)) + ', 72 | FirstName, 73 | LastName, 74 | TwitterHandle 75 | FROM OPENROWSET( 76 | BULK ''' + @bulkFile + ''', 77 | DATA_SOURCE = ''Azure-Storage'', 78 | FIRSTROW=2, 79 | FORMATFILE=''csv/csv.fmt'', 80 | FORMATFILE_DATA_SOURCE = ''Azure-Storage'') as t 81 | '; 82 | --PRINT @sql; 83 | EXEC(@sql); 84 | GO 85 | 86 | -------------------------------------------------------------------------------- /SQL/enable-bulk-load.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Create a database Master Key to store credentials 3 | */ 4 | CREATE MASTER KEY ENCRYPTION BY PASSWORD = 'A-$tr0ng|PaSSw0Rd!'; 5 | GO 6 | 7 | /* 8 | Create a crendential to store SAS key. 9 | To get the SAS key either use Azure Storage Explorer 10 | or AZ CLI: 11 | az storage account generate-sas --account-name --permissions rl --services b --resource-types sco --expiry 2020-01-01 12 | 13 | */ 14 | CREATE DATABASE SCOPED CREDENTIAL [Storage-Credentials] 15 | WITH IDENTITY = 'SHARED ACCESS SIGNATURE', 16 | SECRET = ''; -- without leading "?" 17 | GO 18 | 19 | SELECT * FROM sys.[database_scoped_credentials] 20 | GO 21 | 22 | /* 23 | Create external data source 24 | */ 25 | CREATE EXTERNAL DATA SOURCE [Azure-Storage] 26 | WITH 27 | ( 28 | TYPE = BLOB_STORAGE, 29 | LOCATION = 'https://.blob.core.windows.net', 30 | CREDENTIAL= [Storage-Credentials] 31 | ); 32 | 33 | SELECT * FROM sys.[external_data_sources] 34 | GO 35 | 36 | -------------------------------------------------------------------------------- /csv/csv.fmt: -------------------------------------------------------------------------------- 1 | 13.0 2 | 3 3 | 4 | 1 SQLCHAR 0 0 , 1 FirstName "" 5 | 2 SQLCHAR 0 0 , 2 LastName "" 6 | 3 SQLCHAR 0 0 "\r\n" 3 TwitterHandle "" 7 | 8 | -------------------------------------------------------------------------------- /csv/test.csv: -------------------------------------------------------------------------------- 1 | #Sample Data 2 | FirstName,LastName,TwitterHandle 3 | Davide,Mauri,@maurid 4 | 5 | -------------------------------------------------------------------------------- /function/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-azuretools.vscode-azurefunctions", 4 | "ms-vscode.csharp" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /function/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Attach to .NET Functions", 6 | "type": "coreclr", 7 | "request": "attach", 8 | "processId": "${command:azureFunctions.pickProcess}" 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /function/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "azureFunctions.deploySubpath": "bin/Release/netcoreapp2.1/publish", 3 | "azureFunctions.projectLanguage": "C#", 4 | "azureFunctions.projectRuntime": "~2", 5 | "debug.internalConsoleOptions": "neverOpen", 6 | "azureFunctions.preDeployTask": "publish" 7 | } -------------------------------------------------------------------------------- /function/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "clean", 6 | "command": "dotnet", 7 | "args": [ 8 | "clean", 9 | "/property:GenerateFullPaths=true", 10 | "/consoleloggerparameters:NoSummary" 11 | ], 12 | "type": "process", 13 | "problemMatcher": "$msCompile" 14 | }, 15 | { 16 | "label": "build", 17 | "command": "dotnet", 18 | "args": [ 19 | "build", 20 | "/property:GenerateFullPaths=true", 21 | "/consoleloggerparameters:NoSummary" 22 | ], 23 | "type": "process", 24 | "dependsOn": "clean", 25 | "group": { 26 | "kind": "build", 27 | "isDefault": true 28 | }, 29 | "problemMatcher": "$msCompile" 30 | }, 31 | { 32 | "label": "clean release", 33 | "command": "dotnet", 34 | "args": [ 35 | "clean", 36 | "--configuration", 37 | "Release", 38 | "/property:GenerateFullPaths=true", 39 | "/consoleloggerparameters:NoSummary" 40 | ], 41 | "type": "process", 42 | "problemMatcher": "$msCompile" 43 | }, 44 | { 45 | "label": "publish", 46 | "command": "dotnet", 47 | "args": [ 48 | "publish", 49 | "--configuration", 50 | "Release", 51 | "/property:GenerateFullPaths=true", 52 | "/consoleloggerparameters:NoSummary" 53 | ], 54 | "type": "process", 55 | "dependsOn": "clean release", 56 | "problemMatcher": "$msCompile" 57 | }, 58 | { 59 | "type": "func", 60 | "dependsOn": "build", 61 | "options": { 62 | "cwd": "${workspaceFolder}/bin/Debug/netcoreapp2.1" 63 | }, 64 | "command": "host start", 65 | "isBackground": true, 66 | "problemMatcher": "$func-watch" 67 | } 68 | ] 69 | } -------------------------------------------------------------------------------- /function/ImportCSV.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using Microsoft.Azure.WebJobs; 4 | using Microsoft.Azure.WebJobs.Host; 5 | using Microsoft.Extensions.Logging; 6 | using Microsoft.WindowsAzure.Storage.Blob; 7 | using System.Configuration; 8 | using System.Data; 9 | using Microsoft.Data.SqlClient; 10 | using Dapper; 11 | 12 | namespace Microsoft.AzureSQL.Samples 13 | { 14 | public static class ImportCSV 15 | { 16 | [FunctionName("ImportCSV")] 17 | public static void Run([BlobTrigger("csv/_auto/{name}", Connection = "AzureStorage")]Stream blob, string name, ILogger log) 18 | { 19 | log.LogInformation($"Blob trigger function processed blob: {name}, size: {blob.Length} bytes"); 20 | 21 | string azureSQLConnectionString = Environment.GetEnvironmentVariable("AzureSQL"); 22 | 23 | if (!name.EndsWith(".csv")) 24 | { 25 | log.LogInformation($"Blob '{name}' doesn't have the .csv extension. Skipping processing."); 26 | return; 27 | } 28 | 29 | log.LogInformation($"Blob '{name}' found. Uploading to Azure SQL"); 30 | 31 | SqlConnection conn = null; 32 | try 33 | { 34 | conn = new SqlConnection(azureSQLConnectionString); 35 | conn.Execute("EXEC dbo.BulkLoadFromAzure @sourceFileName", new { @sourceFileName = name }, commandTimeout: 180); 36 | log.LogInformation($"Blob '{name}' uploaded"); 37 | } 38 | catch (SqlException se) 39 | { 40 | log.LogError($"Exception Trapped: {se.Message}"); 41 | } 42 | finally 43 | { 44 | conn?.Close(); 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /function/functions-csv-to-sql.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | netcoreapp2.1 4 | v2 5 | Microsoft.AzureSQL.Samples 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | PreserveNewest 16 | 17 | 18 | PreserveNewest 19 | Never 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /function/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0" 3 | } -------------------------------------------------------------------------------- /function/run-local.txt: -------------------------------------------------------------------------------- 1 | # Install Azure Function 2.0 core tools 2 | https://docs.microsoft.com/en-us/azure/azure-functions/functions-run-local 3 | 4 | # Tell Azure Storage Emulator to initialize and use the existing SQL Server instance 5 | # (PowerShell) 6 | & 'C:\Program Files (x86)\Microsoft SDKs\Azure\Storage Emulator\AzureStorageEmulator.exe' init /server localhostt 7 | 8 | # Start Azure Storage Emulator 9 | # (PowerShell) 10 | & 'C:\Program Files (x86)\Microsoft SDKs\Azure\Storage Emulator\AzureStorageEmulator.exe' start 11 | --------------------------------------------------------------------------------