├── AzureDevOpsTemplates └── azure-pipelines-Single-Serverless-Pool.yml ├── LICENSE.md ├── README.md └── scripts ├── Script0001 - Create table.sql └── Script0002 - Alter table.sql /AzureDevOpsTemplates/azure-pipelines-Single-Serverless-Pool.yml: -------------------------------------------------------------------------------- 1 | # Sample of how to deploy to an Azure Synapse Serverless SQL Pool 2 | # You can use a self-hosted agent installed locally for this to work 3 | # You can also use a Microsoft-hosted agent for the Azure services 4 | 5 | # For this pipeline you also need to setup the below pipeline variables 6 | # sqlinstance - Your Serverless SQL Pool endpoint 7 | #(the one that ends with -ondemand) 8 | # database - The database in the SQL Pool you want the update deployed to 9 | # Note it has to exist BEFORE this the scripts are run 10 | # UserName - User name to connect to the Serverless SQL Pool endpoint 11 | # try and keep this secret 12 | # Pw - Password of above user, MUST be wrapped in single quotes for it to work. 13 | # For example, 'SecretPW123' 14 | # Definitely keep this one secret 15 | 16 | # Optional variable groups you can use 17 | # Uncomment the below to use them 18 | # variables: 19 | # - group: AzureSynapseServerlessKeyVault 20 | # - group: AzureSynapseServerlessNonSensitive 21 | 22 | trigger: 23 | - main 24 | 25 | pool: 26 | vmImage: 'windows-latest' 27 | 28 | stages: 29 | 30 | - stage: RunScripts 31 | displayName: 'Run Migration-based scripts' 32 | 33 | jobs: 34 | - job: 'RunScripts' 35 | displayName: 'Run Migration-based scripts' 36 | steps: 37 | - task: PowerShell@2 38 | displayName: 'Install dbops PoSh Module' 39 | inputs: 40 | targetType: 'inline' 41 | script: 'Install-Module -Name dbops -Force -PassThru' 42 | 43 | - task: PowerShell@2 44 | displayName: 'Run migration-based scripts' 45 | inputs: 46 | targetType: 'inline' 47 | script: | 48 | $SecurePw=ConvertTo-SecureString $(Pw) –asplaintext –force 49 | Install-DBOScript -ScriptPath scripts -sqlinstance $(sqlinstance) -Database $(Database) -UserName $(UserName) -Password $($SecurePw) -SchemaVersionTable $null 50 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Kevin Chant 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Azure Synapse Serverless SQL Pool Database Project 2 | 3 | Example of a Migration-Based deployment that deploys to an Azure Synapse serverless SQL Pool using Azure DevOps. Based on a blog post I wrote called '[CI/CD for serverless SQL pools using Azure DevOps](https://bit.ly/3FWCHQl)'. 4 | 5 | You can see this template in action in the [video for the November 2022 edition of the Azure Synapse Analytics and MVP series](https://www.youtube.com/watch?v=87gNrueVRFU). Click on the link to view. 6 | 7 | **Please note:** This solution was devised before dacpac support for serverless SQL Pools became available. You can find a template to perform CI/CD based on creating a dacpac from a database project in my '[AzureDevOps-SynpaseServerlessSQLPool-dacpac repository](https://github.com/kevchant/AzureDevOps-SynpaseServerlessSQLPool-dacpac)' 8 | 9 | A brief overview is below. However, there is also a [wiki for this repository](https://github.com/kevchant/AzureDevOps-SynapseServerlessSQLPool/wiki). 10 | 11 | It contains an example YAML file that you can use as a YAML pipeline in Azure Pipelines. You can find it in the AzureDevOpsTemplates folder. In order to use it in Azure Pipelines you can either import or fork this repository into another GitHub repository, or into [Azure Repos](https://bit.ly/3s4uO77). 12 | 13 | Afterwards, you can select the YAML file in Azure Pipelines and tailor the pipeline to suit your needs. You can find a guide on how to select the YAML file whilst setting up a YAML Pipeline this in a blog post I wrote called '[Connect a Database Project in Azure Repos to Azure Pipelines](https://bit.ly/3uF1Iv9)'. 14 | 15 | You can find the recommended variables inside the YAML file. Avoid putting sensitive information directly into the YAML file (like your connection details). One thing I must stress here is that the password MUST be wrapped in single quotes in the secret for it to work. 16 | 17 | You can use the logic in 'Classic Editor' instead by adding the tasks into the GUI and transferring the logic over. Alternatively, you can build an artifact for it using the 'Classic Editor' and use the 'Releases' feature for deployments. Personally, I prefer doing the deployment using a YAML pipeline. 18 | 19 | Please note that the databases must already exist in the serverless SQL Pool for this to work. 20 | 21 | In addition, you might want to create a file in Azure Data Lake storage that contains the headings used for the SchemaVersions table (https://dbup.readthedocs.io/en/latest/more-info/journaling/). From there, you can try using it as the SchemaVersions table in the code to log updates. 22 | 23 | This repository is provided "as is" based on the [MIT license](https://opensource.org/licenses/MIT). Basically, I am not responsible for your use of it. 24 | -------------------------------------------------------------------------------- /scripts/Script0001 - Create table.sql: -------------------------------------------------------------------------------- 1 | IF NOT EXISTS (SELECT * FROM sys.external_file_formats WHERE name = 'SynapseParquetFormat') 2 | CREATE EXTERNAL FILE FORMAT [SynapseParquetFormat] 3 | WITH ( FORMAT_TYPE = PARQUET) 4 | GO 5 | 6 | IF NOT EXISTS (SELECT * FROM sys.external_data_sources WHERE name = 'nyctlc_azureopendatastorage_blob_core_windows_net') 7 | CREATE EXTERNAL DATA SOURCE [nyctlc_azureopendatastorage_blob_core_windows_net] 8 | WITH ( 9 | LOCATION = 'https://azureopendatastorage.blob.core.windows.net/nyctlc', 10 | ) 11 | Go 12 | 13 | IF NOT EXISTS (SELECT * FROM sys.external_tables WHERE name = 'nyc_tlc_yellow_trip_ext') 14 | CREATE EXTERNAL TABLE nyc_tlc_yellow_trip_ext ( 15 | [vendorID] varchar(8000), 16 | [tpepPickupDateTime] datetime2(7), 17 | [tpepDropoffDateTime] datetime2(7), 18 | [passengerCount] int, 19 | [tripDistance] float, 20 | [puLocationId] varchar(8000), 21 | [doLocationId] varchar(8000), 22 | [startLon] float, 23 | [startLat] float, 24 | [endLon] float, 25 | [endLat] float, 26 | [rateCodeId] int, 27 | [storeAndFwdFlag] varchar(8000), 28 | [paymentType] varchar(8000), 29 | [fareAmount] float, 30 | [extra] float, 31 | [mtaTax] float, 32 | [improvementSurcharge] varchar(8000), 33 | [tipAmount] float, 34 | [tollsAmount] float, 35 | [totalAmount] float 36 | ) 37 | WITH ( 38 | LOCATION = 'yellow/puYear=2014/puMonth=3/*.parquet', 39 | -- LOCATION = 'yellow/puYear=*/puMonth=*/*.parquet' 40 | DATA_SOURCE = [nyctlc_azureopendatastorage_blob_core_windows_net], 41 | FILE_FORMAT = [SynapseParquetFormat] 42 | ) 43 | GO 44 | -------------------------------------------------------------------------------- /scripts/Script0002 - Alter table.sql: -------------------------------------------------------------------------------- 1 | IF EXISTS (SELECT * FROM sys.external_tables WHERE name = 'nyc_tlc_yellow_trip_ext') 2 | DROP EXTERNAL TABLE nyc_tlc_yellow_trip_ext 3 | GO 4 | 5 | CREATE EXTERNAL TABLE nyc_tlc_yellow_trip_ext ( 6 | [vendorID] varchar(8000), 7 | [tpepPickupDateTime] datetime2(7), 8 | [tpepDropoffDateTime] datetime2(7), 9 | [passengerCount] int, 10 | [tripDistance] float, 11 | [puLocationId] varchar(8000), 12 | [doLocationId] varchar(8000), 13 | [startLon] float, 14 | [startLat] float, 15 | [endLon] float, 16 | [endLat] float, 17 | [rateCodeId] int, 18 | [storeAndFwdFlag] varchar(8000), 19 | [paymentType] varchar(8000), 20 | [fareAmount] float, 21 | [extra] float, 22 | [mtaTax] float, 23 | [improvementSurcharge] varchar(8000), 24 | [tipAmount] float, 25 | [tollsAmount] float, 26 | [totalAmount] float 27 | ) 28 | WITH ( 29 | LOCATION = 'yellow/puYear=2014/puMonth=3/*.parquet', 30 | -- LOCATION = 'yellow/puYear=*/puMonth=*/*.parquet' 31 | DATA_SOURCE = [nyctlc_azureopendatastorage_blob_core_windows_net], 32 | FILE_FORMAT = [SynapseParquetFormat] 33 | ) 34 | GO 35 | --------------------------------------------------------------------------------