├── SqlPools ├── AzFunctions │ ├── azProdataSql │ │ ├── Suspend-AdventureWorksDW-Trigger │ │ │ ├── sample.dat │ │ │ ├── function.json │ │ │ ├── readme.md │ │ │ └── run.ps1 │ │ ├── .funcignore │ │ ├── HttpTrigger1 │ │ │ ├── sample.dat │ │ │ ├── function.json │ │ │ └── run.ps1 │ │ ├── Resume-AzSynapseSqlPool │ │ │ ├── sample.dat │ │ │ ├── function.json │ │ │ └── run.ps1 │ │ ├── Suspend-AzSynapseSqlPool │ │ │ ├── sample.dat │ │ │ ├── function.json │ │ │ └── run.ps1 │ │ ├── .vscode │ │ │ ├── extensions.json │ │ │ ├── settings.json │ │ │ ├── tasks.json │ │ │ └── launch.json │ │ ├── Suspend-AzSqlDatabase │ │ │ ├── sample.dat │ │ │ ├── function.json │ │ │ └── run.ps1 │ │ ├── .gitignore │ │ ├── requirements.psd1 │ │ ├── host.json │ │ └── profile.ps1 │ ├── .vscode │ │ ├── extensions.json │ │ ├── settings.json │ │ ├── tasks.json │ │ └── launch.json │ └── Test-azProdataSql │ │ └── Test-Suspend-AzSqlDatabase.ps1 ├── Demos │ ├── SyanpseTools.code-workspace │ ├── 01-00. CS Intro.ipynb │ ├── 01-04. Is it worth it.ipynb │ ├── 01.07. CS Productionisation.ipynb │ ├── 02.5 StatsOptimize.ipynb │ ├── 01-02. How do we fix Density.ipynb │ ├── 02.2 Locating Stale stats.ipynb │ ├── 04.0 Links and References.ipynb │ └── 01.06. CS-DeletedDelta.ipynb ├── Monitoring │ ├── sp_WhoWasActive.sql │ ├── QueryStore │ │ ├── 01. DMV Summary.sql │ │ ├── 02_QueryStore_Samples.sql │ │ └── QueryStore.sql │ ├── sp_WhoIsActive.sql │ ├── Requests.sql │ └── vTableSizes.sql └── Maintenance │ ├── BuildSolution.ps1 │ ├── CommandLog.sql │ ├── OverlappingStatistics.sql │ ├── vStats.sql │ ├── vPartitionStats.sql │ ├── vTableStats.sql │ ├── vColumnStoreStats.sql │ ├── ColumnStoreOptimize.sql │ ├── Readme.md │ └── StatsOptimize.sql └── README.md /SqlPools/AzFunctions/azProdataSql/Suspend-AdventureWorksDW-Trigger/sample.dat: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/.funcignore: -------------------------------------------------------------------------------- 1 | .git* 2 | .vscode 3 | local.settings.json 4 | test -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/HttpTrigger1/sample.dat: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Azure" 3 | } 4 | -------------------------------------------------------------------------------- /SqlPools/Demos/SyanpseTools.code-workspace: -------------------------------------------------------------------------------- 1 | { 2 | "folders": [ 3 | { 4 | "path": "." 5 | } 6 | ], 7 | "settings": {} 8 | } -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/Resume-AzSynapseSqlPool/sample.dat: -------------------------------------------------------------------------------- 1 | { 2 | "DatabaseName": "AdventureWorksDW", 3 | "ServerName": "aw-dev" 4 | 5 | } -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/Suspend-AzSynapseSqlPool/sample.dat: -------------------------------------------------------------------------------- 1 | { 2 | "DatabaseName": "AdventureWorksDW", 3 | "ServerName": "aw-dev" 4 | 5 | } -------------------------------------------------------------------------------- /SqlPools/AzFunctions/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-azuretools.vscode-azurefunctions", 4 | "ms-vscode.PowerShell" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-azuretools.vscode-azurefunctions", 4 | "ms-vscode.PowerShell" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/Suspend-AzSqlDatabase/sample.dat: -------------------------------------------------------------------------------- 1 | { 2 | "DatabaseName": "AdventureWorksDW", 3 | "ServerName": "aw-dev.sql.azuresynapse.net" 4 | 5 | 6 | } 7 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Azure Functions artifacts 3 | bin 4 | obj 5 | appsettings.json 6 | local.settings.json 7 | 8 | # Azurite artifacts 9 | __blobstorage__ 10 | __queuestorage__ 11 | __azurite_db*__.json -------------------------------------------------------------------------------- /SqlPools/AzFunctions/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "azureFunctions.deploySubpath": "azProdataSql", 3 | "azureFunctions.projectLanguage": "PowerShell", 4 | "azureFunctions.projectRuntime": "~4", 5 | "debug.internalConsoleOptions": "neverOpen" 6 | } -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "azureFunctions.deploySubpath": ".", 3 | "azureFunctions.projectLanguage": "PowerShell", 4 | "azureFunctions.projectRuntime": "~4", 5 | "debug.internalConsoleOptions": "neverOpen" 6 | } -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "type": "func", 6 | "command": "host start", 7 | "problemMatcher": "$func-powershell-watch", 8 | "isBackground": true 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/Suspend-AdventureWorksDW-Trigger/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "name": "Timer", 5 | "type": "timerTrigger", 6 | "direction": "in", 7 | "schedule": "0 0 19 * * *" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/Test-azProdataSql/Test-Suspend-AzSqlDatabase.ps1: -------------------------------------------------------------------------------- 1 | using namespace System.Net 2 | 3 | 4 | 5 | $Request ="{DatabaseName:SwatDW,ServerName:swat-dev.database.windows.net,ResourceGroupName:swat-dev-rg}" 6 | 7 | .\azProdataSql\Suspend-AzSqlDatabase\run.ps1 $Request 8 | 9 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "type": "func", 6 | "command": "host start", 7 | "problemMatcher": "$func-powershell-watch", 8 | "isBackground": true, 9 | "options": { 10 | "cwd": "${workspaceFolder}/azProdataSql" 11 | } 12 | } 13 | ] 14 | } -------------------------------------------------------------------------------- /SqlPools/AzFunctions/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Attach to PowerShell Functions", 6 | "type": "PowerShell", 7 | "request": "attach", 8 | "customPipeName": "AzureFunctionsPSWorker", 9 | "runspaceId": 1, 10 | "preLaunchTask": "func: host start" 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/HttpTrigger1/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "authLevel": "function", 5 | "type": "httpTrigger", 6 | "direction": "in", 7 | "name": "Request", 8 | "methods": [ 9 | "get", 10 | "post" 11 | ] 12 | }, 13 | { 14 | "type": "http", 15 | "direction": "out", 16 | "name": "Response" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Attach to PowerShell Functions", 6 | "type": "PowerShell", 7 | "request": "attach", 8 | "customPipeName": "AzureFunctionsPSWorker", 9 | "runspaceId": 1, 10 | "preLaunchTask": "func: host start" 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/Resume-AzSynapseSqlPool/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "authLevel": "function", 5 | "type": "httpTrigger", 6 | "direction": "in", 7 | "name": "Request", 8 | "methods": [ 9 | "get", 10 | "post" 11 | ] 12 | }, 13 | { 14 | "type": "http", 15 | "direction": "out", 16 | "name": "Response" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/Suspend-AzSqlDatabase/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "authLevel": "function", 5 | "type": "httpTrigger", 6 | "direction": "in", 7 | "name": "Request", 8 | "methods": [ 9 | "get", 10 | "post" 11 | ] 12 | }, 13 | { 14 | "type": "http", 15 | "direction": "out", 16 | "name": "Response" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/Suspend-AzSynapseSqlPool/function.json: -------------------------------------------------------------------------------- 1 | { 2 | "bindings": [ 3 | { 4 | "authLevel": "function", 5 | "type": "httpTrigger", 6 | "direction": "in", 7 | "name": "Request", 8 | "methods": [ 9 | "get", 10 | "post" 11 | ] 12 | }, 13 | { 14 | "type": "http", 15 | "direction": "out", 16 | "name": "Response" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/requirements.psd1: -------------------------------------------------------------------------------- 1 | # This file enables modules to be automatically managed by the Functions service. 2 | # See https://aka.ms/functionsmanageddependency for additional information. 3 | # 4 | @{ 5 | # For latest supported version, go to 'https://www.powershellgallery.com/packages/Az'. 6 | # To use the Az module in your function app, please uncomment the line below. 7 | # 'Az' = '6.*' 8 | 'Az.Sql' = "3.5" 9 | 'Az.Synapse' ="0.18" 10 | } -------------------------------------------------------------------------------- /SqlPools/Monitoring/sp_WhoWasActive.sql: -------------------------------------------------------------------------------- 1 | /****** Object: StoredProcedure [dbo].[sp_WhoWasActive] Script Date: 08/05/2021 18:10:25 ******/ 2 | SET ANSI_NULLS ON 3 | GO 4 | 5 | SET QUOTED_IDENTIFIER ON 6 | GO 7 | 8 | CREATE PROC [dbo].[sp_WhoWasActive] AS 9 | /* 10 | Desc: Return Running Statements 11 | History: 29/03/2021 Bob, Created 12 | */ 13 | BEGIN 14 | SET NOCOUNT ON; 15 | 16 | SELECT TOP 500 * FROM Requests 17 | ORDER BY submit_time desc 18 | 19 | END 20 | GO 21 | 22 | 23 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "logging": { 4 | "applicationInsights": { 5 | "samplingSettings": { 6 | "isEnabled": true, 7 | "excludedTypes": "Request" 8 | } 9 | } 10 | }, 11 | "extensionBundle": { 12 | "id": "Microsoft.Azure.Functions.ExtensionBundle", 13 | "version": "[2.*, 3.0.0)" 14 | }, 15 | "managedDependency": { 16 | "enabled": true 17 | }, 18 | "functionTimeout": "00:10:00" 19 | , 20 | "Subscription": "6fcdeaed-9120-4fb3-979b-fd3f522c5b9b" 21 | } 22 | -------------------------------------------------------------------------------- /SqlPools/Maintenance/BuildSolution.ps1: -------------------------------------------------------------------------------- 1 | # Build Syanpsse MaintenanceSolution 2 | # https://github.com/ProdataSQL/SynapseTools 3 | # Compile Individual TSQL Files into one single file 4 | # 5 | $Solution ="MaintenanceSolution.sql" 6 | 7 | Set-Location $PSScriptRoot 8 | Copy-Item -Path CommandLog.sql -Destination $Solution 9 | Get-Content -Path vTableStats.sql | Add-Content -Path $Solution 10 | Get-Content -Path vPartitionStats.sql | Add-Content -Path $Solution 11 | Get-Content -Path vStats.sql | Add-Content -Path $Solution 12 | Get-Content -Path vColumnStoreStats.sql | Add-Content -Path $Solution 13 | Get-Content -Path *Optimize.sql | Add-Content -Path $Solution 14 | 15 | Write-Host "Done" -------------------------------------------------------------------------------- /SqlPools/Monitoring/QueryStore/01. DMV Summary.sql: -------------------------------------------------------------------------------- 1 | /* Query Store DMV SUmmary for Syanpse SQL Pool */ 2 | 3 | 4 | /* Core Query Store Data. Notice that CPU and memory not stored */ 5 | SELECT TOP 10 * 6 | FROM [sys].[query_store_query]; 7 | GO 8 | 9 | 10 | /* Query Text. Notice that SQL is tokenised to remove literals */ 11 | SELECT TOP 10 * 12 | FROM [sys].[query_store_query_text]; 13 | GO 14 | 15 | SELECT TOP 10 * 16 | FROM [sys].[query_store_plan]; 17 | GO 18 | 19 | 20 | /* run time stats. Notice that a lot of the metrics do not get shown in SQL Pools */ 21 | SELECT TOP 10 * 22 | FROM [sys].[query_store_runtime_stats]; 23 | GO 24 | 25 | /* Note. There are no wait stats like traditional SQL*/ -------------------------------------------------------------------------------- /SqlPools/Monitoring/sp_WhoIsActive.sql: -------------------------------------------------------------------------------- 1 | /****** Object: StoredProcedure [dbo].[sp_WhoIsActive] Script Date: 08/05/2021 18:10:05 ******/ 2 | SET ANSI_NULLS ON 3 | GO 4 | 5 | SET QUOTED_IDENTIFIER ON 6 | GO 7 | 8 | CREATE PROC [dbo].[sp_WhoIsActive] AS 9 | /* 10 | Desc: Return Running Statements 11 | History: 29/03/2021 Bob, Created 12 | */ 13 | BEGIN 14 | SET NOCOUNT ON 15 | DECLARE @distribution_id int, @spid int 16 | 17 | SELECT * FROM dbo.Requests where status='running' 18 | 19 | SELECT TOP 1 @distribution_id= distribution_id , @spid=spid FROM sys.dm_pdw_sql_requests where status='running' 20 | IF @distribution_id is not null 21 | BEGIN 22 | DBCC PDW_SHOWEXECUTIONPLAN (@distribution_id, @spid); 23 | END 24 | 25 | select* from sys.dm_pdw_sql_requests where status='running' 26 | END 27 | GO 28 | 29 | 30 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/Suspend-AdventureWorksDW-Trigger/readme.md: -------------------------------------------------------------------------------- 1 | # TimerTrigger - PowerShell 2 | 3 | The `TimerTrigger` makes it incredibly easy to have your functions executed on a schedule. This sample demonstrates a simple use case of calling your function every 5 minutes. 4 | 5 | ## How it works 6 | 7 | For a `TimerTrigger` to work, you provide a schedule in the form of a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression)(See the link for full details). A cron expression is a string with 6 separate expressions which represent a given schedule via patterns. The pattern we use to represent every 5 minutes is `0 */5 * * * *`. This, in plain text, means: "When seconds is equal to 0, minutes is divisible by 5, for any hour, day of the month, month, day of the week, or year". 8 | 9 | ## Learn more 10 | 11 | Documentation 12 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/HttpTrigger1/run.ps1: -------------------------------------------------------------------------------- 1 | using namespace System.Net 2 | 3 | # Input bindings are passed in via param block. 4 | param($Request, $TriggerMetadata) 5 | 6 | # Write to the Azure Functions log stream. 7 | Write-Host "PowerShell HTTP trigger function processed a request." 8 | 9 | # Interact with query parameters or the body of the request. 10 | $name = $Request.Query.Name 11 | if (-not $name) { 12 | $name = $Request.Body.Name 13 | } 14 | 15 | $body = "This HTTP triggered function executed successfully. Pass a name in the query string or in the request body for a personalized response." 16 | 17 | if ($name) { 18 | $body = "Hello, $name. This HTTP triggered function executed successfully." 19 | } 20 | 21 | # Associate values to output bindings by calling 'Push-OutputBinding'. 22 | Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ 23 | StatusCode = [HttpStatusCode]::OK 24 | Body = $body 25 | }) 26 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/profile.ps1: -------------------------------------------------------------------------------- 1 | # Azure Functions profile.ps1 2 | # 3 | # This profile.ps1 will get executed every "cold start" of your Function App. 4 | # "cold start" occurs when: 5 | # 6 | # * A Function App starts up for the very first time 7 | # * A Function App starts up after being de-allocated due to inactivity 8 | # 9 | # You can define helper functions, run commands, or specify environment variables 10 | # NOTE: any variables defined that are not environment variables will get reset after the first execution 11 | 12 | # Authenticate with Azure PowerShell using MSI. 13 | # Remove this if you are not planning on using MSI or Azure PowerShell. 14 | if ($env:MSI_SECRET) { 15 | Disable-AzContextAutosave -Scope Process | Out-Null 16 | Connect-AzAccount -Identity 17 | } 18 | Set-AzContext -Subscription $Env:Subscription 19 | # Uncomment the next line to enable legacy AzureRm alias in Azure PowerShell. 20 | # Enable-AzureRmAlias 21 | 22 | # You can also define functions or aliases that can be referenced in any of your PowerShell functions. 23 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/Suspend-AdventureWorksDW-Trigger/run.ps1: -------------------------------------------------------------------------------- 1 | # Input bindings are passed in via param block. 2 | param($Timer) 3 | 4 | # Get the current universal time in the default string format 5 | $currentUTCtime = (Get-Date).ToUniversalTime() 6 | 7 | # The 'IsPastDue' porperty is 'true' when the current function invocation is later than scheduled. 8 | if ($Timer.IsPastDue) { 9 | Write-Host "PowerShell timer is running late!" 10 | } 11 | 12 | # Interact with query parameters or the body of the request. 13 | $ServerName = $Request.Body.ServerName 14 | $DatabaseName = $Request.Body.DatabaseName 15 | 16 | If (!$ServerName ) {$ServerName =$env:ServerName } 17 | If (!$DatabaseName ) {$DatabaseName =$env:DatabaseName } 18 | 19 | Write-Host "ServerName =" + $ServerName 20 | Write-Host "DatabaseName =" + $DatabaseName 21 | 22 | Suspend-AzSynapseSqlPool -WorkspaceName $ServerName -Name $DatabaseName 23 | 24 | 25 | # Write an information log with the current time. 26 | Write-Host "PowerShell timer trigger function ran! TIME: $currentUTCtime" 27 | -------------------------------------------------------------------------------- /SqlPools/Maintenance/CommandLog.sql: -------------------------------------------------------------------------------- 1 | SET ANSI_NULLS ON 2 | GO 3 | SET QUOTED_IDENTIFIER ON 4 | GO 5 | IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[CommandLog]') AND type in (N'U')) 6 | CREATE TABLE [dbo].[CommandLog] 7 | ( 8 | [ID] [int] IDENTITY(1,1) NOT NULL, 9 | [DatabaseName] [sysname] NULL, 10 | [SchemaName] [sysname] NULL, 11 | [ObjectName] [sysname] NULL, 12 | [ObjectType] [char](2) NULL, 13 | [IndexName] [sysname] NULL, 14 | [IndexType] [tinyint] NULL, 15 | [StatisticsName] [sysname] NULL, 16 | [PartitionNumber] [int] NULL, 17 | [ExtendedInfo] [nvarchar](max) NULL, 18 | [Command] [nvarchar](max) NOT NULL, 19 | [CommandType] [nvarchar](60) NOT NULL, 20 | [StartTime] [datetime] NOT NULL, 21 | [EndTime] [datetime] NULL, 22 | [ErrorNumber] [int] NULL, 23 | [ErrorMessage] [nvarchar](max) NULL, 24 | CONSTRAINT [PK_CommandLog] PRIMARY KEY NONCLUSTERED 25 | ( 26 | [ID] ASC 27 | ) NOT ENFORCED 28 | ) 29 | WITH 30 | ( 31 | DISTRIBUTION = ROUND_ROBIN, 32 | CLUSTERED INDEX 33 | ( 34 | [ID] ASC 35 | ) 36 | ) 37 | GO 38 | 39 | 40 | -------------------------------------------------------------------------------- /SqlPools/Monitoring/QueryStore/02_QueryStore_Samples.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | /* Slowest Queries in last Week */ 4 | SELECT TOP 10 * 5 | FROM QueryStore 6 | WHERE [last_execution_time] > DATEADD(day, -7, GETUTCDATE()) 7 | ORDER BY avg_Duration desc 8 | 9 | /* Last 10 Queries Ran */ 10 | SELECT TOP 10 * 11 | FROM QueryStore 12 | ORDER BY last_execution_time desc 13 | 14 | 15 | /* Queries with > 1 Plan Executed in last week */ 16 | SELECT query_id 17 | , min(query_text) as query_text 18 | , count(*) as Plans 19 | , sum(count_executions) as executions 20 | , sum(avg_duration * count_executions) / sum(count_executions) as avg_duration 21 | , min(min_duration) as min_duration 22 | , max(max_duration) as max_duration , max(last_execution_time) as last_execution_time 23 | FROM QueryStore 24 | WHERE [last_execution_time] > DATEADD(day, -7, GETUTCDATE()) 25 | GROUP BY query_id 26 | HAVING COUNT(*) > 1 27 | order by count(*) desc 28 | 29 | /* Show all the Plans for the query with most plans*/ 30 | SELECT TOP 10 * from QueryStore 31 | where query_id=1949 32 | 33 | 34 | /* Find all calls from a Stored Procedure */ 35 | SELECT * FROM dbo.QueryStore 36 | WHERE [object_id] = OBJECT_ID(N'Sales.usp_MyProc'); 37 | 38 | 39 | -------------------------------------------------------------------------------- /SqlPools/Monitoring/QueryStore/QueryStore.sql: -------------------------------------------------------------------------------- 1 | 2 | SET ANSI_NULLS ON 3 | GO 4 | SET QUOTED_IDENTIFIER ON 5 | GO 6 | IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[QueryStore]') AND type in (N'V')) 7 | EXEC dbo.sp_executesql @statement = N'CREATE VIEW [dbo].[QueryStore] AS SELECT 1 as Dummy' 8 | GO 9 | ALTER VIEW [dbo].[QueryStore] 10 | AS SELECT Txt.query_text_id as query_id, Txt.query_sql_text as query_text, sp.plan_id, sp.query_plan as query_plan_xml, Qry.count_compiles, qry.last_execution_time 11 | , rs.count_executions 12 | , rs.avg_duration 13 | , rs.min_duration 14 | , rs.max_duration 15 | , Qry.object_id 16 | FROM sys.query_store_plan AS sp 17 | INNER JOIN sys.query_store_query AS Qry 18 | ON sp.query_id = Qry.query_id 19 | INNER JOIN sys.query_store_query_text AS Txt 20 | ON Qry.query_text_id = Txt.query_text_id 21 | INNER JOIN ( 22 | select plan_id, sum(count_executions) as count_executions 23 | , sum(rs.avg_duration * rs. count_executions) / sum(rs.count_executions)/1000000 as avg_duration 24 | , min(convert(numeric (14,3),rs.min_duration )) /1000000 as min_duration 25 | , max(convert(numeric (14,3),rs.max_duration )) /1000000 as max_duration 26 | from sys.query_store_runtime_stats rs 27 | GROUP BY rs.plan_id 28 | ) rs on rs.plan_id=sp.plan_id 29 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/Resume-AzSynapseSqlPool/run.ps1: -------------------------------------------------------------------------------- 1 | using namespace System.Net 2 | 3 | # Input bindings are passed in via param block. 4 | param($Request, $TriggerMetadata) 5 | 6 | try { 7 | 8 | # Write to the Azure Functions log stream. 9 | Write-Host "PowerShell HTTP trigger Started." 10 | 11 | # Interact with query parameters or the body of the request. 12 | $ServerName = $Request.Body.ServerName 13 | $DatabaseName = $Request.Body.DatabaseName 14 | 15 | If (!$ServerName ) {$ServerName =$env:ServerName } 16 | If (!$DatabaseName ) {$DatabaseName =$env:DatabaseName } 17 | 18 | Write-Host "ServerName =" + $ServerName 19 | Write-Host "DatabaseName =" + $DatabaseName 20 | 21 | $body= Resume-AzSynapseSqlPool -WorkspaceName $ServerName -Name $DatabaseName 22 | 23 | # Associate values to output bindings by calling 'Push-OutputBinding'. 24 | Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ 25 | StatusCode = [HttpStatusCode]::OK 26 | Body = $body 27 | }) 28 | 29 | } 30 | catch { 31 | Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ 32 | StatusCode = [HttpStatusCode]::ExpectationFailed 33 | Body = $error[0].ToString() + $error[0].InvocationInfo.PositionMessage 34 | }) 35 | 36 | } 37 | 38 | 39 | -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/Suspend-AzSynapseSqlPool/run.ps1: -------------------------------------------------------------------------------- 1 | using namespace System.Net 2 | 3 | # Input bindings are passed in via param block. 4 | param($Request, $TriggerMetadata) 5 | 6 | try { 7 | 8 | # Write to the Azure Functions log stream. 9 | Write-Host "PowerShell HTTP trigger Started." 10 | 11 | # Interact with query parameters or the body of the request. 12 | $ServerName = $Request.Body.ServerName 13 | $DatabaseName = $Request.Body.DatabaseName 14 | 15 | If (!$ServerName ) {$ServerName =$env:ServerName } 16 | If (!$DatabaseName ) {$DatabaseName =$env:DatabaseName } 17 | 18 | Write-Host "ServerName =" + $ServerName 19 | Write-Host "DatabaseName =" + $DatabaseName 20 | 21 | $body= Suspend-AzSynapseSqlPool -WorkspaceName $ServerName -Name $DatabaseName 22 | 23 | # Associate values to output bindings by calling 'Push-OutputBinding'. 24 | Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ 25 | StatusCode = [HttpStatusCode]::OK 26 | Body = $body 27 | }) 28 | 29 | } 30 | catch { 31 | Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ 32 | StatusCode = [HttpStatusCode]::ExpectationFailed 33 | Body = $error[0].ToString() + $error[0].InvocationInfo.PositionMessage 34 | }) 35 | 36 | } 37 | 38 | 39 | -------------------------------------------------------------------------------- /SqlPools/Maintenance/OverlappingStatistics.sql: -------------------------------------------------------------------------------- 1 | WITH autostats ( 2 | object_id 3 | ,stats_id 4 | ,name 5 | ,column_id 6 | ) 7 | AS ( 8 | SELECT sys.stats.object_id 9 | ,sys.stats.stats_id 10 | ,sys.stats.name 11 | ,sc.column_id 12 | FROM sys.stats 13 | INNER JOIN sys.stats_columns sc ON sys.stats.object_id = sc.object_id 14 | AND sys.stats.stats_id = sc.stats_id 15 | WHERE sys.stats.auto_created = 1 16 | AND sc.stats_column_id = 1 17 | ) 18 | SELECT t.object_id 19 | ,s.name AS schema_name 20 | ,t.name AS table_name 21 | ,'' + c.name + '' + ss.name + '' + '' 22 | ,'DROP STATISTICS [' + OBJECT_SCHEMA_NAME(ss.object_id) + '].[' + OBJECT_NAME(ss.object_id) + '].[' + autostats.name + ']' AS SqlCommand 23 | FROM sys.stats ss 24 | INNER JOIN sys.stats_columns sc ON ss.object_id = sc.object_id 25 | AND ss.stats_id = sc.stats_id 26 | INNER JOIN autostats ON sc.object_id = autostats.object_id 27 | AND sc.column_id = autostats.column_id 28 | INNER JOIN sys.columns c ON ss.object_id = c.object_id 29 | AND sc.column_id = c.column_id 30 | INNER JOIN sys.tables t ON t.object_id = ss.object_id 31 | INNER JOIN sys.schemas s ON s.schema_id = t.schema_id 32 | WHERE ss.auto_created = 0 33 | AND sc.stats_column_id = 1 34 | AND sc.stats_id != autostats.stats_id 35 | AND OBJECTPROPERTY(ss.object_id, 'IsMsShipped') = 0 -------------------------------------------------------------------------------- /SqlPools/AzFunctions/azProdataSql/Suspend-AzSqlDatabase/run.ps1: -------------------------------------------------------------------------------- 1 | using namespace System.Net 2 | 3 | # Input bindings are passed in via param block. 4 | param($Request, $TriggerMetadata) 5 | 6 | try { 7 | 8 | # Write to the Azure Functions log stream. 9 | Write-Host "PowerShell HTTP trigger Started." 10 | 11 | # Interact with query parameters or the body of the request. 12 | $ResourceGroupName = $Request.Body.ResourceGroupName 13 | $ServerName = $Request.Body.ServerName 14 | $DatabaseName = $Request.Body.DatabaseName 15 | 16 | If (!$ResourceGroupName) {$ResourceGroupName =$env:ResourceGroupName} 17 | If (!$ServerName ) {$ServerName =$env:ServerName } 18 | If (!$DatabaseName ) {$DatabaseName =$env:DatabaseName } 19 | 20 | Write-Host "$ServerName =" + $env:ServerName 21 | Write-Host "$DatabaseName =" + $env:DatabaseName 22 | 23 | 24 | $body=Suspend-AzSqlDatabase -ServerName $ServerName -DatabaseName $DatabaseName -ResourceGroupName $ResourceGroupName 25 | 26 | # Associate values to output bindings by calling 'Push-OutputBinding'. 27 | Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ 28 | StatusCode = [HttpStatusCode]::OK 29 | Body = $body 30 | }) 31 | 32 | } 33 | catch { 34 | Push-OutputBinding -Name Response -Value ([HttpResponseContext]@{ 35 | StatusCode = [HttpStatusCode]::ExpectationFailed 36 | Body = $error[0].ToString() + $error[0].InvocationInfo.PositionMessage 37 | }) 38 | 39 | } 40 | 41 | 42 | -------------------------------------------------------------------------------- /SqlPools/Maintenance/vStats.sql: -------------------------------------------------------------------------------- 1 | SET ANSI_NULLS ON 2 | GO 3 | SET QUOTED_IDENTIFIER ON 4 | GO 5 | IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[vStats]') AND type in (N'V')) 6 | EXEC dbo.sp_executesql @statement = N'CREATE VIEW [dbo].[vStats] AS SELECT 1 as Dummy' 7 | GO 8 | ALTER VIEW [dbo].[vStats] AS SELECT 9 | /* 10 | Description:Return data on individual Statistics including last updated date 11 | Source: https://github.com/ProdataSQL/SynapseTools 12 | History: 12/08/2021 Bob, Created 13 | 07/07/2022 Bob, Added Stats Sample Rate 14 | 09/07/2022 Bob, Added user_created as we dont want to update system stats on CCS 15 | */ 16 | ss.object_id, ss.name as stat_name , vs.table_name, vs.schema_name 17 | , ss.stats_id, ss.auto_created, ss.filter_definition, STATS_DATE(ss.object_id, ss.stats_id) as last_updated_date 18 | , sc.stat_columns 19 | , vs.stats_row_count 20 | , vs.actual_row_count 21 | , vs.stats_difference_percent 22 | , vs.stats_sample_rate 23 | , 'UPDATE STATISTICS ' + quotename(vs.[schema_name]) + '.' + quotename(vs.[table_name]) + ' (' + ss.name + ')' 24 | + coalesce(case when vs.stats_sample_rate >=100 THEN ' WITH FULLSCAN' ELSE ' WITH SAMPLE ' + convert(varchar,vs.stats_sample_rate) + ' PERCENT' END,'') as sqlCommand 25 | , ss.user_created 26 | FROM sys.stats ss 27 | INNER JOIN ( 28 | select sc.object_id, sc.stats_id, string_agg(c.name, ',') as stat_columns 29 | from sys.stats_columns sc 30 | INNER JOIN sys.columns c on c.object_id=sc.object_id and c.column_id=sc.column_id 31 | GROUP BY sc.object_id, stats_id 32 | ) sc on sc.object_id=ss.object_id and sc.stats_id=ss.stats_id 33 | INNER JOIN dbo.vTableStats vs on vs.object_id=ss.object_id; 34 | GO 35 | GO 36 | 37 | -------------------------------------------------------------------------------- /SqlPools/Demos/01-00. CS Intro.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "kernelspec": { 4 | "name": "SQL", 5 | "display_name": "SQL", 6 | "language": "sql" 7 | }, 8 | "language_info": { 9 | "name": "sql", 10 | "version": "" 11 | }, 12 | "extensions": { 13 | "azuredatastudio": { 14 | "version": 1, 15 | "views": [] 16 | } 17 | } 18 | }, 19 | "nbformat_minor": 2, 20 | "nbformat": 4, 21 | "cells": [ 22 | { 23 | "cell_type": "markdown", 24 | "source": [ 25 | "# ColumnStoreOptimize - Introduction" 26 | ], 27 | "metadata": { 28 | "azdata_cell_guid": "6bc6a6a2-166d-4792-b337-46ce4274cf59", 29 | "extensions": { 30 | "azuredatastudio": { 31 | "views": [] 32 | } 33 | } 34 | }, 35 | "attachments": {} 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "source": [ 40 | "A columnstore contains multiple row groups (and segments) with two types\n", 41 | "\n", 42 | "- closed - compressed row groups using ColumnStore compression\n", 43 | "- open - A delta row group containing rowstore clustered index. There can be deltastores for inserted AND deleted rows\n", 44 | "\n", 45 | "![Clustered columnstore index](https://docs.microsoft.com/en-us/sql/relational-databases/indexes/media/sql-server-pdw-columnstore-physicalstorage.gif?view=sql-server-ver16)" 46 | ], 47 | "metadata": { 48 | "azdata_cell_guid": "fa81beb0-83f8-467f-bfee-c80d05e46021", 49 | "extensions": { 50 | "azuredatastudio": { 51 | "views": [] 52 | } 53 | } 54 | }, 55 | "attachments": {} 56 | } 57 | ] 58 | } -------------------------------------------------------------------------------- /SqlPools/Maintenance/vPartitionStats.sql: -------------------------------------------------------------------------------- 1 | SET ANSI_NULLS ON 2 | GO 3 | SET QUOTED_IDENTIFIER ON 4 | GO 5 | IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[vPartitionStats]') AND type in (N'V')) 6 | EXEC dbo.sp_executesql @statement = N'CREATE VIEW [dbo].[vPartitionStats] AS SELECT 1 as Dummy' 7 | GO 8 | ALTER VIEW [dbo].[vPartitionStats] 9 | AS with 10 | /* 11 | Description:Return data on outdated and missing statistics. Use this for Index Maintenance 12 | Source: https://github.com/ProdataSQL/SynapseTools 13 | Concept copied from https://github.com/abrahams1/Azure_Synapse_Toolbox/tree/master/SQL_Queries/Statistics 14 | History: 12/08/2021 Bob, Created 15 | */ 16 | cmp_summary as 17 | ( 18 | select tm.object_id, ps.partition_number, sum(ps.row_count) cmp_row_count 19 | from sys.dm_pdw_nodes_db_partition_stats ps 20 | inner join sys.pdw_nodes_tables nt on nt.object_id=ps.object_id and ps.distribution_id=nt.distribution_id 21 | inner join sys.pdw_table_mappings tm on tm.physical_name=nt.name 22 | group by tm.object_id, ps.partition_number 23 | 24 | ) 25 | , ctl_summary as 26 | ( 27 | select t.object_id, s.name [schema_name], t.name table_name, case when ps.name is not null then p.partition_number end as partition_number, ps.name as partition_scheme, i.type_desc table_type, dp.distribution_policy_desc distribution_type, sum(p.rows) ctl_row_count 28 | from sys.schemas s 29 | inner join sys.tables t on t.schema_id=s.schema_id 30 | inner join sys.pdw_table_distribution_properties dp on dp.object_id=t.object_id 31 | inner join sys.indexes i on i.object_id=t.object_id and i.index_id<2 32 | inner join sys.partitions p on p.object_id=t.object_id and p.index_id=i.index_id 33 | left join sys.partition_schemes ps on ps.data_space_id=i.data_space_id 34 | group by t.object_id, s.name, t.name, i.type_desc, dp.distribution_policy_desc,p.partition_number, ps.name 35 | ) 36 | , [all_results] as 37 | ( 38 | select ctl.object_id, ctl.[schema_name], ctl.table_name, ctl.partition_number, ctl.table_type, ctl.distribution_type 39 | , ctl.ctl_row_count as stats_row_count, cmp.cmp_row_count as actual_row_count, convert(decimal(10,2),(abs(ctl.ctl_row_count - cmp.cmp_row_count)*100.0 / nullif(cmp.cmp_row_count,0))) stats_difference_percent 40 | 41 | from ctl_summary ctl 42 | inner join cmp_summary cmp on ctl.object_id=cmp.object_id and (ctl.partition_number=cmp.partition_number or ctl.partition_number is null) 43 | 44 | ) 45 | select * from [all_results]; 46 | GO 47 | 48 | 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SynapseTools 2 | ## Welcome 3 | Welcome to the [Prodata](https://www.prodata.ie) open source library for dedicated SQL Pool maintenance and monitoring. The code contained here is open source/free with no warranty or support implied. Feel free to ping me bob@prodata.ie if you found it useful. 4 | 5 | 6 | ## Getting Started 7 | * Install Maintenance Solution from [MaintenanceSolution.sql](https://github.com/ProdataSQL/SynapseTools/blob/main/SqlPools/Maintenance/MaintenanceSolution.sql) 8 | * View [Maintenance Source Code](https://github.com/ProdataSQL/SynapseTools/blob/main/SqlPools/Maintenance) 9 | * View [Monitoring Source Code](https://github.com/ProdataSQL/SynapseTools/blob/main/SqlPools/Monitoring) 10 | 11 | Click on one of the above linkes ot get started or you can read more details, syntax and example from the below section 12 | 13 | 14 | ## SQL Pool Maintenance Solution 15 | Goto [Maintenance](https://github.com/ProdataSQL/SynapseTools/blob/main/SqlPools/Maintenance) for details on the Synapse SQL Pool maintenance solutiuon for Columnstore and Statistics maintenance. 16 | https://github.com/ProdataSQL/SynapseTools/blob/main/SqlPools/Maintenance 17 | 18 | This solution will help automate daily/weekly maintenance in a simialr syntax to the ubiquitous [Ola Hallogren SQL Maintenance Solution](https://ola.hallengren.com/), but for SQL dedicated Pools. 19 | 20 | Note that maintenance on Synapse SQL Pools is very different than the traditional DB Engine due to scale out archietcture and less DMVs exposed to track usage. 21 | 22 | 23 | ## SQL Pool Monitoring 24 | Selection of sample stored procedures to help with SQL Pool monitoring based on the SQL Pool DMVs and the awesome [sp_WhoIsActive](http://whoisactive.com/) from [Adam machanic](http://dataeducation.com/about/). Albeit a lot more limited due to restrictions in SQL Pool DMVs. 25 | 26 | https://github.com/ProdataSQL/SynapseTools/tree/main/SqlPools/Monitoring 27 | 28 | Views and SProcs include 29 | * [Requests](https://github.com/ProdataSQL/SynapseTools/blob/main/SqlPools/Monitoring/Requests.sql) View. 30 | Showing all requests and an analysis of step timing. use this to spot shuffles, movement and other performance issues 31 | * [sp_WhoIsActive](https://github.com/ProdataSQL/SynapseTools/blob/main/SqlPools/Monitoring/sp_WhoIsActive.sql). A wrapper for the Requests View showing currently running requests and timing 32 | * [sp_WhoWasActive](https://github.com/ProdataSQL/SynapseTools/blob/main/SqlPools/Monitoring/sp_WhoWasActive.sql). A wrapper for the Requests View showing historical requests and timing. 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /SqlPools/Demos/01-04. Is it worth it.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "kernelspec": { 4 | "name": "SQL", 5 | "display_name": "SQL", 6 | "language": "sql" 7 | }, 8 | "language_info": { 9 | "name": "sql", 10 | "version": "" 11 | }, 12 | "extensions": { 13 | "azuredatastudio": { 14 | "version": 1, 15 | "views": [] 16 | } 17 | } 18 | }, 19 | "nbformat_minor": 2, 20 | "nbformat": 4, 21 | "cells": [ 22 | { 23 | "cell_type": "markdown", 24 | "source": [ 25 | "# ColumnStoreOptimize - Is it worth it ?" 26 | ], 27 | "metadata": { 28 | "azdata_cell_guid": "6bc6a6a2-166d-4792-b337-46ce4274cf59", 29 | "extensions": { 30 | "azuredatastudio": { 31 | "views": [] 32 | } 33 | } 34 | }, 35 | "attachments": {} 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "source": [ 40 | "## As as a test , I took a **60 million** row table with a basic aggregate query and creates two tables\n", 41 | "\n", 42 | "a) One with 64 row Groups.\n", 43 | "\n", 44 | "b) One with 582 row groups. Almost worst case\n", 45 | "\n", 46 | "![](https://i0.wp.com/prodata.ie/wp-content/uploads/2021/08/image-10.png?resize=481,288&ssl=1)" 47 | ], 48 | "metadata": { 49 | "azdata_cell_guid": "fa81beb0-83f8-467f-bfee-c80d05e46021", 50 | "extensions": { 51 | "azuredatastudio": { 52 | "views": [] 53 | } 54 | } 55 | }, 56 | "attachments": {} 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "source": [ 61 | "## 1.1 Lessons learned\n", 62 | "\n", 63 | "- Watch out for \"incremental\" fact tables inserting less than 60 million rows at a time.\n", 64 | "- Measure queries before and after a REORG" 65 | ], 66 | "metadata": { 67 | "azdata_cell_guid": "c9e3f785-68fe-4310-9eb7-ea6abf14a0b7", 68 | "extensions": { 69 | "azuredatastudio": { 70 | "views": [] 71 | } 72 | } 73 | }, 74 | "attachments": {} 75 | } 76 | ] 77 | } -------------------------------------------------------------------------------- /SqlPools/Monitoring/Requests.sql: -------------------------------------------------------------------------------- 1 | /****** Object: View [dbo].[Requests] Script Date: 08/05/2021 18:09:21 ******/ 2 | SET ANSI_NULLS ON 3 | GO 4 | 5 | SET QUOTED_IDENTIFIER ON 6 | GO 7 | 8 | CREATE VIEW [dbo].[Requests] AS WITH 9 | /* 10 | Description: Return Data for SQLDW Requets, Steps, Operations in summary from 11 | used By: sp_WhoIsActive, sp_WhoWasActive 12 | History: 11/04/2021 Bob, Created for Perf Tuning 13 | 11/04/2021 Bob, Filtered out Requests with no steps 14 | */ 15 | requests as 16 | ( 17 | select row_number() over (order by r.request_id desc ) as [#] , r.request_id, r.session_id, r.command 18 | , r.submit_time, r.start_time, r.total_elapsed_time 19 | , end_time 20 | , r.status 21 | , r.resource_class 22 | , getdate() as collection_time 23 | , s.login_name 24 | , s.app_name 25 | , s.client_id 26 | , s.sql_spid 27 | from sys.dm_pdw_exec_requests r 28 | INNER JOIN sys.dm_pdw_exec_sessions s on s.session_id=r.session_id 29 | WHERE r.session_id <> SESSION_ID ( ) 30 | AND app_name <> 'Microsoft SQL Server Management Studio' 31 | ), steps as ( 32 | select request_id, operation_type , total_elapsed_time, SUBSTRING(command,1,(CHARINDEX(' ',command + ' ')-1)) as command 33 | From sys.dm_pdw_request_steps 34 | WHERE location_type='Compute' 35 | ) 36 | select TOP 5000 r.[#], r.request_id, 37 | r.start_time 38 | , r.end_time 39 | , FORMAT(datediff( hh ,r.start_time ,r.end_time ), 'D2') + ':' + FORMAT(datediff( mi ,r.start_time ,r.end_time ) % 60, 'D2') 40 | + ':' + FORMAT(datediff( ss ,r.start_time ,r.end_time ) % 60 % 60, 'D2') AS [Duration] 41 | , r.status, r.total_elapsed_time as request_time_ms 42 | , m.total_elapsed_time as step_time_ms 43 | , r.command 44 | , m.steps, op.op_commands, m.operations 45 | , r.session_id 46 | , r.submit_time 47 | 48 | , r.resource_class 49 | , r.collection_time 50 | , r.login_name 51 | , r.app_name 52 | , r.client_id 53 | , r.sql_spid 54 | from requests r 55 | LEFT JOIN ( 56 | SELECT request_id, string_agg(command,',') WITHIN GROUP (ORDER BY total_elapsed_time DESC) as op_commands 57 | FROM ( 58 | SELECT request_id, command + '(' + convert(varchar,count(*)) + ':' + convert(varchar,sum(total_elapsed_time)) +'ms)' as command , sum(total_elapsed_time) as total_elapsed_time 59 | FROM STEPS 60 | WHERE operation_type='OnOperation' 61 | group by request_id,command 62 | ) op1 63 | GROUP BY request_id 64 | ) op on op.request_id= r.request_id 65 | INNER JOIN ( 66 | SELECT request_id, string_agg(operation,',') WITHIN GROUP (ORDER BY total_elapsed_time DESC) as operations, count(*) as steps, sum(total_elapsed_time) as total_elapsed_time 67 | FROM ( 68 | SELECT request_id, replace(operation_type,'Operation','') + '(' + convert(varchar,count(*)) + ':' + convert(varchar,sum(total_elapsed_time)) +'ms)' as operation, sum(total_elapsed_time) as total_elapsed_time, count(*) as steps 69 | FROM STEPS 70 | group by request_id,operation_type 71 | ) m1 72 | GROUP BY request_id 73 | ) m on m.request_id= r.request_id 74 | order by r.submit_time desc; 75 | GO 76 | 77 | 78 | -------------------------------------------------------------------------------- /SqlPools/Maintenance/vTableStats.sql: -------------------------------------------------------------------------------- 1 | SET ANSI_NULLS ON 2 | GO 3 | SET QUOTED_IDENTIFIER ON 4 | GO 5 | IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[vTableStats]') AND type in (N'V')) 6 | EXEC dbo.sp_executesql @statement = N'CREATE VIEW [dbo].[vTableStats] AS SELECT 1 as Dummy' 7 | GO 8 | ALTER VIEW [dbo].[vTableStats] AS with 9 | /* 10 | Description:Return data on outdated and missing statistics. Use this for Index Maintenance 11 | Source: https://github.com/ProdataSQL/SynapseTools 12 | Concept copied from https://github.com/abrahams1/Azure_Synapse_Toolbox/tree/master/SQL_Queries/Statistics 13 | History: 12/08/2021 Bob, Created 14 | */ 15 | cmp_summary as 16 | ( 17 | select tm.object_id, sum(ps.row_count) cmp_row_count 18 | , case when sum(ps.row_count) =0 then 100 else sqrt(sum(ps.row_count)) *1000/ sum(ps.row_count)*100 end as stats_sample_rate 19 | , convert(bigint,SQRT(1000 * sum(ps.row_count))) as dynamic_threshold_rows 20 | from sys.dm_pdw_nodes_db_partition_stats ps 21 | inner join sys.pdw_nodes_tables nt on nt.object_id=ps.object_id and ps.distribution_id=nt.distribution_id 22 | inner join sys.pdw_table_mappings tm on tm.physical_name=nt.name 23 | where ps.index_id<2 24 | group by tm.object_id 25 | 26 | ) 27 | , ctl_summary as 28 | ( 29 | select t.object_id, s.name [schema_name], t.name table_name, i.type_desc table_type, dp.distribution_policy_desc distribution_type, sum(p.rows) ctl_row_count 30 | from sys.schemas s 31 | inner join sys.tables t on t.schema_id=s.schema_id 32 | inner join sys.pdw_table_distribution_properties dp on dp.object_id=t.object_id 33 | inner join sys.indexes i on i.object_id=t.object_id and i.index_id<2 34 | inner join sys.partitions p on p.object_id=t.object_id and p.index_id=i.index_id 35 | group by t.object_id, s.name, t.name, i.type_desc, dp.distribution_policy_desc 36 | ) 37 | , [all_results] as 38 | ( 39 | select ctl.object_id, ctl.[schema_name], ctl.table_name, ctl.table_type, ctl.distribution_type 40 | , ctl.ctl_row_count as stats_row_count, cmp.cmp_row_count as actual_row_count, convert(decimal(10,2),(abs(ctl.ctl_row_count - cmp.cmp_row_count)*100.0 / nullif(cmp.cmp_row_count,0))) stats_difference_percent 41 | , convert(int,case when cmp.stats_sample_rate < 1 then 1 when cmp.stats_sample_rate > 100 then 100 else cmp.stats_sample_rate end ) as stats_sample_rate 42 | , cmp.dynamic_threshold_rows 43 | from ctl_summary ctl 44 | inner join cmp_summary cmp on ctl.object_id=cmp.object_id 45 | ) 46 | select [object_id], [schema_name], [table_name], [table_type], [distribution_type], [stats_row_count], [actual_row_count], [stats_difference_percent] 47 | , dynamic_threshold_rows 48 | , [stats_sample_rate] 49 | , case when stats_difference_percent >=20 or abs([actual_row_count] - [stats_row_count]) > dynamic_threshold_rows then 1 else 0 end as recommend_update 50 | , 'UPDATE STATISTICS ' + quotename([schema_name]) + '.' + quotename([table_name]) 51 | + coalesce(case when stats_sample_rate >=100 THEN ' WITH FULLSCAN' ELSE ' WITH SAMPLE ' + convert(varchar,stats_sample_rate) + ' PERCENT' END,'') as sqlCommand 52 | from [all_results]; 53 | GO 54 | 55 | 56 | -------------------------------------------------------------------------------- /SqlPools/Maintenance/vColumnStoreStats.sql: -------------------------------------------------------------------------------- 1 | SET ANSI_NULLS ON 2 | GO 3 | SET QUOTED_IDENTIFIER ON 4 | GO 5 | IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[vColumnstoreStats]') AND type in (N'V')) 6 | EXEC dbo.sp_executesql @statement = N'CREATE VIEW [dbo].[vColumnstoreStats] AS SELECT 1 as Dummy' 7 | GO 8 | ALTER VIEW [dbo].[vColumnstoreStats] AS WITH ColumnStats as ( 9 | /* 10 | Description:View for querying ColumnStore Stats and Fragmentation. 11 | used By: ColumnstoreOptimze SProc 12 | History: 13 | 10/08/2021 Bob, Created for Synapse Dedicated Pools 14 | 07/07/2022 Bob, Adjusted density calculation if < 60 million 15 | */ 16 | SELECT t.object_id as object_id 17 | , s.name AS [schema_name] 18 | , t.name AS [table_name] 19 | , case when ps.name is null then null else rg.[partition_number] end as partition_number 20 | , rg.[total_rows] AS [row_count] 21 | , i.name as index_name 22 | , ps.name as partition_scheme 23 | , CASE WHEN rg.[State] = 3 THEN rg.[total_rows] END as compressed_row_count 24 | , CASE WHEN rg.[State] = 3 Then rg.[deleted_rows] ELSE 0 end as deleted_row_count 25 | , CASE WHEN rg.[State] = 1 THEN rg.[total_rows] ELSE 0 END as open_row_count 26 | , CASE WHEN rg.[State] = 3 THEN 1 ELSE 0 END as compressed_rowgroup_count 27 | , CASE WHEN rg.[State] = 1 THEN 1 ELSE 0 END as open_rowgroup_count 28 | , CASE WHEN rg.[State] IN (1,3) THEN 0 ELSE 1 END as other_rowgroup_count /* Tombstone or invisible*/ 29 | FROM sys.[pdw_nodes_column_store_row_groups] rg 30 | INNER JOIN sys.[pdw_nodes_tables] nt ON rg.[object_id] = nt.[object_id] 31 | AND rg.[pdw_node_id] = nt.[pdw_node_id] 32 | AND rg.[distribution_id] = nt.[distribution_id] 33 | INNER JOIN sys.[pdw_table_mappings] mp ON nt.[name] = mp.[physical_name] 34 | INNER JOIN sys.[tables] t ON mp.[object_id] = t.[object_id] 35 | INNER JOIN sys.[schemas] s ON t.[schema_id] = s.[schema_id] 36 | INNER JOIN sys.indexes i on i.object_id=t.object_id and i.type=5 /* CCI */ 37 | LEFT JOIN sys.partition_schemes ps on i.data_space_id=ps.data_space_id 38 | WHERE rg.[State] IN (1,3) 39 | ) 40 | SELECT getdate() AS [execution_date] 41 | , DB_Name() AS [database_name] 42 | , cs.schema_name 43 | , cs.table_name 44 | , cs.[partition_number] 45 | , cs.partition_scheme 46 | , cs.object_id 47 | , cs.index_name 48 | , sum(cs.row_count) as row_count 49 | , sum(deleted_row_count) as deleted_row_count 50 | , count(*) as row_group_count 51 | , sum(cs.compressed_row_count) as compressed_row_count 52 | , sum(compressed_rowgroup_count) as compressed_rowgroup_count 53 | , sum(cs.open_rowgroup_count) as open_rowgroup_count 54 | , sum(cs.open_row_count) as open_row_count 55 | , max(cs.compressed_row_count) as compressed_row_max 56 | , avg(cs.compressed_row_count) as compressed_row_avg 57 | , 100-convert(numeric(10,4),convert(float,avg(cs.compressed_row_count)) / CASE WHEN sum(cs.compressed_row_count) > 60000000 then 1048576 else max(cs.compressed_row_count) end )*100 as fragmentation_density 58 | , convert(numeric(10,4),convert(float,sum(cs.deleted_row_count)) / coalesce(max(cs.compressed_row_count),1048576)/60)*100 as fragmentation_deletes 59 | , convert(numeric(10,4),convert(float,sum(cs.open_row_count)) / coalesce(max(cs.compressed_row_count),1048576)/60)*100 as fragmentation_open 60 | FROM ColumnStats cs 61 | 62 | GROUP BY cs.schema_name 63 | , cs.table_name 64 | , cs.[partition_number] 65 | , cs.partition_scheme 66 | , cs.index_name 67 | , cs.object_id; 68 | GO 69 | 70 | 71 | -------------------------------------------------------------------------------- /SqlPools/Demos/01.07. CS Productionisation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "kernelspec": { 4 | "name": "SQL", 5 | "display_name": "SQL", 6 | "language": "sql" 7 | }, 8 | "language_info": { 9 | "name": "sql", 10 | "version": "" 11 | } 12 | }, 13 | "nbformat_minor": 2, 14 | "nbformat": 4, 15 | "cells": [ 16 | { 17 | "cell_type": "markdown", 18 | "source": [ 19 | "### Splitting up Your Work\n", 20 | "\n", 21 | "You can use the @Tables parameter and the @Timelimit to avoid having to al maintenance work in one window. Example below." 22 | ], 23 | "metadata": { 24 | "azdata_cell_guid": "b17c1e49-153d-441a-add4-42035fef0f89" 25 | }, 26 | "attachments": {} 27 | }, 28 | { 29 | "cell_type": "code", 30 | "source": [ 31 | "/* Do Default Maintenance on All Fact Tables except those in staging schema \r\n", 32 | " reorg if > 20% fragmented (may cause churn if this low. Recommend 25%)\r\n", 33 | " Only do a rebuild if > 1 million deletes\r\n", 34 | " Stop after 1 hour\r\n", 35 | " Dont actually run just list commands\r\n", 36 | "*/\r\n", 37 | "\r\n", 38 | "exec [dbo].[ColumnstoreOptimize] \r\n", 39 | "@Tables='Fact%,-stg.%'\r\n", 40 | ",@DensityThreshold=20\r\n", 41 | ",@OpenThreshold=null\r\n", 42 | ",@DeleteThreshold=1000000\r\n", 43 | ",@TimeLimit =3600\r\n", 44 | ",@Execute='N'\r\n", 45 | "\r\n", 46 | "\r\n", 47 | "" 48 | ], 49 | "metadata": { 50 | "azdata_cell_guid": "23dc7197-ca5f-4e86-a719-6cadc321e0d9", 51 | "tags": [], 52 | "language": "sql" 53 | }, 54 | "outputs": [ 55 | { 56 | "output_type": "display_data", 57 | "data": { 58 | "text/html": "Date and time: 2022-07-07 20:03:06" 59 | }, 60 | "metadata": {} 61 | }, 62 | { 63 | "output_type": "display_data", 64 | "data": { 65 | "text/html": "Server: aw-dev" 66 | }, 67 | "metadata": {} 68 | }, 69 | { 70 | "output_type": "display_data", 71 | "data": { 72 | "text/html": "Version: 10.0.15665.0" 73 | }, 74 | "metadata": {} 75 | }, 76 | { 77 | "output_type": "display_data", 78 | "data": { 79 | "text/html": "Edition: SQL Azure" 80 | }, 81 | "metadata": {} 82 | }, 83 | { 84 | "output_type": "display_data", 85 | "data": { 86 | "text/html": "Parameters: @Tables= 'Fact%,-stg.%', @DensityThreshold = 20.00, @OpenThreshold = 200000, @DeleteThreshold = 1000000, @Execute = N" 87 | }, 88 | "metadata": {} 89 | }, 90 | { 91 | "output_type": "display_data", 92 | "data": { 93 | "text/html": "Version: Microsoft Azure SQL Data Warehouse - 10.0.15665.0 Jul 6 2022 10:30:07 Copyright (c) Microsoft Corporation" 94 | }, 95 | "metadata": {} 96 | }, 97 | { 98 | "output_type": "display_data", 99 | "data": { 100 | "text/html": "Source: https://github.com/ProdataSQL/SynapseTools/tree/main/SqlPools/Maintenance" 101 | }, 102 | "metadata": {} 103 | }, 104 | { 105 | "output_type": "display_data", 106 | "data": { 107 | "text/html": "Date and time: 2022-07-07 20:03:09" 108 | }, 109 | "metadata": {} 110 | }, 111 | { 112 | "output_type": "display_data", 113 | "data": { 114 | "text/html": "SqlCommand: ALTER INDEX ClusteredIndex_baff5ac740d64e1694b250169905d4cf ON [dbo].[FactFinance100m] REORGANIZE" 115 | }, 116 | "metadata": {} 117 | }, 118 | { 119 | "output_type": "display_data", 120 | "data": { 121 | "text/html": "Total execution time: 00:00:03.211" 122 | }, 123 | "metadata": {} 124 | } 125 | ], 126 | "execution_count": 2 127 | } 128 | ] 129 | } -------------------------------------------------------------------------------- /SqlPools/Monitoring/vTableSizes.sql: -------------------------------------------------------------------------------- 1 | /****** Object: View [dbo].[vTableSizes] Script Date: 08/05/2021 18:09:40 ******/ 2 | SET ANSI_NULLS ON 3 | GO 4 | 5 | SET QUOTED_IDENTIFIER ON 6 | GO 7 | 8 | CREATE VIEW [dbo].[vTableSizes] 9 | AS WITH base 10 | AS 11 | ( 12 | SELECT 13 | GETDATE() AS [execution_time] 14 | , DB_NAME() AS [database_name] 15 | , s.name AS [schema_name] 16 | , t.name AS [table_name] 17 | , QUOTENAME(s.name)+'.'+QUOTENAME(t.name) AS [two_part_name] 18 | , nt.[name] AS [node_table_name] 19 | , ROW_NUMBER() OVER(PARTITION BY nt.[name] ORDER BY (SELECT NULL)) AS [node_table_name_seq] 20 | , tp.[distribution_policy_desc] AS [distribution_policy_name] 21 | , nt.[distribution_id] AS [distribution_id] 22 | , i.[type] AS [index_type] 23 | , i.[type_desc] AS [index_type_desc] 24 | , nt.[pdw_node_id] AS [pdw_node_id] 25 | , pn.[type] AS [pdw_node_type] 26 | , pn.[name] AS [pdw_node_name] 27 | , di.name AS [dist_name] 28 | , di.position AS [dist_position] 29 | , nps.[partition_number] AS [partition_nmbr] 30 | , nps.[reserved_page_count] AS [reserved_space_page_count] 31 | , nps.[reserved_page_count] - nps.[used_page_count] AS [unused_space_page_count] 32 | , nps.[in_row_data_page_count] 33 | + nps.[row_overflow_used_page_count] 34 | + nps.[lob_used_page_count] AS [data_space_page_count] 35 | , nps.[reserved_page_count] 36 | - (nps.[reserved_page_count] - nps.[used_page_count]) 37 | - ([in_row_data_page_count] 38 | + [row_overflow_used_page_count]+[lob_used_page_count]) AS [index_space_page_count] 39 | , nps.[row_count] AS [row_count] 40 | from sys.schemas s 41 | join sys.tables t ON s.[schema_id] = t.[schema_id] 42 | join sys.indexes i ON t.[object_id] = i.[object_id] 43 | AND i.[index_id] <= 1 44 | join sys.pdw_table_distribution_properties tp ON t.[object_id] = tp.[object_id] 45 | join sys.pdw_table_mappings tm ON t.[object_id] = tm.[object_id] 46 | join sys.pdw_nodes_tables nt ON tm.[physical_name] = nt.[name] 47 | join sys.dm_pdw_nodes pn ON nt.[pdw_node_id] = pn.[pdw_node_id] 48 | join sys.pdw_distributions di ON nt.[distribution_id] = di.[distribution_id] 49 | join sys.dm_pdw_nodes_db_partition_stats nps ON nt.[object_id] = nps.[object_id] 50 | AND nt.[pdw_node_id] = nps.[pdw_node_id] 51 | AND nt.[distribution_id] = nps.[distribution_id] 52 | ) 53 | , size 54 | AS 55 | ( 56 | SELECT 57 | [execution_time] 58 | , [database_name] 59 | , [schema_name] 60 | , [table_name] 61 | , [two_part_name] 62 | , [node_table_name] 63 | , [node_table_name_seq] 64 | , [distribution_policy_name] 65 | , [distribution_id] 66 | , [index_type] 67 | , [index_type_desc] 68 | , [pdw_node_id] 69 | , [pdw_node_type] 70 | , [pdw_node_name] 71 | , [dist_name] 72 | , [dist_position] 73 | , [partition_nmbr] 74 | , [reserved_space_page_count] 75 | , [unused_space_page_count] 76 | , [data_space_page_count] 77 | , [index_space_page_count] 78 | , [row_count] 79 | , ([reserved_space_page_count] * 8.0) AS [reserved_space_KB] 80 | , ([reserved_space_page_count] * 8.0)/1000 AS [reserved_space_MB] 81 | , ([reserved_space_page_count] * 8.0)/1000000 AS [reserved_space_GB] 82 | , ([reserved_space_page_count] * 8.0)/1000000000 AS [reserved_space_TB] 83 | , ([unused_space_page_count] * 8.0) AS [unused_space_KB] 84 | , ([unused_space_page_count] * 8.0)/1000 AS [unused_space_MB] 85 | , ([unused_space_page_count] * 8.0)/1000000 AS [unused_space_GB] 86 | , ([unused_space_page_count] * 8.0)/1000000000 AS [unused_space_TB] 87 | , ([data_space_page_count] * 8.0) AS [data_space_KB] 88 | , ([data_space_page_count] * 8.0)/1000 AS [data_space_MB] 89 | , ([data_space_page_count] * 8.0)/1000000 AS [data_space_GB] 90 | , ([data_space_page_count] * 8.0)/1000000000 AS [data_space_TB] 91 | , ([index_space_page_count] * 8.0) AS [index_space_KB] 92 | , ([index_space_page_count] * 8.0)/1000 AS [index_space_MB] 93 | , ([index_space_page_count] * 8.0)/1000000 AS [index_space_GB] 94 | , ([index_space_page_count] * 8.0)/1000000000 AS [index_space_TB] 95 | FROM base 96 | ) 97 | SELECT * 98 | FROM size; 99 | GO 100 | 101 | 102 | -------------------------------------------------------------------------------- /SqlPools/Demos/02.5 StatsOptimize.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "kernelspec": { 4 | "name": "SQL", 5 | "display_name": "SQL", 6 | "language": "sql" 7 | }, 8 | "language_info": { 9 | "name": "sql", 10 | "version": "" 11 | }, 12 | "extensions": { 13 | "azuredatastudio": { 14 | "version": 1, 15 | "views": [] 16 | } 17 | } 18 | }, 19 | "nbformat_minor": 2, 20 | "nbformat": 4, 21 | "cells": [ 22 | { 23 | "cell_type": "markdown", 24 | "source": [ 25 | "## StatsOptimize\n", 26 | "\n", 27 | "StatsOptimize is stored procedure for updating Statistics for Synapse SQL Pools. This can be critical for SQL Pools as while they support auto create of stats they do not support auto updte of statistics. This procedure is like a replacement for auto update and allows for more control and flexibility.\n", 28 | "\n", 29 | "Key features include:\n", 30 | "\n", 31 | "- Dynamic determine modification level based on imporved algorithm.\n", 32 | "- Dynamically determine sampling level or support setting sample level.\n", 33 | "- Support removal of duplicate statistics (covering same column).\n", 34 | "- [Ola Hallogren](https://ola.hallengren.com/) style features such as CommandLog, @Time Limit, and @Table parameer to set scope.\n", 35 | "\n", 36 | "All executed commands are logged to the [CommandLog](https://github.com/ProdataSQL/SynapseTools/blob/main/SqlPools/Maintenance/CommandLog.sql) table.\n", 37 | "\n", 38 | "It is based on Best practise guidance from MS sites below and community.\n", 39 | "\n", 40 | "- [https://docs.microsoft.com/en-us/azure/synapse-analytics/sql/develop-tables-statistics](https://docs.microsoft.com/en-us/azure/synapse-analytics/sql/develop-tables-statistics)\n", 41 | "- [https://github.com/abrahams1/Azure\\_Synapse\\_Toolbox/tree/master/SQL\\_Queries/Statistics](https://github.com/abrahams1/Azure_Synapse_Toolbox/tree/master/SQL_Queries/Statistics)\n", 42 | "- [https://www.sqlskills.com/blogs/tim/when-updating-statistics-is-too-expensive/](https://www.sqlskills.com/blogs/tim/when-updating-statistics-is-too-expensive/)\n", 43 | "- [https://www.sqlskills.com/blogs/erin/updating-statistics-with-ola-hallengrens-script/](https://www.sqlskills.com/blogs/erin/updating-statistics-with-ola-hallengrens-script/)\n", 44 | "- [https://docs.microsoft.com/en-us/sql/relational-databases/statistics/statistics?view=sql-server-2017](https://docs.microsoft.com/en-us/sql/relational-databases/statistics/statistics?view=sql-server-2017)\n", 45 | "- [https://ola.hallengren.com/](https://ola.hallengren.com/)\n", 46 | "\n", 47 | "### Usage:\n", 48 | "\n", 49 | "```\n", 50 | "exec [dbo].[StatsOptimize] , @Tables, @StatisticsModificationLevel, @StatisticsSample ,@OnlyModifiedStatistics, @DeleteOverlappingStats, @TimeLimit , @Execute \n", 51 | "\n", 52 | "```\n", 53 | "\n", 54 | "#### Parameters\n", 55 | "\n", 56 | "##### @Tables\n", 57 | "\n", 58 | "Select Tables and optionally columns to be included. The minus character is used to exclude objects and wildcards (%) are also supported as SQL Like clause. Use this to exclude more complex tables, exclude staging, or only include relevant schemas and objects \n", 59 | "\n", 60 | "| Value | Description |\n", 61 | "| --- | --- |\n", 62 | "| Null | All Tables in Pool |\n", 63 | "| dbo.% | Tables in dbo schema |\n", 64 | "| %.Fact% | All Fact tables, regardless of Schema |\n", 65 | "| %.Fact%,-FactBig | All Fact tables, Except one called FactBig |\n", 66 | "| %.%.Date,%.%.AccountKey | ONLY do stats maintenance on Date and AccountKey columns |\n", 67 | "\n", 68 | "Note that usually we just do stats maintenance at the table level, but there is also support for specifing column(s). This is a special case to support columns that need frequent updates like low cardinatlity incremental values (Eg Business Date) which is recommended by Microsoft [here](https://docs.microsoft.com/en-us/azure/synapse-analytics/sql/develop-tables-statistics)\n", 69 | "\n", 70 | "#### @StatisticsModificationLevel\n", 71 | "\n", 72 | "By default the SProc will only update a stat if the number of modified rows is greater than either\n", 73 | "\n", 74 | "- 20% or specified value in parameter\n", 75 | "- an Adaptive algorithm of SQRT(1000 \\* \\[row count)\\] based on improved stats algorithm introduced in SQL 2014\n", 76 | "\n", 77 | "The SProc uses the view vTableStats to return meta data such as the nuber of rows per table and the number of rows in the statistcis. The main difference between SQL Pools and a traditional SQL Server is that in a SQL Pool we can only track the estimated row count at the the table/partition level and not per statistic. Eg we do not have [sys.dm\\_db\\_stats\\_properties](https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-stats-properties-transact-sql?view=sql-server-ver15) but we do have [pdw\\_table\\_distribution\\_properties](https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-pdw-table-distribution-properties-transact-sql?view=aps-pdw-2016-au7) with per stat data.\n", 78 | "\n", 79 | "This ommission is largely due to the fact that statistics are per distribution, so it is much more complex to amalgamate and sychronise the set of 60 stat objects. This is also why we have no auto stats for SQL Pools (as of 31/08/2021).\n", 80 | "\n", 81 | "#### @StatisticsSample\n", 82 | "\n", 83 | "This is the sample rate. if this is null, then the SProc uses an adaptive sample rate\n", 84 | "\n", 85 | "- sqrt(\\[row count\\]) \\*1000/ \\[row count\\]\\*100\n", 86 | "\n", 87 | "While syanpse uses a default of 20% for the sample rate, there is a recommendation to use about 2% when we reach a billion rows. The adaptive sample rate generates recommendations such as:\n", 88 | "\n", 89 | "| Row Count | Sample % |\n", 90 | "| --- | --- |\n", 91 | "| 1000 | FULLSCAN |\n", 92 | "| 100k | FULLSCAN |\n", 93 | "| 1 million | FULLSCAN |\n", 94 | "| 10 million | 31 |\n", 95 | "| 100 million | 10 |\n", 96 | "| 1 billion | 3 |\n", 97 | "| 10 billion+ | 1 |\n", 98 | "\n", 99 | "#### @OnlyModifiedStatistics\n", 100 | "\n", 101 | "Default Y. Set this to N to\n", 102 | "\n", 103 | "#### @DeleteOverlappingStats\n", 104 | "\n", 105 | "Default N. Set to Y to delete any auto stats which overlap an existing statistic.\n", 106 | "\n", 107 | "#### @TimeLimit\n", 108 | "\n", 109 | "Default Null or infinite. Set a time limit in seconds for the job to run. No more commands will be started after time limit (but existing ones will finish). Use this if you have a short maintenace windows and do not want to exceed time.\n", 110 | "\n", 111 | "#### @Execute\n", 112 | "\n", 113 | "Default Y. Set to N to only show commands but not execute or log them. Useful for seeing experimental maintenance commands before actually executing them.\n", 114 | "\n", 115 | "### Example Usage\n", 116 | "\n", 117 | "Default Best Practise with smart defaults.\n", 118 | "\n", 119 | "```\n", 120 | "exec dbo.[StatsOptimize] @Tables=null, @StatisticsModificationLevel=null, @StatisticsSample=null ,@OnlyModifiedStatistics=null,@DeleteOverlappingStats=null, @TimeLimit=null , @Execute=null \n", 121 | "\n", 122 | "```\n", 123 | "\n", 124 | "Update stats in dbo schema except FactBig and use a FULLSCAN if any rows have changed\n", 125 | "\n", 126 | "```\n", 127 | "exec dbo.[StatsOptimize] @Tables='dbo.%,-FactBig', @StatisticsModificationLevel=0, @StatisticsSample=100 ,@OnlyModifiedStatistics=null,@DeleteOverlappingStats=null, @TimeLimit=null , @Execute=null \n", 128 | "\n", 129 | "```\n", 130 | "\n", 131 | "Update all Column Stats based on Columns DateKey or AccountKey\n", 132 | "\n", 133 | "```\n", 134 | "exec dbo.[StatsOptimize] @Tables='%.%.DateKey,%.%.AccountKey', @StatisticsModificationLevel=0, @StatisticsSample=null ,@OnlyModifiedStatistics=null,@DeleteOverlappingStats=null, @TimeLimit=null , @Execute=null \n", 135 | "\n", 136 | "```" 137 | ], 138 | "metadata": { 139 | "azdata_cell_guid": "545952d2-2b6c-40b2-91bc-7357e2c63004" 140 | }, 141 | "attachments": {} 142 | } 143 | ] 144 | } -------------------------------------------------------------------------------- /SqlPools/Maintenance/ColumnStoreOptimize.sql: -------------------------------------------------------------------------------- 1 | SET ANSI_NULLS ON 2 | GO 3 | SET QUOTED_IDENTIFIER ON 4 | GO 5 | IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[ColumnstoreOptimize]') AND type in (N'P')) 6 | EXEC dbo.sp_executesql @statement = N'CREATE PROC [dbo].[ColumnstoreOptimize] AS SELECT 1 as Dummy' 7 | GO 8 | /****** Object: StoredProcedure [dbo].[ColumnstoreOptimize] Script Date: 22/08/2021 15:20:01 ******/ 9 | SET ANSI_NULLS ON 10 | GO 11 | SET QUOTED_IDENTIFIER ON 12 | GO 13 | ALTER PROC [dbo].[ColumnstoreOptimize] @Tables [nvarchar](4000),@DensityThreshold [numeric](6,2),@OpenThreshold [int],@DeleteThreshold [int],@TimeLimit [int],@Execute [char](1) AS 14 | /* 15 | Description: Maintenance for Clustered Column Store Indexes. This accounts for three scenarios and will issue appropriate command 16 | - Fragmentation caused by deletes (REBUILD) 17 | - Fragmentation caused by open row stores (delta) aka inserts (REORGANIZE WITH (COMPRESS_ALL_ROW_GROUPS = ON) 18 | - Fragmentation caused byt low density in row stores (REORGANIZE) 19 | Example: exec [dbo].[ColumnstoreOptimize] null, 0,0,0,null,'N' /* Default Defrag and also remove open/deleted rows if > 60 RowGroups */ 20 | exec [dbo].[ColumnstoreOptimize] '%.Fact%',-1,-1,0,null,'N' /* Only remove deleted rows on fact Tables */ 21 | exec [dbo].[ColumnstoreOptimize] '%.FactFinance',0,null,null,'N' /* Force rebuild of ColumnStore on Fact Finance */ 22 | History: 23 | 10/08/2021 Bob, Created for Synapse SQL Maintenance 24 | 25 | */ 26 | BEGIN 27 | SET NOCOUNT ON; 28 | DECLARE @MinRowgroupCount int=120 /* Ignore Density in Columns stores with under 120 row groups, or 2 per distribution (about 130 million rows)*/ 29 | DECLARE @table_name sysname 30 | , @schema_name sysname 31 | , @object_id int 32 | , @SqlCommand nvarchar(4000) 33 | , @row_group_count int 34 | , @row_count bigint 35 | , @compressed_rowgroup_count bigint 36 | , @compressed_row_count bigint 37 | , @deleted_row_count bigint 38 | , @open_row_count int 39 | , @fragmentation_density numeric(10,2) 40 | , @fragmentation_open numeric(10,2) 41 | , @fragmentation_deletes numeric(10,2) 42 | , @partition_number int 43 | , @index_name sysname 44 | , @StartTime datetime 45 | , @db_name sysname =db_name() 46 | , @ID int 47 | , @LastID int 48 | , @object_name sysname 49 | , @ExtendedInfo nvarchar(4000) 50 | , @Error int 51 | , @ErrorMessage nvarchar(4000) 52 | , @Parameters nvarchar(max) 53 | , @ProcStartTime datetime 54 | , @StartMessage nvarchar(max) 55 | , @Duration int 56 | 57 | SELECT @DensityThreshold =coalesce( @DensityThreshold,25) 58 | , @OpenThreshold =coalesce(@OpenThreshold ,200000) 59 | , @DeleteThreshold =coalesce(@DeleteThreshold ,200000) 60 | , @Execute =coalesce(@Execute,'Y') 61 | 62 | 63 | SET @Parameters = '@Tables= ' + ISNULL('''' + REPLACE(@Tables,'''','''''') + '''','ALL') 64 | SET @Parameters += ', @DensityThreshold = ' + ISNULL(CAST( @DensityThreshold AS nvarchar),'NULL') 65 | SET @Parameters += ', @OpenThreshold = ' + ISNULL(CAST(@OpenThreshold AS nvarchar),'NULL') 66 | SET @Parameters += ', @DeleteThreshold = ' + ISNULL(CAST(@DeleteThreshold AS nvarchar),'NULL') 67 | SET @Parameters += ', @Execute = ' + ISNULL(CAST(@Execute AS nvarchar),'NULL') 68 | 69 | SET @StartMessage = 'Date and time: ' + CONVERT(nvarchar,getdate(),120) 70 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 71 | SET @StartMessage = 'Server: ' + CAST(SERVERPROPERTY('ServerName') AS nvarchar(max)) 72 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 73 | SET @StartMessage = 'Version: ' + CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(max)) 74 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 75 | SET @StartMessage = 'Edition: ' + CAST(SERVERPROPERTY('Edition') AS nvarchar(max)) 76 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 77 | SET @StartMessage = 'Parameters: ' + @Parameters 78 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 79 | SET @StartMessage = 'Version: ' + @@version 80 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 81 | SET @StartMessage = 'Source: https://github.com/ProdataSQL/SynapseTools/tree/main/SqlPools/Maintenance' 82 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 83 | 84 | 85 | IF OBJECT_ID('dbo.CommandLog') is null 86 | CREATE TABLE [dbo].[CommandLog]( 87 | [ID] [int] IDENTITY(1,1) NOT NULL, 88 | [DatabaseName] [sysname] NULL, 89 | [SchemaName] [sysname] NULL, 90 | [ObjectName] [sysname] NULL, 91 | [ObjectType] [char](2) NULL, 92 | [IndexName] [sysname] NULL, 93 | [IndexType] [tinyint] NULL, 94 | [StatisticsName] [sysname] NULL, 95 | [PartitionNumber] [int] NULL, 96 | [ExtendedInfo] [nvarchar](max) NULL, 97 | [Command] [nvarchar](max) NOT NULL, 98 | [CommandType] [nvarchar](60) NOT NULL, 99 | [StartTime] [datetime] NOT NULL, 100 | [EndTime] [datetime] NULL, 101 | [ErrorNumber] [int] NULL, 102 | [ErrorMessage] [nvarchar](max) NULL, 103 | CONSTRAINT [PK_CommandLog] PRIMARY KEY NONCLUSTERED ( [ID] ASC) NOT ENFORCED ) 104 | WITH ( DISTRIBUTION = ROUND_ROBIN,CLUSTERED INDEX (ID)) 105 | 106 | 107 | if OBJECT_ID('tempdb.dbo.#work_queue') is not null 108 | DROP TABLE #work_queue 109 | 110 | ;CREATE TABLE #work_queue 111 | WITH ( DISTRIBUTION = ROUND_ROBIN,CLUSTERED INDEX (table_name)) 112 | AS 113 | With Param1 as ( 114 | SELECT CASE WHEN left(ltrim(ss.value),1) IN ('+','-') THEN left(ltrim(ss.value),1) else '+' end as Op 115 | , CASE WHEN left(ltrim(ss.value),1) IN ('+','-') THEN substring(ltrim(ss.value),2,4000) else ltrim(ss.value) END as [Object] 116 | FROM string_split(@Tables,',') ss 117 | ), 118 | Param2 as ( 119 | SELECT ss.Op 120 | , case when charindex ('.', ss.Object) > 0 then left( ss.Object,charindex ('.', ss.Object) -1) else '%' END as [Schema] 121 | , case when charindex ('.', ss.Object) > 0 then substring( ss.Object,charindex ('.', ss.Object)+1, 4000) else case when ss.Object like 'ALL%' then '%' else ss.object END END as [Table] 122 | FROM Param1 ss 123 | ), ParamTables as ( 124 | select t.object_id, s.name as [Schema], t.name as [Table] 125 | FROM Param2 p 126 | CROSS JOIN sys.tables t 127 | INNER JOIN sys.schemas s on s.schema_id =t.schema_id 128 | WHERE t.type='U' and is_external=0 129 | and ( s.name like p.[schema] AND t.name like p.[Table] ) 130 | GROUP BY t.object_id , s.name, t.name 131 | having max(p.Op) ='+' 132 | ) 133 | SELECT ROW_NUMBER() OVER (ORDER BY cs.schema_name, cs.table_name, cs.partition_number) as ID , cs.object_id, quotename(cs.schema_name) + '.' + quotename(cs.table_name) as table_name, cs.schema_name, cs.table_name as object_name, cs.partition_number, '' 134 | + CASE WHEN cs.deleted_row_count > @DeleteThreshold and @DeleteThreshold >= 0 then 'REBUILD ' + coalesce(' PARTITION=' + convert(varchar,cs.partition_number) ,'') 135 | WHEN cs.open_row_count >= @OpenThreshold and @OpenThreshold > -1 then 'REORGANIZE' + coalesce(' PARTITION=' + convert(varchar,cs.partition_number),'') + ' WITH (COMPRESS_ALL_ROW_GROUPS = ON)' 136 | WHEN cs.[fragmentation_density] >= @DensityThreshold and cs.[compressed_rowgroup_count] > @MinRowgroupCount and @DensityThreshold > -1 then 'REORGANIZE' + coalesce(' PARTITION=' + convert(varchar,cs.partition_number),'') 137 | ELSE 'N/A' 138 | END as SqlCommand 139 | , cs.compressed_row_count 140 | , cs.compressed_rowgroup_count 141 | , cs.[row_group_count] 142 | , cs.[row_count] 143 | , cs.deleted_row_count 144 | , cs.open_row_count 145 | , cs.[fragmentation_density] 146 | , cs.[fragmentation_deletes] 147 | , cs.[fragmentation_open] 148 | , i.[name] as index_name 149 | FROM [dbo].[vColumnstoreStats] cs 150 | INNER JOIN sys.indexes i on i.object_id=cs.object_id and i.type=5 /* Ony CCS */ 151 | INNER JOIN ParamTables t on t.object_id=cs.object_id 152 | WHERE (cs.[fragmentation_density] > @DensityThreshold and cs.[compressed_rowgroup_count] > @MinRowgroupCount and @DensityThreshold >=0 ) 153 | OR (cs.deleted_row_count > @DeleteThreshold and @DeleteThreshold > -1) 154 | OR (cs.open_row_count >= @OpenThreshold and @OpenThreshold > -1) 155 | 156 | SELECT TOP 1 @ID=ID,@ProcStartTime =getdate(), @Duration =0 FROM #work_queue ORDER BY ID 157 | WHILE (@ID is not null) AND (@Duration < @TimeLimit OR @TimeLimit is null) 158 | BEGIN 159 | SET @object_id=null 160 | SELECT @LastID=q.ID, @object_id=q.[object_id], @schema_name =[schema_name], @table_name=table_name, @partition_number= partition_number, @sqlCommand =sqlCommand, @row_group_count=row_group_count, @row_count=row_count, @deleted_row_count=deleted_row_count, @open_row_count=open_row_count 161 | , @fragmentation_density = fragmentation_density, @fragmentation_open= fragmentation_open, @fragmentation_deletes=fragmentation_deletes, @index_name=index_name 162 | , @StartTime=getdate(), @compressed_row_count =compressed_row_count, @compressed_rowgroup_count=compressed_rowgroup_count, @object_name=[object_name] 163 | FROM #work_queue q 164 | WHERE ID = @ID 165 | 166 | SET @ExtendedInfo='' + convert(varchar,@compressed_rowgroup_count) + '' + convert(varchar,@row_count ) + '' + convert(varchar,@open_row_count ) + '' + convert(varchar,@deleted_row_count) + '' + convert( varchar,@fragmentation_density) +'%' 167 | SET @SqlCommand='ALTER INDEX ' + @index_name + ' ON '+ @table_name + ' ' + @SqlCommand 168 | 169 | SET @StartMessage = 'Date and time: ' + CONVERT(nvarchar,SYSDATETIME(),120) 170 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 171 | RAISERROR('%s',10,1,@table_name) WITH NOWAIT 172 | SET @StartMessage = 'SqlCommand: ' + @sqlCommand 173 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 174 | RAISERROR('%s',10,1,@ExtendedInfo) WITH NOWAIT 175 | 176 | IF @object_id is not null and @Execute='Y' 177 | BEGIN 178 | INSERT INTO dbo.CommandLog (DatabaseName, SchemaName, ObjectName, ObjectType, IndexName, IndexType, PartitionNumber, Command, CommandType, StartTime, ExtendedInfo) 179 | VALUES (@db_name, @schema_name, @object_name, 'U',@index_name, 5, @partition_number, @SqlCommand, 'ALTER INDEX', @StartTime, @ExtendedInfo) 180 | 181 | SELECT TOP 1 @ID=ID from dbo.CommandLog ORDER BY StartTime desc 182 | 183 | BEGIN TRY 184 | exec (@SqlCommand) 185 | END TRY 186 | BEGIN CATCH 187 | SET @Error = ERROR_NUMBER() 188 | SET @ErrorMessage = ERROR_MESSAGE() 189 | 190 | SET @ErrorMessage = 'Msg ' + CAST(ERROR_NUMBER() AS nvarchar) + ', ' + ISNULL(ERROR_MESSAGE(),'') 191 | UPDATE dbo.CommandLog SET [ErrorNumber]=@Error, ErrorMessage=@ErrorMessage 192 | WHERE ID=@ID 193 | RAISERROR ('%s',16,1,@ErrorMessage) WITH NOWAIT; 194 | END CATCH 195 | UPDATE dbo.CommandLog SET EndTime=getdate() WHERE ID=@ID 196 | END 197 | 198 | SET @ID=null 199 | SELECT TOP 1 @ID =ID FROM #work_queue WHERE ID > @LastID ORDER BY ID 200 | SET @Duration=datediff(second,@ProcStartTime, getdate()) 201 | 202 | END 203 | END; 204 | GO -------------------------------------------------------------------------------- /SqlPools/Maintenance/Readme.md: -------------------------------------------------------------------------------- 1 | # Maintenance Solution 2 | 3 | [](https://www.youtube.com/embed/G73WcVxPNmk) 4 | 5 | 6 | ## ColumnstoreOptimize 7 | ColumnstoreOptimize is stored procedure for maintaining columnstore indexes in SQL dedicated Pools. Key features include. 8 | - Automatically determine if REORGANISE or REBUILD requried based on density, open rows and deleted rows. 9 | - Automatically support closin open rowgroups with threshold. 10 | - Automatically support removing deleted rowgs from the delat store based on threshold. 11 | - [Ola Hallogren](https://ola.hallengren.com/) style features such as CommandLog, @Time Limit, and @Table parameer to set scope. 12 | 13 | All executed commands are logged to the [CommandLog](https://github.com/ProdataSQL/SynapseTools/blob/main/SqlPools/Maintenance/CommandLog.sql) table. 14 | 15 | It is based on Best practise guidance from MS sites below and community. 16 | - https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-tables-index 17 | - https://github.com/rgl/azure-content/blob/master/articles/sql-data-warehouse/sql-data-warehouse-manage-columnstore-indexes.md 18 | - https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-memory-optimizations-for-columnstore-compression 19 | - https://docs.microsoft.com/en-gb/archive/blogs/sqlserverstorageengine/columnstore-index-defragmentation-using-reorganize-command 20 | - https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/performance-tuning-ordered-cci 21 | - https://github.com/NikoNeugebauer/CISL/blob/master/SQL%20DW/alignment.sql 22 |
23 | 24 | This is an improved and productionised version of the sample view provided by Microsoft (vColumnstoreDensity]) [here](https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-tables-index) 25 | 26 | ### Usage: 27 | ```` 28 | exec [dbo].[ColumnstoreOptimize] , @Tables, @DensityThreshold, @OpenThreshold ,@DeleteThreshold, @TimeLimit , @Execute 29 | ```` 30 | 31 | 32 | #### Parameters 33 | 34 | ##### @Tables 35 | Select Tables to be included. The minus character is used to exclude objects and wildcards (%) are also supported as SQL Like clause. Use this to exclude more complex tables, exclude staging, or only include relevant schemas and objects 36 | |--|--| 37 | | Value | Description | 38 | |--|--| 39 | | Null | All Tables in Pool | 40 | | dbo.% | Tables in dbo schema | 41 | | %.Fact% | All Fact tables, regardless of Schema | 42 | | %.Fact%,-FactBig | All Fact tables, Except one called FactBig | 43 | 44 | #### @DensityThreshold 45 | Default 25%. This is the difference between the avg row size of a compressed row group and the ideal value (1024*1024). If the fragementation density in a target table excludes the specified value, then a REGORGANISE will be issued. 46 | 47 | Note: in some cases we may get churn where a table alwasy has a low density, in this case use the view vCS_rg_physical_stats to examine the TRIM reason and consider excluding table or using a higher threshold. 48 | 49 | Common TRIM reasons that prevent optimisation: 50 | * MEMORY_LIMITATION is a sign that the job needs to run with more memory. Eg a higher [resource class](https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/resource-classes-for-workload-management) such as xlargerc. 51 | * DICTIONARY_SIZE is a sign that a large high cardinality string is taking more than the 16MB allowed. In this case not much cab done without reducing cardinality, or moving columns to a different table. 52 | 53 | See [Maximising rowgroup quality for columnstore indexes in dedicated SQL pool](https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-memory-optimizations-for-columnstore-compression) 54 | 55 | 56 | #### @OpenThreshold 57 | Default 200k. Force a close of open row groups if the number of uncompressed Inserted rows that are in the delta store is > threshold. You can use the view vColumnStoreStats to see how many uncompressed rows are in each table. 58 | 59 | Note that the tuple mover will eventually close open rows, but its threshold is very high. 60 | 61 | #### @DeleteThreshold 62 | Default 200k. Force a REBUILD (expensive) if the number of deletes rows is greater than the threshold. 63 | 64 | Note that the tuple mover will rebuild compressed rowgroups and remove dleeted rows when more than 10% of rows are deleted, but we may want to do earlier and more pro-active maintenance. 65 | 66 | #### @TimeLimit 67 | Default Null or infinite. Set a time limit in seconds for the job to run. No more commands will be started after time limit (but existing ones will finish). Use this if you have a short maintenace windows and do not want to exceed time. 68 | 69 | #### @Execute 70 | Default Y. Set to N to only show commands but not execute or log them. Useful for seeing experimental maintenance commands before actually executing them. 71 | 72 | ### Example Usage 73 | Optimise FactFinance table if more than 30% fragmentation. Do not close open row groups or remove deleted rows. Onlt show commnds, do not execute. 74 | ``` 75 | exec [dbo].[ColumnstoreOptimize] @Tables= '%.FactFinance',@DensityThreshold=30, @OpenThreshold=-1,@DeleteThreshold=-1,@TimeLimit=null, @Execute ='N' 76 | ``` 77 | 78 | Do default maintenance on all fact tables except FactBigNasty. Only run for 1 hour 79 | ``` 80 | exec [dbo].[ColumnstoreOptimize] @Tables= 'ALL,-FactBigNasty',@DensityThreshold=null, @OpenThreshold=null,@DeleteThreshold=null,@TimeLimit=3600, @Execute =null 81 | ``` 82 | Do default maintenance only in dbo schema. 83 | ``` 84 | exec [dbo].[ColumnstoreOptimize] @Tables= 'dbo.%',@DensityThreshold=null, @OpenThreshold=null,@DeleteThreshold=null,@TimeLimit=3600, @Execute =null 85 | ``` 86 | Close any open rowgroups in all tables if more than 100k rows. Do not deal with deleted rows or look at density. 87 | ``` 88 | exec [dbo].[ColumnstoreOptimize] @Tables= '%.%',@DensityThreshold=-1, @OpenThreshold=100000,@DeleteThreshold=-1,@TimeLimit=null, @Execute =null 89 | ``` 90 | 91 |
92 | 93 | ## StatsOptimize 94 | StatsOptimize is stored procedure for updating Statistics for Synapse SQL Pools. This can be critical for SQL Pools as while they support auto create of stats they do not support auto updte of statistics. This procedure is like a replacement for auto update and allows for more control and flexibility. 95 | 96 | Key features include: 97 | - Dynamic determine modification level based on imporved algorithm. 98 | - Dynamically determine sampling level or support setting sample level. 99 | - Support removal of duplicate statistics (covering same column). 100 | - [Ola Hallogren](https://ola.hallengren.com/) style features such as CommandLog, @Time Limit, and @Table parameer to set scope. 101 | 102 | All executed commands are logged to the [CommandLog](https://github.com/ProdataSQL/SynapseTools/blob/main/SqlPools/Maintenance/CommandLog.sql) table. 103 | 104 | It is based on Best practise guidance from MS sites below and community. 105 | - https://docs.microsoft.com/en-us/azure/synapse-analytics/sql/develop-tables-statistics 106 | - https://github.com/abrahams1/Azure_Synapse_Toolbox/tree/master/SQL_Queries/Statistics 107 | - https://www.sqlskills.com/blogs/tim/when-updating-statistics-is-too-expensive/ 108 | - https://www.sqlskills.com/blogs/erin/updating-statistics-with-ola-hallengrens-script/ 109 | - https://docs.microsoft.com/en-us/sql/relational-databases/statistics/statistics?view=sql-server-2017 110 | - https://ola.hallengren.com/ 111 | 112 | 113 | ### Usage: 114 | ```` 115 | exec [dbo].[StatsOptimize] , @Tables, @StatisticsModificationLevel, @StatisticsSample ,@OnlyModifiedStatistics, @DeleteOverlappingStats, @TimeLimit , @Execute 116 | ```` 117 | #### Parameters 118 | 119 | ##### @Tables 120 | Select Tables nd optionally columns to be included. The minus character is used to exclude objects and wildcards (%) are also supported as SQL Like clause. Use this to exclude more complex tables, exclude staging, or only include relevant schemas and objects 121 | |--|--| 122 | | Value | Description | 123 | |--|--| 124 | | Null | All Tables in Pool | 125 | | dbo.% | Tables in dbo schema | 126 | | %.Fact% | All Fact tables (prefix of Fact), regardless of Schema | 127 | | %.Fact%,-FactBig | All Fact tables, Except one called FactBig | 128 | | %.%.Date,%.%.AccountKey | ONLY do stats maintenance on Date and AccountKey columns. 129 | 130 | Note that usually we just do stats maintenance at the table level, but there is also support for specifing column(s). This is a special case to support columns that need frequent updates like low cardinatlity incremental values (Eg Business Date) which is recommended by Microsoft [here](https://docs.microsoft.com/en-us/azure/synapse-analytics/sql/develop-tables-statistics) 131 | 132 | #### @StatisticsModificationLevel 133 | By default the SProc will only update a stat if the number of modified rows is greater than either 134 | - 20% or specified value in parameter 135 | - an Adaptive algorithm of SQRT(1000 * [row count)] based on improved stats algorithm introduced in SQL 2014 136 | 137 | The SProc uses the view vTableStats to return meta data such as the nuber of rows per table and the number of rows in the statistcis. The main difference between SQL Pools and a traditional SQL Server is that in a SQL Pool we can only track the estimated row count at the the table/partition level and not per statistic. Eg we do not have [sys.dm_db_stats_properties](https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-stats-properties-transact-sql?view=sql-server-ver15) but we do have [pdw_table_distribution_properties](https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-pdw-table-distribution-properties-transact-sql?view=aps-pdw-2016-au7) with per stat data. 138 | 139 | This ommission is largely due to the fact that statistics are per distribution, so it is much more complex to amalgamate and sychronise the set of 60 stat objects. This is also why we have no auto stats for SQL Pools (as of 31/08/2021). 140 | 141 | 142 | #### @StatisticsSample 143 | This is the sample rate. if this is null, then the SProc uses an adaptive sample rate 144 | - sqrt([row count]) *1000/ [row count]*100 145 | 146 | While syanpse uses a default of 20% for the sample rate, there is a recommendation to use about 2% when we reach a billion rows. The adaptive sample rate generates recommendations such as: 147 | 148 | 149 | | Row Count | Sample % | 150 | |--|--| 151 | | 1000 | FULLSCAN | 152 | | 100k | FULLSCAN | 153 | | 1 million | FULLSCAN | 154 | | 10 million | 31 | 155 | | 100 million | 10 | 156 | | 1 billion | 3 | 157 | | 10 billion+ | 1 | 158 | 159 | #### @OnlyModifiedStatistics 160 | Default Y. Set this to N to 161 | 162 | #### @DeleteOverlappingStats 163 | Default N. Set to Y to delete any auto stats which overlap an existing statistic. 164 | 165 | #### @TimeLimit 166 | Default Null or infinite. Set a time limit in seconds for the job to run. No more commands will be started after time limit (but existing ones will finish). Use this if you have a short maintenace windows and do not want to exceed time. 167 | 168 | #### @Execute 169 | Default Y. Set to N to only show commands but not execute or log them. Useful for seeing experimental maintenance commands before actually executing them. 170 | 171 | ### Example Usage 172 | Default Best Practise with smart defaults. 173 | ``` 174 | exec dbo.[StatsOptimize] @Tables=null, @StatisticsModificationLevel=null, @StatisticsSample=null ,@OnlyModifiedStatistics=null,@DeleteOverlappingStats=null, @TimeLimit=null , @Execute=null 175 | ``` 176 | 177 | Update stats in dbo schema except FactBig and use a FULLSCAN if any rows have changed 178 | ``` 179 | exec dbo.[StatsOptimize] @Tables='dbo.%,-FactBig', @StatisticsModificationLevel=0, @StatisticsSample=100 ,@OnlyModifiedStatistics=null,@DeleteOverlappingStats=null, @TimeLimit=null , @Execute=null 180 | ``` 181 | 182 | Update all Column Stats based on Columns DateKey or AccountKey 183 | ``` 184 | exec dbo.[StatsOptimize] @Tables='%.%.DateKey,%.%.AccountKey', @StatisticsModificationLevel=0, @StatisticsSample=null ,@OnlyModifiedStatistics=null,@DeleteOverlappingStats=null, @TimeLimit=null , @Execute=null 185 | ``` 186 | 187 | 188 | 189 | -------------------------------------------------------------------------------- /SqlPools/Demos/01-02. How do we fix Density.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "kernelspec": { 4 | "name": "SQL", 5 | "display_name": "SQL", 6 | "language": "sql" 7 | }, 8 | "language_info": { 9 | "name": "sql", 10 | "version": "" 11 | }, 12 | "extensions": { 13 | "azuredatastudio": { 14 | "version": 1, 15 | "views": [] 16 | } 17 | } 18 | }, 19 | "nbformat_minor": 2, 20 | "nbformat": 4, 21 | "cells": [ 22 | { 23 | "cell_type": "markdown", 24 | "source": [ 25 | "## How do we Fix Density Fragmentation\n", 26 | "\n", 27 | "We could fix this with INDEX REORG \n", 28 | "\n", 29 | "REORG ALTER INDEX \\[ClusteredIndex\\_1fba0db5c48b40288124497ec2198389\\] ON \\[dbo\\].\\[FactFinance100m\\] REORGANIZE\n", 30 | "\n", 31 | "But, lets introduce the GitHub Library function ColumnStoreOptimize which can locate all Column Stores with low density and REORG them [https://github.com/ProdataSQL/SynapseTools/tree/main/SqlPools/Maintenance](https://github.com/ProdataSQL/SynapseTools/tree/main/SqlPools/Maintenance)" 32 | ], 33 | "metadata": { 34 | "azdata_cell_guid": "b2704311-c14c-4863-a619-f4dad5b90778" 35 | }, 36 | "attachments": {} 37 | }, 38 | { 39 | "cell_type": "code", 40 | "source": [ 41 | "/* ALTER INDEX ClusteredIndex_1fba0db5c48b40288124497ec2198389 ON [dbo].[FactFinance100m] REORGANIZE */\r\n", 42 | "exec [dbo].[ColumnstoreOptimize] @Tables='FactFinance100m'\r\n", 43 | ",@DensityThreshold=25 /* Default=25 */\r\n", 44 | ",@OpenThreshold=null\r\n", 45 | ",@DeleteThreshold=null\r\n", 46 | ",@TimeLimit =null\r\n", 47 | ", @Execute='Y'\r\n", 48 | "" 49 | ], 50 | "metadata": { 51 | "azdata_cell_guid": "8fab48bc-e202-46d5-9f7e-087a780f9fdd", 52 | "extensions": { 53 | "azuredatastudio": { 54 | "views": [] 55 | } 56 | }, 57 | "tags": [], 58 | "language": "sql" 59 | }, 60 | "outputs": [ 61 | { 62 | "output_type": "display_data", 63 | "data": { 64 | "text/html": "Date and time: 2022-07-08 12:59:40" 65 | }, 66 | "metadata": {} 67 | }, 68 | { 69 | "output_type": "display_data", 70 | "data": { 71 | "text/html": "Server: aw-dev" 72 | }, 73 | "metadata": {} 74 | }, 75 | { 76 | "output_type": "display_data", 77 | "data": { 78 | "text/html": "Version: 10.0.15665.0" 79 | }, 80 | "metadata": {} 81 | }, 82 | { 83 | "output_type": "display_data", 84 | "data": { 85 | "text/html": "Edition: SQL Azure" 86 | }, 87 | "metadata": {} 88 | }, 89 | { 90 | "output_type": "display_data", 91 | "data": { 92 | "text/html": "Parameters: @Tables= 'FactFinance100m', @DensityThreshold = 25.00, @OpenThreshold = 200000, @DeleteThreshold = 200000, @Execute = Y" 93 | }, 94 | "metadata": {} 95 | }, 96 | { 97 | "output_type": "display_data", 98 | "data": { 99 | "text/html": "Version: Microsoft Azure SQL Data Warehouse - 10.0.15665.0 Jul 7 2022 07:03:56 Copyright (c) Microsoft Corporation" 100 | }, 101 | "metadata": {} 102 | }, 103 | { 104 | "output_type": "display_data", 105 | "data": { 106 | "text/html": "Source: https://github.com/ProdataSQL/SynapseTools/tree/main/SqlPools/Maintenance" 107 | }, 108 | "metadata": {} 109 | }, 110 | { 111 | "output_type": "display_data", 112 | "data": { 113 | "text/html": "Date and time: 2022-07-08 12:59:42" 114 | }, 115 | "metadata": {} 116 | }, 117 | { 118 | "output_type": "display_data", 119 | "data": { 120 | "text/html": "SqlCommand: ALTER INDEX ClusteredIndex_766ace22025346dba8b1db2efab6ee27 ON [dbo].[FactFinance100m] REORGANIZE" 121 | }, 122 | "metadata": {} 123 | }, 124 | { 125 | "output_type": "display_data", 126 | "data": { 127 | "text/html": "Total execution time: 00:00:13.103" 128 | }, 129 | "metadata": {} 130 | } 131 | ], 132 | "execution_count": 3 133 | }, 134 | { 135 | "cell_type": "code", 136 | "source": [ 137 | "/* Now we can check commandlog and also imprves Stats\r\n", 138 | " From 180 row groups to just 60 (one per distibution)\r\n", 139 | " */\r\n", 140 | "SELECT TOP 1 * FROM dbo.CommandLog ORDER BY StartTime DESC\r\n", 141 | "SELECT * From vColumnstoreStats where table_name ='factFinance100m'\r\n", 142 | "" 143 | ], 144 | "metadata": { 145 | "azdata_cell_guid": "dbcf391b-fcd1-4e63-8284-b8433fd7ed47", 146 | "extensions": { 147 | "azuredatastudio": { 148 | "views": [] 149 | } 150 | }, 151 | "language": "sql" 152 | }, 153 | "outputs": [ 154 | { 155 | "output_type": "display_data", 156 | "data": { 157 | "text/html": "(1 row affected)" 158 | }, 159 | "metadata": {} 160 | }, 161 | { 162 | "output_type": "display_data", 163 | "data": { 164 | "text/html": "(1 row affected)" 165 | }, 166 | "metadata": {} 167 | }, 168 | { 169 | "output_type": "display_data", 170 | "data": { 171 | "text/html": "Total execution time: 00:00:01.439" 172 | }, 173 | "metadata": {} 174 | }, 175 | { 176 | "output_type": "execute_result", 177 | "execution_count": 4, 178 | "data": { 179 | "application/vnd.dataresource+json": { 180 | "schema": { 181 | "fields": [ 182 | { 183 | "name": "ID" 184 | }, 185 | { 186 | "name": "DatabaseName" 187 | }, 188 | { 189 | "name": "SchemaName" 190 | }, 191 | { 192 | "name": "ObjectName" 193 | }, 194 | { 195 | "name": "ObjectType" 196 | }, 197 | { 198 | "name": "IndexName" 199 | }, 200 | { 201 | "name": "IndexType" 202 | }, 203 | { 204 | "name": "StatisticsName" 205 | }, 206 | { 207 | "name": "PartitionNumber" 208 | }, 209 | { 210 | "name": "ExtendedInfo" 211 | }, 212 | { 213 | "name": "Command" 214 | }, 215 | { 216 | "name": "CommandType" 217 | }, 218 | { 219 | "name": "StartTime" 220 | }, 221 | { 222 | "name": "EndTime" 223 | }, 224 | { 225 | "name": "ErrorNumber" 226 | }, 227 | { 228 | "name": "ErrorMessage" 229 | } 230 | ] 231 | }, 232 | "data": [ 233 | { 234 | "0": "86", 235 | "1": "AdventureWorksDW", 236 | "2": "dbo", 237 | "3": "FactFinance100m", 238 | "4": "U ", 239 | "5": "ClusteredIndex_e2f1410ade274620bfb52e91dcee6ddb", 240 | "6": "5", 241 | "7": "NULL", 242 | "8": "NULL", 243 | "9": "4202300000004170880048.72 %", 244 | "10": "ALTER INDEX ClusteredIndex_e2f1410ade274620bfb52e91dcee6ddb ON [dbo].[FactFinance100m] REORGANIZE WITH (COMPRESS_ALL_ROW_GROUPS = ON)", 245 | "11": "ALTER INDEX", 246 | "12": "2022-07-08 14:33:59.787", 247 | "13": "2022-07-08 14:34:07.160", 248 | "14": "NULL", 249 | "15": "NULL" 250 | } 251 | ] 252 | }, 253 | "text/html": "
IDDatabaseNameSchemaNameObjectNameObjectTypeIndexNameIndexTypeStatisticsNamePartitionNumberExtendedInfoCommandCommandTypeStartTimeEndTimeErrorNumberErrorMessage
86AdventureWorksDWdboFactFinance100mU ClusteredIndex_e2f1410ade274620bfb52e91dcee6ddb5NULLNULL<ExtendedInfo><RowGroups>420</RowGroups><Rows>230000000</Rows><OpenRows>4170880</OpenRows><DeletedRows>0</DeletedRows><DensityFragmentation>48.72 %</DensityFragmentation></ExtendedInfo>ALTER INDEX ClusteredIndex_e2f1410ade274620bfb52e91dcee6ddb ON [dbo].[FactFinance100m] REORGANIZE WITH (COMPRESS_ALL_ROW_GROUPS = ON)ALTER INDEX2022-07-08 14:33:59.7872022-07-08 14:34:07.160NULLNULL
" 254 | }, 255 | "metadata": {} 256 | }, 257 | { 258 | "output_type": "execute_result", 259 | "execution_count": 4, 260 | "data": { 261 | "application/vnd.dataresource+json": { 262 | "schema": { 263 | "fields": [ 264 | { 265 | "name": "execution_date" 266 | }, 267 | { 268 | "name": "database_name" 269 | }, 270 | { 271 | "name": "schema_name" 272 | }, 273 | { 274 | "name": "table_name" 275 | }, 276 | { 277 | "name": "partition_number" 278 | }, 279 | { 280 | "name": "partition_scheme" 281 | }, 282 | { 283 | "name": "object_id" 284 | }, 285 | { 286 | "name": "index_name" 287 | }, 288 | { 289 | "name": "row_count" 290 | }, 291 | { 292 | "name": "deleted_row_count" 293 | }, 294 | { 295 | "name": "row_group_count" 296 | }, 297 | { 298 | "name": "compressed_row_count" 299 | }, 300 | { 301 | "name": "compressed_rowgroup_count" 302 | }, 303 | { 304 | "name": "open_rowgroup_count" 305 | }, 306 | { 307 | "name": "open_row_count" 308 | }, 309 | { 310 | "name": "compressed_row_max" 311 | }, 312 | { 313 | "name": "compressed_row_avg" 314 | }, 315 | { 316 | "name": "fragmentation_density" 317 | }, 318 | { 319 | "name": "fragmentation_deletes" 320 | }, 321 | { 322 | "name": "fragmentation_open" 323 | } 324 | ] 325 | }, 326 | "data": [ 327 | { 328 | "0": "2022-07-08 14:35:45.453", 329 | "1": "AdventureWorksDW", 330 | "2": "dbo", 331 | "3": "FactFinance100m", 332 | "4": "NULL", 333 | "5": "NULL", 334 | "6": "290868153", 335 | "7": "ClusteredIndex_e2f1410ade274620bfb52e91dcee6ddb", 336 | "8": "230000000", 337 | "9": "0", 338 | "10": "300", 339 | "11": "230000000", 340 | "12": "300", 341 | "13": "0", 342 | "14": "0", 343 | "15": "1048576", 344 | "16": "766666", 345 | "17": "26.8900", 346 | "18": "0.0000", 347 | "19": "0.0000" 348 | } 349 | ] 350 | }, 351 | "text/html": "
execution_datedatabase_nameschema_nametable_namepartition_numberpartition_schemeobject_idindex_namerow_countdeleted_row_countrow_group_countcompressed_row_countcompressed_rowgroup_countopen_rowgroup_countopen_row_countcompressed_row_maxcompressed_row_avgfragmentation_densityfragmentation_deletesfragmentation_open
2022-07-08 14:35:45.453AdventureWorksDWdboFactFinance100mNULLNULL290868153ClusteredIndex_e2f1410ade274620bfb52e91dcee6ddb230000000030023000000030000104857676666626.89000.00000.0000
" 352 | }, 353 | "metadata": {} 354 | } 355 | ], 356 | "execution_count": 4 357 | } 358 | ] 359 | } -------------------------------------------------------------------------------- /SqlPools/Maintenance/StatsOptimize.sql: -------------------------------------------------------------------------------- 1 | SET ANSI_NULLS ON 2 | GO 3 | SET QUOTED_IDENTIFIER ON 4 | GO 5 | IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[StatsOptimize]') AND type in (N'P')) 6 | EXEC dbo.sp_executesql @statement = N'CREATE PROC [dbo].[StatsOptimize] AS SELECT 1 as Dummy' 7 | GO 8 | ALTER PROC [dbo].[StatsOptimize] @Tables [varchar](4000),@StatisticsModificationLevel [int],@StatisticsSample [int],@OnlyModifiedStatistics [char](1),@DeleteOverlappingStats [char](1),@TimeLimit [int],@Execute [char](1) AS 9 | BEGIN 10 | /* 11 | description: Stats Maintenance for Synapse SQL Pools 12 | Concepts crteated from from 13 | - ola halogren scripts https://ola.hallengren.com/ 14 | - https://github.com/abrahams1/Azure_Synapse_Toolbox/tree/master/SQL_Queries/Statistics 15 | - https://www.sqlskills.com/blogs/tim/when-updating-statistics-is-too-expensive/ 16 | - https://www.sqlskills.com/blogs/erin/updating-statistics-with-ola-hallengrens-script/ 17 | - https://docs.microsoft.com/en-us/sql/relational-databases/statistics/statistics?view=sql-server-2017 Why SQRT (rows*1000) 18 | Example: 19 | --Default Best Practise with smart defaults 20 | exec dbo.[StatsOptimize] null,null,null,null,null,null,null 21 | 22 | --Exclude 1 Table and use a 5% change threshold (plus auto at SQRT algorithm). Dont add Missing Stats or Delete Overlapping Stats 23 | exec [StatsOptimize] @Tables='ALL,-FactFinance', @StatisticsModificationLevel=5, @StatisticsSample=null,@OnlyModifiedStatistics ='Y' ,@DeleteOverlappingStats ='N', @TimeLimit=null,@Execute='N' 24 | 25 | --Stats Updates with FULL Sample (no missing or de-dupe), only if > 10% difference. Stop after 1 hour 26 | exec [StatsOptimize] null,10,100,'Y','N',3600,'N' 27 | 28 | --Stats Updates just in dbo Schema 29 | exec [StatsOptimize] @Tables='dbo.%' ,@StatisticsModificationLevel =null, @StatisticsSample=null, @OnlyModifiedStatistics=null,@DeleteOverlappingStats=null,@TimeLimit=null,@Execute='N' 30 | 31 | --Only Delete Overlapping Statistics 32 | exec dbo.[StatsOptimize] null,null,null,'N','Y',null,'N' 33 | 34 | --Force Update of All Stats in STG Schema 35 | exec dbo.[StatsOptimize] 'stg.%',0,null,null,null,null,null 36 | 37 | History: 12/08/2021 Bob, Created 38 | 31/08/2021 Bob, Added Adaptive SamplingRate. Fixed bug in removing duplicate stats (null stats name) 39 | */ 40 | 41 | --Default Parameters (Synapse cant do default parameters at declaration) 42 | SET NOCOUNT ON 43 | SET ARITHABORT ON 44 | SET NUMERIC_ROUNDABORT OFF 45 | SELECT @Tables =coalesce(@Tables,'ALL') 46 | , @OnlyModifiedStatistics =coalesce(@OnlyModifiedStatistics,'Y') 47 | , @StatisticsModificationLevel=coalesce(@StatisticsModificationLevel,null) --Default ius to sue new SQRT algorithm 48 | , @StatisticsSample =coalesce(@StatisticsSample,null) 49 | , @DeleteOverlappingStats =coalesce(@DeleteOverlappingStats,'N') 50 | , @Execute=coalesce(@Execute,'Y') 51 | 52 | 53 | 54 | DECLARE @StartMessage nvarchar(max) 55 | DECLARE @EndMessage nvarchar(max) 56 | DECLARE @DatabaseMessage nvarchar(max) 57 | DECLARE @ErrorMessage nvarchar(max) 58 | DECLARE @Severity int 59 | DECLARE @Parameters nvarchar(max) 60 | DECLARE @EmptyLine nvarchar(max) = CHAR(9) 61 | DECLARE @CurrentCommand nvarchar(max) 62 | DECLARE @CurrentMessage nvarchar(max) 63 | DECLARE @CurrentSeverity int 64 | DECLARE @CurrentState int 65 | DECLARE @Version numeric(18,10) = CAST(LEFT(CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(max)),CHARINDEX('.',CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(max))) - 1) + '.' + REPLACE(RIGHT(CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(max)), LEN(CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(max))) - CHARINDEX('.',CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(max)))),'.','') AS numeric(18,10)) 66 | DECLARE @ID int 67 | , @LastID int 68 | , @object_id int 69 | , @schema_name sysname 70 | , @table_name sysname 71 | , @stats_row_count bigint 72 | , @actual_row_count bigint 73 | , @stats_difference_percent numeric (10,4) 74 | , @sqlCommand nvarchar(4000) 75 | , @UpdateLevel bigint 76 | , @ExtendedInfo nvarchar(4000) 77 | , @DatabaseName sysname =db_name() 78 | , @StartTime datetime 79 | , @EndTime datetime 80 | , @LogID int 81 | , @Error int 82 | , @ProcStartTime datetime 83 | , @Duration int 84 | , @StatisticsName sysname 85 | 86 | IF OBJECT_ID('tempdb.dbo.#Errors') is not null DROP TABLE #Errors 87 | CREATE TABLE #Errors (ID int IDENTITY, 88 | [Message] nvarchar(max) NOT NULL, 89 | Severity int NOT NULL, 90 | [State] int) WITH (DISTRIBUTION=ROUND_ROBIN, HEAP) 91 | 92 | IF OBJECT_ID('tempdb.dbo.#rsStats') is not null DROP TABLE #rsStats 93 | CREATE TABLE #rsStats (ID int IDENTITY 94 | , object_id int not null 95 | , schema_name sysname not null 96 | , table_name sysname not null 97 | , stats_name sysname not null 98 | , stats_row_count bigint null 99 | , actual_row_count bigint null 100 | , stats_difference_percent numeric (10,2) null 101 | , SqlCommand nvarchar(4000) not null 102 | , UpdateLevel bigint 103 | , ExtendedInfo nvarchar(4000) not null) WITH (DISTRIBUTION=ROUND_ROBIN, HEAP) 104 | 105 | 106 | ---------------------------------------------------------------------------------------------------- 107 | --// Log initial information //-- 108 | ---------------------------------------------------------------------------------------------------- 109 | SET @Parameters = '@Tables= ' + ISNULL('''' + REPLACE(@Tables,'''','''''') + '''','ALL') 110 | SET @Parameters += ', @OnlyModifiedStatistics = ' + ISNULL('''' + REPLACE(@OnlyModifiedStatistics,'''','''''') + '''','NULL') 111 | SET @Parameters += ', @StatisticsModificationLevel = ' + ISNULL(CAST(@StatisticsModificationLevel AS nvarchar),'NULL') 112 | SET @Parameters += ', @StatisticsSample = ' + ISNULL(CAST(@StatisticsSample AS nvarchar),'NULL') 113 | SET @Parameters += ', @DeleteOverlappingStats = ' + ISNULL(CAST(@DeleteOverlappingStats AS nvarchar),'NULL') 114 | SET @Parameters += ', @TimeLimit = ' + ISNULL(CAST(@TimeLimit AS nvarchar),'NULL') 115 | SET @Parameters += ', @Execute = ' + ISNULL(CAST(@Execute AS nvarchar),'NULL') 116 | 117 | SET @StartMessage = 'Date and time: ' + CONVERT(nvarchar,getdate(),120) 118 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 119 | 120 | SET @StartMessage = 'Server: ' + CAST(SERVERPROPERTY('ServerName') AS nvarchar(max)) 121 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 122 | 123 | SET @StartMessage = 'Version: ' + CAST(SERVERPROPERTY('ProductVersion') AS nvarchar(max)) 124 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 125 | 126 | SET @StartMessage = 'Edition: ' + CAST(SERVERPROPERTY('Edition') AS nvarchar(max)) 127 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 128 | 129 | SET @StartMessage = 'Parameters: ' + @Parameters 130 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 131 | 132 | SET @StartMessage = 'Version: ' + @@version 133 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 134 | 135 | SET @StartMessage = 'Source: https://github.com/ProdataSQL/SynapseTools/tree/main/SqlPools/Maintenance' 136 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 137 | 138 | RAISERROR(@EmptyLine,10,1) WITH NOWAIT 139 | 140 | 141 | IF object_id ('dbo.vTableStats','V') is null 142 | BEGIN 143 | INSERT INTO #Errors ([Message], Severity, [State]) 144 | SELECT 'The View [vTableStats is required. Please download/install this.', 16, 1 145 | END 146 | ---------------------------------------------------------------------------------------------------- 147 | IF @OnlyModifiedStatistics NOT IN('Y','N') OR @OnlyModifiedStatistics IS NULL 148 | BEGIN 149 | INSERT INTO #Errors ([Message], Severity, [State]) 150 | SELECT 'The value for the parameter @OnlyModifiedStatistics is not supported.', 16, 1 151 | END 152 | 153 | ---------------------------------------------------------------------------------------------------- 154 | 155 | IF @StatisticsModificationLevel < 0 OR @StatisticsModificationLevel > 100 156 | BEGIN 157 | INSERT INTO #Errors ([Message], Severity, [State]) 158 | SELECT 'The value for the parameter @StatisticsModificationLevel is not supported.', 16, 1 159 | END 160 | 161 | ---------------------------------------------------------------------------------------------------- 162 | 163 | IF @StatisticsSample <= 0 OR @StatisticsSample > 100 164 | BEGIN 165 | INSERT INTO #Errors ([Message], Severity, [State]) 166 | SELECT 'The value for the parameter @StatisticsSample is not supported.', 16, 1 167 | END 168 | 169 | IF @TimeLimit < 0 170 | BEGIN 171 | INSERT INTO #Errors ([Message], Severity, [State]) 172 | SELECT 'The value for the parameter @TimeLimit is not supported.', 16, 1 173 | END 174 | 175 | ---------------------------------------------------------------------------------------------------- 176 | --// Raise errors //-- 177 | ---------------------------------------------------------------------------------------------------- 178 | SELECT TOP 1 @CurrentMessage =Message, @CurrentSeverity=Severity, @CurrentState=State 179 | FROM #Errors 180 | 181 | IF @CurrentMessage is not null 182 | BEGIN 183 | SELECT * FROM #Errors 184 | RAISERROR('%s', @CurrentSeverity, @CurrentState, @CurrentMessage) WITH NOWAIT 185 | RAISERROR(@EmptyLine, 10, 1) WITH NOWAIT 186 | END 187 | 188 | 189 | /* Select Objects for Inclusion in Scan based on "Table" level Stats */ 190 | ;With Param1 as ( 191 | SELECT CASE WHEN left(ltrim(ss.value),1) IN ('+','-') THEN left(ltrim(ss.value),1) else '+' end as Op 192 | , CASE WHEN left(ltrim(ss.value),1) IN ('+','-') THEN substring(ltrim(ss.value),2,4000) else ltrim(ss.value) END as [Object] 193 | FROM string_split(@Tables,',') ss 194 | ), 195 | Param2 as ( 196 | SELECT ss.Op 197 | , case when charindex ('.', ss.Object) > 0 then left( ss.Object,charindex ('.', ss.Object) -1) else '%' END as [Schema] 198 | , case when charindex ('.', ss.Object) > 0 then substring( ss.Object,charindex ('.', ss.Object)+1, 4000) else case when ss.Object like 'ALL%' then '%' else ss.object END END as [Object] 199 | FROM Param1 ss 200 | ), Param3 as ( 201 | SELECT ss.Op, ss.[Schema] 202 | , case when charindex ('.', ss.Object) > 0 then left( ss.Object,charindex ('.', ss.Object)-1) else case when ss.Object like 'ALL%' then '%' else ss.object END END as [Table] 203 | , case when charindex ('.', ss.Object) > 0 then substring( ss.Object,charindex ('.', ss.Object)+1, 4000) else null END as [Column] 204 | FROM Param2 ss 205 | ) 206 | , ParamTables as ( 207 | select t.object_id, s.name as [Schema], t.name as [Table] 208 | FROM Param3 p 209 | CROSS JOIN sys.tables t 210 | INNER JOIN sys.schemas s on s.schema_id =t.schema_id 211 | WHERE t.type='U' and is_external=0 212 | and ( s.name like p.[schema] AND t.name like p.[Table] ) 213 | AND (p.[Column] is null or p.[Column]='%') 214 | GROUP BY t.object_id , s.name, t.name, p.[Column] 215 | having max(p.Op) ='+' 216 | ), ParamColumns as ( 217 | select t.object_id, s.name as [Schema], t.name as [Table], c.name as [Column] 218 | FROM Param3 p 219 | CROSS JOIN sys.tables t 220 | INNER JOIN sys.schemas s on s.schema_id =t.schema_id 221 | INNER JOIN sys.columns c on c.object_id=t.object_id 222 | WHERE t.type='U' and is_external=0 223 | and ( s.name like p.[schema] AND t.name like p.[Table] ) 224 | AND (p.[Column] <> '%' and p.[Column] is not null) 225 | AND c.name like p.[Column] 226 | GROUP BY t.object_id , s.name, t.name, c.name 227 | having max(p.Op) ='+' 228 | ) 229 | INSERT INTO #rsStats (object_id, schema_name, table_name, stats_name, stats_row_count, actual_row_count, stats_difference_percent, SqlCommand, UpdateLevel, ExtendedInfo) 230 | /* Table Level Stats Management (Recommended) */ 231 | SELECT * FROM ( 232 | SELECT TOP 1000000 s.[object_id], s.[schema_name], s.[table_name], 'ALL' as stats_name, s.[stats_row_count], s.[actual_row_count], s.[stats_difference_percent] 233 | , 'UPDATE STATISTICS ' + quotename(s.[schema_name]) + '.' + quotename(s.[table_name]) 234 | + coalesce(case when coalesce(@StatisticsSample , s.[stats_sample_rate]) =100 THEN ' WITH FULLSCAN' ELSE ' WITH SAMPLE ' + convert(varchar,coalesce(@StatisticsSample , s.[stats_sample_rate])) + ' PERCENT' END,'') 235 | as SqlCommand 236 | , convert(bigint,SQRT(1000 * s.[actual_row_count])) as UpdateLevel 237 | ,'' + convert (varchar, s.stats_row_count) + '' + convert (varchar, s.actual_row_count) + '' + convert (varchar,convert(bigint,SQRT(1000 * s.[actual_row_count]))) + '' as ExtendedInfo 238 | FROM dbo.vTableStats s 239 | INNER JOIN ParamTables p on p.object_id=s.object_id 240 | WHERE (@OnlyModifiedStatistics ='Y' and ( 241 | ( s.stats_difference_percent >= @StatisticsModificationLevel and s.stats_difference_percent>0) 242 | OR (abs( [actual_row_count]- [stats_row_count]) >= SQRT(1000 * s.[actual_row_count]) and s.[actual_row_count] > 0 ) 243 | ) 244 | ) 245 | ORDER BY s.[schema_name], s.[table_name] 246 | UNION ALL 247 | SELECT TOP 1000000 s.[object_id], s.[schema_name], s.[table_name], s.stat_name as stats_name, s.[stats_row_count], s.[actual_row_count], s.[stats_difference_percent] 248 | , 'UPDATE STATISTICS ' + quotename(s.[schema_name]) + '.' + quotename(s.[table_name]) + '(' + s.stat_name + ')' 249 | + coalesce(case when coalesce(@StatisticsSample , s.[stats_sample_rate]) =100 THEN ' WITH FULLSCAN' ELSE ' WITH SAMPLE ' + convert(varchar,coalesce(@StatisticsSample , s.[stats_sample_rate])) + ' PERCENT' END,'') 250 | as SqlCommand 251 | , convert(bigint,SQRT(1000 * s.[actual_row_count])) as UpdateLevel 252 | ,'' + convert (varchar, s.stats_row_count) + '' + convert (varchar, s.actual_row_count) + '' + convert (varchar,convert(bigint,SQRT(1000 * s.[actual_row_count]))) + '' + s.stat_columns + '' as ExtendedInfo 253 | FROM dbo.vStats s 254 | INNER JOIN ParamColumns p on p.object_id=s.object_id and (s.stat_columns like p.[Column] + ',%' or s.stat_columns =p.[Column]) 255 | WHERE (@OnlyModifiedStatistics ='Y' and ( 256 | ( s.stats_difference_percent >= @StatisticsModificationLevel and s.stats_difference_percent>0) 257 | OR (abs( [actual_row_count]- [stats_row_count]) >= SQRT(1000 * s.[actual_row_count]) and s.[actual_row_count] > 0 ) 258 | ) 259 | AND (s.user_created=1 OR s.auto_created=1) 260 | ) 261 | ) a 262 | 263 | SELECT TOP 1 @ID =ID FROM #rsStats ORDER BY ID 264 | SET @ProcStartTime =getdate() 265 | SET @Duration =0 266 | WHILE @ID IS NOT NULL AND (@Duration < @TimeLimit OR @TimeLimit is null) 267 | BEGIN 268 | SELECT @LastID =ID 269 | , @object_id=rs.object_id 270 | , @schema_name=rs.schema_name 271 | , @table_name=rs.table_name 272 | , @stats_row_count=rs.stats_row_count 273 | , @actual_row_count = rs.actual_row_count 274 | , @stats_difference_percent=rs.stats_difference_percent 275 | , @SqlCommand =rs.SqlCommand 276 | , @UpdateLevel =rs.UpdateLevel 277 | , @ExtendedInfo =rs.ExtendedInfo 278 | FROM #rsStats rs 279 | WHERE rs.ID=@ID 280 | 281 | SET @EndMessage = 'Date and time: ' + CONVERT(nvarchar,SYSDATETIME(),120) 282 | RAISERROR('%s',10,1,@EndMessage) WITH NOWAIT 283 | SET @StartMessage = 'SqlCommand: ' + @sqlCommand 284 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 285 | 286 | IF @Execute='Y' 287 | BEGIN 288 | SET @StartTime=getdate() 289 | INSERT INTO dbo.CommandLog (DatabaseName, SchemaName, ObjectName, ObjectType, StatisticsName, ExtendedInfo, Command, StartTime, CommandType) 290 | VALUES (@DatabaseName, @schema_name, @table_name, 'U', 'ALL',@ExtendedInfo, @SqlCommand, @StartTime,'UPDATE STATISTICS') 291 | 292 | SELECT TOP 1 @LogID=ID from dbo.CommandLog ORDER BY StartTime desc 293 | 294 | BEGIN TRY 295 | exec (@SqlCommand) 296 | END TRY 297 | BEGIN CATCH 298 | SET @Error = ERROR_NUMBER() 299 | SET @ErrorMessage = ERROR_MESSAGE() 300 | 301 | SET @ErrorMessage = 'Msg ' + CAST(ERROR_NUMBER() AS nvarchar) + ', ' + ISNULL(ERROR_MESSAGE(),'') 302 | UPDATE dbo.CommandLog SET [ErrorNumber]=@Error, ErrorMessage=@ErrorMessage 303 | WHERE ID=@LogID 304 | RAISERROR ('%s',16,1,@ErrorMessage) WITH NOWAIT; 305 | END CATCH 306 | UPDATE dbo.CommandLog SET EndTime=getdate() WHERE ID=@LogID 307 | END 308 | SET @ID=null 309 | SELECT TOP 1 @ID =ID FROM #rsStats WHERE ID > @LastID ORDER BY ID 310 | SET @Duration=datediff(second,@ProcStartTime, getdate()) 311 | END 312 | 313 | /* Delete Overlapping Statistics */ 314 | IF @DeleteOverlappingStats='Y' 315 | BEGIN 316 | TRUNCATE TABLE #rsStats; 317 | ;WITH autostats ( 318 | object_id 319 | ,stats_id 320 | ,name 321 | ,column_id 322 | ) 323 | AS ( 324 | SELECT sys.stats.object_id 325 | ,sys.stats.stats_id 326 | ,sys.stats.name 327 | ,sc.column_id 328 | FROM sys.stats 329 | INNER JOIN sys.stats_columns sc ON sys.stats.object_id = sc.object_id 330 | AND sys.stats.stats_id = sc.stats_id 331 | WHERE sys.stats.auto_created = 1 332 | AND sc.stats_column_id = 1 333 | ) 334 | INSERT INTO #rsStats (object_id, schema_name, table_name, stats_name, ExtendedInfo ,SqlCommand ) 335 | SELECT t.object_id 336 | ,s.name AS schema_name 337 | ,t.name AS table_name 338 | , autostats.name as stats_name 339 | ,'' + c.name + '' + ss.name + '' + '' 340 | ,'DROP STATISTICS [' + OBJECT_SCHEMA_NAME(ss.object_id) + '].[' + OBJECT_NAME(ss.object_id) + '].[' + autostats.name + ']' AS SqlCommand 341 | FROM sys.stats ss 342 | INNER JOIN sys.stats_columns sc ON ss.object_id = sc.object_id 343 | AND ss.stats_id = sc.stats_id 344 | INNER JOIN autostats ON sc.object_id = autostats.object_id 345 | AND sc.column_id = autostats.column_id 346 | INNER JOIN sys.columns c ON ss.object_id = c.object_id 347 | AND sc.column_id = c.column_id 348 | INNER JOIN sys.tables t ON t.object_id = ss.object_id 349 | INNER JOIN sys.schemas s ON s.schema_id = t.schema_id 350 | WHERE ss.auto_created = 0 351 | AND sc.stats_column_id = 1 352 | AND sc.stats_id != autostats.stats_id 353 | AND OBJECTPROPERTY(ss.object_id, 'IsMsShipped') = 0 354 | 355 | SELECT TOP 1 @ID =ID FROM #rsStats ORDER BY ID 356 | SET @ProcStartTime =getdate() 357 | SET @Duration =0 358 | WHILE @ID IS NOT NULL AND (@Duration < @TimeLimit OR @TimeLimit is null) 359 | BEGIN 360 | SELECT @LastID =ID 361 | , @object_id=rs.object_id 362 | , @schema_name=rs.schema_name 363 | , @table_name=rs.table_name 364 | , @SqlCommand =rs.SqlCommand 365 | , @ExtendedInfo =rs.ExtendedInfo 366 | FROM #rsStats rs 367 | WHERE rs.ID=@ID 368 | 369 | SET @EndMessage = 'Date and time: ' + CONVERT(nvarchar,SYSDATETIME(),120) 370 | RAISERROR('%s',10,1,@EndMessage) WITH NOWAIT 371 | SET @StartMessage = 'SqlCommand: ' + @sqlCommand 372 | RAISERROR('%s',10,1,@StartMessage) WITH NOWAIT 373 | RAISERROR('%s',10,1,@ExtendedInfo) WITH NOWAIT 374 | 375 | IF @Execute='Y' 376 | BEGIN 377 | SET @StartTime=getdate() 378 | INSERT INTO dbo.CommandLog (DatabaseName, SchemaName, ObjectName, ObjectType, StatisticsName, ExtendedInfo, Command, StartTime, CommandType) 379 | VALUES (@DatabaseName, @schema_name, @table_name, 'U', 'ALL',@ExtendedInfo, @SqlCommand, @StartTime,'DROP STATISTICS') 380 | 381 | SELECT TOP 1 @LogID=ID from dbo.CommandLog ORDER BY StartTime desc 382 | 383 | BEGIN TRY 384 | exec (@SqlCommand) 385 | END TRY 386 | BEGIN CATCH 387 | SET @Error = ERROR_NUMBER() 388 | SET @ErrorMessage = ERROR_MESSAGE() 389 | 390 | SET @ErrorMessage = 'Msg ' + CAST(ERROR_NUMBER() AS nvarchar) + ', ' + ISNULL(ERROR_MESSAGE(),'') 391 | UPDATE dbo.CommandLog SET [ErrorNumber]=@Error, ErrorMessage=@ErrorMessage 392 | WHERE ID=@LogID 393 | RAISERROR ('%s',16,1,@ErrorMessage) WITH NOWAIT; 394 | END CATCH 395 | UPDATE dbo.CommandLog SET EndTime=getdate() WHERE ID=@LogID 396 | END 397 | SET @ID=null 398 | SELECT TOP 1 @ID =ID FROM #rsStats WHERE ID > @LastID ORDER BY ID 399 | SET @Duration=datediff(second,@ProcStartTime, getdate()) 400 | END 401 | 402 | END 403 | 404 | ---------------------------------------------------------------------------------------------------- 405 | --// Log completing information //-- 406 | ---------------------------------------------------------------------------------------------------- 407 | SET @EndMessage = 'Date and time: ' + CONVERT(nvarchar,SYSDATETIME(),120) 408 | RAISERROR('%s',10,1,@EndMessage) WITH NOWAIT 409 | 410 | RAISERROR(@EmptyLine,10,1) WITH NOWAIT 411 | 412 | END 413 | GO 414 | 415 | -------------------------------------------------------------------------------- /SqlPools/Demos/02.2 Locating Stale stats.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "kernelspec": { 4 | "name": "SQL", 5 | "display_name": "SQL", 6 | "language": "sql" 7 | }, 8 | "language_info": { 9 | "name": "sql", 10 | "version": "" 11 | }, 12 | "extensions": { 13 | "azuredatastudio": { 14 | "version": 1, 15 | "views": [] 16 | } 17 | } 18 | }, 19 | "nbformat_minor": 2, 20 | "nbformat": 4, 21 | "cells": [ 22 | { 23 | "cell_type": "markdown", 24 | "source": [ 25 | "## 2.2 Locating Stale Stats\n", 26 | "\n", 27 | "While SQL pools do have auto create stats they do not have auto update stats like traditional SQL Server, so we really need to maintain statistics ourlseves.\n", 28 | "\n", 29 | "How can stats get out of date and how can we decide which stats are up to date and which need updating ?\n", 30 | "\n", 31 | "### 2.21 Example of Stale Stats\n", 32 | "\n", 33 | "If a table is created with under 1,000 rows then the SQL pol may create stats, but it will \"assume\" 1,000 rows. Even as new data is loaded this stat will become older.\n", 34 | "\n", 35 | "Its especially a problem if we create empty tables or very small tables before adding more data. For example." 36 | ], 37 | "metadata": { 38 | "azdata_cell_guid": "d0672a9e-0456-4600-9f29-fbd560d7daf2", 39 | "extensions": { 40 | "azuredatastudio": { 41 | "views": [] 42 | } 43 | } 44 | }, 45 | "attachments": {} 46 | }, 47 | { 48 | "cell_type": "code", 49 | "source": [ 50 | "IF OBJECT_ID('[dbo].[FactFinance100m_nostats]') is not null \r\n", 51 | "\tDROP TABLE [dbo].[FactFinance100m_nostats]\r\n", 52 | "GO\r\n", 53 | "CREATE TABLE [dbo].[FactFinance100m_nostats] WITH (\r\n", 54 | "\tDISTRIBUTION = ROUND_ROBIN, CLUSTERED COLUMNSTORE INDEX \r\n", 55 | ") AS\r\n", 56 | "SELECT TOP 100 * FROM [dbo].[FactFinance1b] \r\n", 57 | "GO\r\n", 58 | "\r\n", 59 | "SELECT count(distinct DateKey) as DateKey, count(distinct AccountKey) as AccountKey from FactFinance100m_nostats\r\n", 60 | "\r\n", 61 | "/* Prodata sample view to analyse Stats on Tables */\r\n", 62 | "SELECT * FROM dbo.vStats WHERE table_name='FactFinance100m_nostats'" 63 | ], 64 | "metadata": { 65 | "azdata_cell_guid": "01aad1ff-e63b-4b61-8a13-d232549e6adb", 66 | "tags": [], 67 | "language": "sql" 68 | }, 69 | "outputs": [ 70 | { 71 | "output_type": "display_data", 72 | "data": { 73 | "text/html": "Commands completed successfully." 74 | }, 75 | "metadata": {} 76 | }, 77 | { 78 | "output_type": "display_data", 79 | "data": { 80 | "text/html": "(100 rows affected)" 81 | }, 82 | "metadata": {} 83 | }, 84 | { 85 | "output_type": "display_data", 86 | "data": { 87 | "text/html": "(1 row affected)" 88 | }, 89 | "metadata": {} 90 | }, 91 | { 92 | "output_type": "display_data", 93 | "data": { 94 | "text/html": "(3 rows affected)" 95 | }, 96 | "metadata": {} 97 | }, 98 | { 99 | "output_type": "display_data", 100 | "data": { 101 | "text/html": "Total execution time: 00:00:07.197" 102 | }, 103 | "metadata": {} 104 | }, 105 | { 106 | "output_type": "execute_result", 107 | "execution_count": 3, 108 | "data": { 109 | "application/vnd.dataresource+json": { 110 | "schema": { 111 | "fields": [ 112 | { 113 | "name": "DateKey" 114 | }, 115 | { 116 | "name": "AccountKey" 117 | } 118 | ] 119 | }, 120 | "data": [ 121 | { 122 | "0": "1", 123 | "1": "1" 124 | } 125 | ] 126 | }, 127 | "text/html": "
DateKeyAccountKey
11
" 128 | }, 129 | "metadata": {} 130 | }, 131 | { 132 | "output_type": "execute_result", 133 | "execution_count": 3, 134 | "data": { 135 | "application/vnd.dataresource+json": { 136 | "schema": { 137 | "fields": [ 138 | { 139 | "name": "object_id" 140 | }, 141 | { 142 | "name": "stat_name" 143 | }, 144 | { 145 | "name": "table_name" 146 | }, 147 | { 148 | "name": "schema_name" 149 | }, 150 | { 151 | "name": "stats_id" 152 | }, 153 | { 154 | "name": "auto_created" 155 | }, 156 | { 157 | "name": "filter_definition" 158 | }, 159 | { 160 | "name": "last_updated_date" 161 | }, 162 | { 163 | "name": "stat_columns" 164 | }, 165 | { 166 | "name": "stats_row_count" 167 | }, 168 | { 169 | "name": "actual_row_count" 170 | }, 171 | { 172 | "name": "stats_difference_percent" 173 | }, 174 | { 175 | "name": "stats_sample_rate" 176 | }, 177 | { 178 | "name": "sqlCommand" 179 | } 180 | ] 181 | }, 182 | "data": [ 183 | { 184 | "0": "354868381", 185 | "1": "ClusteredIndex_790bfcda58a449a7a8517a169f0e7e0a", 186 | "2": "FactFinance100m_nostats", 187 | "3": "dbo", 188 | "4": "1", 189 | "5": "0", 190 | "6": "NULL", 191 | "7": "NULL", 192 | "8": "DateKey,DepartmentGroupKey,ScenarioKey,OrganizationKey,AccountKey,Amount,Date,LineageKey", 193 | "9": "100", 194 | "10": "100", 195 | "11": "0.00", 196 | "12": "100", 197 | "13": "UPDATE STATISTICS [dbo].[FactFinance100m_nostats] (ClusteredIndex_790bfcda58a449a7a8517a169f0e7e0a) WITH FULLSCAN" 198 | }, 199 | { 200 | "0": "354868381", 201 | "1": "_WA_Sys_00000001_1526DC9D", 202 | "2": "FactFinance100m_nostats", 203 | "3": "dbo", 204 | "4": "2", 205 | "5": "1", 206 | "6": "NULL", 207 | "7": "2022-07-08 15:44:33.740", 208 | "8": "DateKey", 209 | "9": "100", 210 | "10": "100", 211 | "11": "0.00", 212 | "12": "100", 213 | "13": "UPDATE STATISTICS [dbo].[FactFinance100m_nostats] (_WA_Sys_00000001_1526DC9D) WITH FULLSCAN" 214 | }, 215 | { 216 | "0": "354868381", 217 | "1": "_WA_Sys_00000005_1526DC9D", 218 | "2": "FactFinance100m_nostats", 219 | "3": "dbo", 220 | "4": "3", 221 | "5": "1", 222 | "6": "NULL", 223 | "7": "2022-07-08 15:44:34.317", 224 | "8": "AccountKey", 225 | "9": "100", 226 | "10": "100", 227 | "11": "0.00", 228 | "12": "100", 229 | "13": "UPDATE STATISTICS [dbo].[FactFinance100m_nostats] (_WA_Sys_00000005_1526DC9D) WITH FULLSCAN" 230 | } 231 | ] 232 | }, 233 | "text/html": "
object_idstat_nametable_nameschema_namestats_idauto_createdfilter_definitionlast_updated_datestat_columnsstats_row_countactual_row_countstats_difference_percentstats_sample_ratesqlCommand
354868381ClusteredIndex_790bfcda58a449a7a8517a169f0e7e0aFactFinance100m_nostatsdbo10NULLNULLDateKey,DepartmentGroupKey,ScenarioKey,OrganizationKey,AccountKey,Amount,Date,LineageKey1001000.00100UPDATE STATISTICS [dbo].[FactFinance100m_nostats] (ClusteredIndex_790bfcda58a449a7a8517a169f0e7e0a) WITH FULLSCAN
354868381_WA_Sys_00000001_1526DC9DFactFinance100m_nostatsdbo21NULL2022-07-08 15:44:33.740DateKey1001000.00100UPDATE STATISTICS [dbo].[FactFinance100m_nostats] (_WA_Sys_00000001_1526DC9D) WITH FULLSCAN
354868381_WA_Sys_00000005_1526DC9DFactFinance100m_nostatsdbo31NULL2022-07-08 15:44:34.317AccountKey1001000.00100UPDATE STATISTICS [dbo].[FactFinance100m_nostats] (_WA_Sys_00000005_1526DC9D) WITH FULLSCAN
" 234 | }, 235 | "metadata": {} 236 | } 237 | ], 238 | "execution_count": 3 239 | }, 240 | { 241 | "cell_type": "code", 242 | "source": [ 243 | "/* Lets now insert 100 million rows */\r\n", 244 | "INSERT INTO FactFinance100m_nostats ([AccountKey], [ScenarioKey], [DepartmentGroupKey], [DateKey], [OrganizationKey], [Amount], [Date], [LineageKey])\r\n", 245 | "SELECT TOP 100000000 [AccountKey], [ScenarioKey], [DepartmentGroupKey], [DateKey], [OrganizationKey], [Amount], [Date], [LineageKey] \r\n", 246 | "FROM [dbo].[FactFinance1b] " 247 | ], 248 | "metadata": { 249 | "azdata_cell_guid": "4634fd25-21b9-4641-aff9-1c3bfb2d77bb", 250 | "tags": [], 251 | "language": "sql" 252 | }, 253 | "outputs": [ 254 | { 255 | "output_type": "display_data", 256 | "data": { 257 | "text/html": "(100000000 rows affected)" 258 | }, 259 | "metadata": {} 260 | }, 261 | { 262 | "output_type": "display_data", 263 | "data": { 264 | "text/html": "Total execution time: 00:02:08.389" 265 | }, 266 | "metadata": {} 267 | } 268 | ], 269 | "execution_count": 4 270 | }, 271 | { 272 | "cell_type": "code", 273 | "source": [ 274 | "/* \r\n", 275 | " We can see that the stats are not super out of date \r\n", 276 | "\r\n", 277 | "*/\r\n", 278 | "SELECT * FROM dbo.vStats WHERE table_name='FactFinance100m_nostats'" 279 | ], 280 | "metadata": { 281 | "azdata_cell_guid": "1f53181f-37ad-41a6-830e-6accff0a4bc5", 282 | "language": "sql" 283 | }, 284 | "outputs": [ 285 | { 286 | "output_type": "display_data", 287 | "data": { 288 | "text/html": "(3 rows affected)" 289 | }, 290 | "metadata": {} 291 | }, 292 | { 293 | "output_type": "display_data", 294 | "data": { 295 | "text/html": "Total execution time: 00:00:01.416" 296 | }, 297 | "metadata": {} 298 | }, 299 | { 300 | "output_type": "execute_result", 301 | "execution_count": 6, 302 | "data": { 303 | "application/vnd.dataresource+json": { 304 | "schema": { 305 | "fields": [ 306 | { 307 | "name": "object_id" 308 | }, 309 | { 310 | "name": "stat_name" 311 | }, 312 | { 313 | "name": "table_name" 314 | }, 315 | { 316 | "name": "schema_name" 317 | }, 318 | { 319 | "name": "stats_id" 320 | }, 321 | { 322 | "name": "auto_created" 323 | }, 324 | { 325 | "name": "filter_definition" 326 | }, 327 | { 328 | "name": "last_updated_date" 329 | }, 330 | { 331 | "name": "stat_columns" 332 | }, 333 | { 334 | "name": "stats_row_count" 335 | }, 336 | { 337 | "name": "actual_row_count" 338 | }, 339 | { 340 | "name": "stats_difference_percent" 341 | }, 342 | { 343 | "name": "stats_sample_rate" 344 | }, 345 | { 346 | "name": "sqlCommand" 347 | } 348 | ] 349 | }, 350 | "data": [ 351 | { 352 | "0": "354868381", 353 | "1": "ClusteredIndex_790bfcda58a449a7a8517a169f0e7e0a", 354 | "2": "FactFinance100m_nostats", 355 | "3": "dbo", 356 | "4": "1", 357 | "5": "0", 358 | "6": "NULL", 359 | "7": "NULL", 360 | "8": "DateKey,DepartmentGroupKey,ScenarioKey,OrganizationKey,AccountKey,Amount,Date,LineageKey", 361 | "9": "100", 362 | "10": "100000100", 363 | "11": "100.00", 364 | "12": "9", 365 | "13": "UPDATE STATISTICS [dbo].[FactFinance100m_nostats] (ClusteredIndex_790bfcda58a449a7a8517a169f0e7e0a) WITH SAMPLE 9 PERCENT" 366 | }, 367 | { 368 | "0": "354868381", 369 | "1": "_WA_Sys_00000001_1526DC9D", 370 | "2": "FactFinance100m_nostats", 371 | "3": "dbo", 372 | "4": "2", 373 | "5": "1", 374 | "6": "NULL", 375 | "7": "2022-07-08 15:44:33.740", 376 | "8": "DateKey", 377 | "9": "100", 378 | "10": "100000100", 379 | "11": "100.00", 380 | "12": "9", 381 | "13": "UPDATE STATISTICS [dbo].[FactFinance100m_nostats] (_WA_Sys_00000001_1526DC9D) WITH SAMPLE 9 PERCENT" 382 | }, 383 | { 384 | "0": "354868381", 385 | "1": "_WA_Sys_00000005_1526DC9D", 386 | "2": "FactFinance100m_nostats", 387 | "3": "dbo", 388 | "4": "3", 389 | "5": "1", 390 | "6": "NULL", 391 | "7": "2022-07-08 15:44:34.317", 392 | "8": "AccountKey", 393 | "9": "100", 394 | "10": "100000100", 395 | "11": "100.00", 396 | "12": "9", 397 | "13": "UPDATE STATISTICS [dbo].[FactFinance100m_nostats] (_WA_Sys_00000005_1526DC9D) WITH SAMPLE 9 PERCENT" 398 | } 399 | ] 400 | }, 401 | "text/html": "
object_idstat_nametable_nameschema_namestats_idauto_createdfilter_definitionlast_updated_datestat_columnsstats_row_countactual_row_countstats_difference_percentstats_sample_ratesqlCommand
354868381ClusteredIndex_790bfcda58a449a7a8517a169f0e7e0aFactFinance100m_nostatsdbo10NULLNULLDateKey,DepartmentGroupKey,ScenarioKey,OrganizationKey,AccountKey,Amount,Date,LineageKey100100000100100.009UPDATE STATISTICS [dbo].[FactFinance100m_nostats] (ClusteredIndex_790bfcda58a449a7a8517a169f0e7e0a) WITH SAMPLE 9 PERCENT
354868381_WA_Sys_00000001_1526DC9DFactFinance100m_nostatsdbo21NULL2022-07-08 15:44:33.740DateKey100100000100100.009UPDATE STATISTICS [dbo].[FactFinance100m_nostats] (_WA_Sys_00000001_1526DC9D) WITH SAMPLE 9 PERCENT
354868381_WA_Sys_00000005_1526DC9DFactFinance100m_nostatsdbo31NULL2022-07-08 15:44:34.317AccountKey100100000100100.009UPDATE STATISTICS [dbo].[FactFinance100m_nostats] (_WA_Sys_00000005_1526DC9D) WITH SAMPLE 9 PERCENT
" 402 | }, 403 | "metadata": {} 404 | } 405 | ], 406 | "execution_count": 6 407 | }, 408 | { 409 | "cell_type": "markdown", 410 | "source": [ 411 | "Rather than \"per stats\" we track meta data per table due  to the limitations in DMVs in the SqlPool. Sample View provided by Prodata below. This provides some critcal information\n", 412 | "\n", 413 | "- stats\\_row\\_count. This is the number of estimated rows when the stats were last rebuilt. Using DMV pdw\\_table\\_distribution\\_properties  \n", 414 | "- actual\\_row\\_count. This is the actual number of rows using the DMV sys.dm\\_pdw\\_nodes\\_db\\_partition\\_stats\n", 415 | "- dynamic\\_threshold\\_row. recommended threshold to update stats using same algorithm(s) as traditonal SQL Server. The lower of 20% difference or   SQRT(1000 \\* \\[row count\\])." 416 | ], 417 | "metadata": { 418 | "azdata_cell_guid": "1cb98a6a-2d3a-49d4-8a8c-5fb1286dae7d" 419 | }, 420 | "attachments": {} 421 | } 422 | ] 423 | } -------------------------------------------------------------------------------- /SqlPools/Demos/04.0 Links and References.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "kernelspec": { 4 | "name": "SQL", 5 | "display_name": "SQL", 6 | "language": "sql" 7 | }, 8 | "language_info": { 9 | "name": "sql", 10 | "version": "" 11 | }, 12 | "extensions": { 13 | "azuredatastudio": { 14 | "version": 1, 15 | "views": [] 16 | } 17 | } 18 | }, 19 | "nbformat_minor": 2, 20 | "nbformat": 4, 21 | "cells": [ 22 | { 23 | "cell_type": "markdown", 24 | "source": [ 25 | "# Links and References\n", 26 | "\n", 27 | " \n", 28 | "\n", 29 | "Columnstore References\n", 34 | "\n", 35 | "Microsoft Docs on SQL Pool Indexing with sample code on density\n", 40 | "\n", 41 | "https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-tables-index\n", 46 | "\n", 47 | "and sample code for maintenance is here\n", 52 | "\n", 53 | "https://github.com/rgl/azure-content/blob/master/articles/sql-data-warehouse/sql-data-warehouse-manage-columnstore-indexes.md\n", 58 | "\n", 59 | "Improving Columnstore indexes in dedicated SQL Pool\n", 64 | "\n", 65 | "https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-memory-optimizations-for-columnstore-compression\n", 70 | "\n", 71 | "Columnstore Index Defragmentation using REORGANIZE Command\n", 76 | "\n", 77 | "https://docs.microsoft.com/en-gb/archive/blogs/sqlserverstorageengine/columnstore-index-defragmentation-using-reorganize-command\n", 82 | "\n", 83 | "Performance tuning with ordered clustered columnstore index\n", 98 | "\n", 99 | "https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/performance-tuning-ordered-cci\n", 104 | "\n", 105 | "Ordered columns store alignment code\n", 110 | "\n", 111 | "https://github.com/NikoNeugebauer/CISL/blob/master/SQL%20DW/alignment.sql\n", 116 | "\n", 117 | "MSFT - Indexing dedicated SQL pool tables in Azure synapse Analytics\n", 122 | "\n", 123 | "https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-tables-index" 128 | ], 129 | "metadata": { 130 | "azdata_cell_guid": "3b257ac1-361e-4815-a61f-5748da07fd6d" 131 | }, 132 | "attachments": {} 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "source": [ 137 | "Statistics References\n", 143 | "\n", 144 | "Create and update statistics on tables - Azure Synapse Analytics | Microsoft Docs\n", 151 | "\n", 152 | "https://docs.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-tables-statistics\n", 158 | "\n", 159 | "Statistics in Synapse SQL. General best practise and starting point.\n", 166 | "\n", 167 | "https://docs.microsoft.com/en-us/azure/synapse-analytics/sql/develop-tables-statistics\n", 173 | "\n", 174 | "General SQL Statistics (Critical document in understanding update threshold like SQRT (SQRT(1,000 *  n) )\n", 191 | "\n", 192 | "https://docs.microsoft.com/en-us/sql/relational-databases/statistics/statistics?view=sql-server-2017\n", 198 | "\n", 199 | "Ola Hallengren Maintenance Framework. The de-facto industry standard for MSQL maintenance on SMP> while it does not support Synapse SQL, it gives great example of concept, standards and syntax.\n", 206 | "\n", 207 | "https://ola.hallengren.com/\n", 213 | "\n", 214 | "Series by Erin from SqlSkills on basics of Stats maintenance using Ola Hallengren scripts and background on thresholds\n", 221 | "\n", 222 | "https://www.sqlskills.com/blogs/erin/updating-statistics-with-ola-hallengrens-script/\n", 228 | "\n", 229 | "https://www.sqlskills.com/blogs/tim/when-updating-statistics-is-too-expensive/\n", 235 | "\n", 236 | "Extended Sample source code for locating stale rows “per table” based on original MS GitHub. We use this a lot in combination with Ola Hallengren syntax and standards.\n", 243 | "\n", 244 | "https://github.com/abrahams1/Azure_Synapse_Toolbox/tree/master/SQL_Queries/Statistics\n", 250 | "\n", 251 | "Microsoft Samples SQLOPS procedures on GitHub for detecting data skew. The query here forms the basis of our maintenance framework in comparing estimated rows to actual rows within partition stats\n", 258 | "\n", 259 | "https://github.com/Microsoft/sql-data-warehouse-samples/blob/main/samples/sqlops/MonitoringScripts/ImpactedTables" 265 | ], 266 | "metadata": { 267 | "azdata_cell_guid": "8e5ebc8a-8fc6-4038-b0e5-67e60262c6dd" 268 | }, 269 | "attachments": {} 270 | }, 271 | { 272 | "cell_type": "markdown", 273 | "source": [ 274 | "## Blogs and GitHub\n", 275 | "https://prodata.ie/2021/08/28/do-we-need-to-maintain-columnstores-in-sql-pools/\n", 276 | "https://prodata.ie/2021/08/12/finding-outdated-and-missing-stats-in-synapse-sql-pools/\n", 277 | "\n", 278 | "https://github.com/ProdataSQL/SynapseTools/tree/main/SqlPools/Maintenance\n", 279 | "" 280 | ], 281 | "metadata": { 282 | "azdata_cell_guid": "2cb54686-038e-4d3b-a977-adccb5683f58" 283 | }, 284 | "attachments": {} 285 | } 286 | ] 287 | } -------------------------------------------------------------------------------- /SqlPools/Demos/01.06. CS-DeletedDelta.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "kernelspec": { 4 | "name": "SQL", 5 | "display_name": "SQL", 6 | "language": "sql" 7 | }, 8 | "language_info": { 9 | "name": "sql", 10 | "version": "" 11 | } 12 | }, 13 | "nbformat_minor": 2, 14 | "nbformat": 4, 15 | "cells": [ 16 | { 17 | "cell_type": "markdown", 18 | "source": [ 19 | "### Demo - Deleted delta Stores" 20 | ], 21 | "metadata": { 22 | "azdata_cell_guid": "75c0f687-c8b5-4a6e-a0dd-14f7f8da0ce3" 23 | } 24 | }, 25 | { 26 | "cell_type": "code", 27 | "source": [ 28 | "/* \r\n", 29 | " ColumnStores are READONLY, so when we delete (or update rows), it simulates R/W by writing a delete followed by an insert into the delta store, and merging this at query time.\r\n", 30 | "\r\n", 31 | " This demo shows impact of Updatees and how we can see the deleted rows in the delta store.\r\n", 32 | "*/\r\n", 33 | "\r\n", 34 | "\tUPDATE [dbo].[FactFinance100m]\r\n", 35 | "\tSET Amount=Amount+1\r\n", 36 | "\twhere DateKey =20131228 \r\n", 37 | "" 38 | ], 39 | "metadata": { 40 | "azdata_cell_guid": "1b2be4a4-5f13-47c6-9ea9-97b86812c946", 41 | "language": "sql" 42 | }, 43 | "outputs": [ 44 | { 45 | "output_type": "display_data", 46 | "data": { 47 | "text/html": "(64758 rows affected)" 48 | }, 49 | "metadata": {} 50 | }, 51 | { 52 | "output_type": "display_data", 53 | "data": { 54 | "text/html": "Total execution time: 00:00:03.717" 55 | }, 56 | "metadata": {} 57 | } 58 | ], 59 | "execution_count": 1 60 | }, 61 | { 62 | "cell_type": "code", 63 | "source": [ 64 | "/* we can see the impact on the deleted/inserted rows and open Delta Stores */\r\n", 65 | "\r\n", 66 | "SELECT * FROM dbo.vColumnstoreStats WHERE table_name='FactFinance100m'\r\n", 67 | "" 68 | ], 69 | "metadata": { 70 | "azdata_cell_guid": "0519a78c-8593-4919-a1ba-ac1fd8a0b52f", 71 | "language": "sql" 72 | }, 73 | "outputs": [ 74 | { 75 | "output_type": "display_data", 76 | "data": { 77 | "text/html": "(1 row affected)" 78 | }, 79 | "metadata": {} 80 | }, 81 | { 82 | "output_type": "display_data", 83 | "data": { 84 | "text/html": "Total execution time: 00:00:01.221" 85 | }, 86 | "metadata": {} 87 | }, 88 | { 89 | "output_type": "execute_result", 90 | "metadata": {}, 91 | "execution_count": 3, 92 | "data": { 93 | "application/vnd.dataresource+json": { 94 | "schema": { 95 | "fields": [ 96 | { 97 | "name": "execution_date" 98 | }, 99 | { 100 | "name": "database_name" 101 | }, 102 | { 103 | "name": "schema_name" 104 | }, 105 | { 106 | "name": "table_name" 107 | }, 108 | { 109 | "name": "partition_number" 110 | }, 111 | { 112 | "name": "partition_scheme" 113 | }, 114 | { 115 | "name": "object_id" 116 | }, 117 | { 118 | "name": "index_name" 119 | }, 120 | { 121 | "name": "row_count" 122 | }, 123 | { 124 | "name": "deleted_row_count" 125 | }, 126 | { 127 | "name": "row_group_count" 128 | }, 129 | { 130 | "name": "compressed_row_count" 131 | }, 132 | { 133 | "name": "compressed_rowgroup_count" 134 | }, 135 | { 136 | "name": "open_rowgroup_count" 137 | }, 138 | { 139 | "name": "open_row_count" 140 | }, 141 | { 142 | "name": "compressed_row_max" 143 | }, 144 | { 145 | "name": "compressed_row_avg" 146 | }, 147 | { 148 | "name": "fragmentation_density" 149 | }, 150 | { 151 | "name": "fragmentation_deletes" 152 | }, 153 | { 154 | "name": "fragmentation_open" 155 | } 156 | ] 157 | }, 158 | "data": [ 159 | { 160 | "0": "2022-07-08 13:12:32.560", 161 | "1": "AdventureWorksDW", 162 | "2": "dbo", 163 | "3": "FactFinance100m", 164 | "4": "NULL", 165 | "5": "NULL", 166 | "6": "162867697", 167 | "7": "ClusteredIndex_bfe60ded6e5d4ff498d34447e5d3f9bf", 168 | "8": "86064758", 169 | "9": "64758", 170 | "10": "239", 171 | "11": "86000000", 172 | "12": "180", 173 | "13": "59", 174 | "14": "64758", 175 | "15": "1000532", 176 | "16": "477777", 177 | "17": "54.4400", 178 | "18": "0.1100", 179 | "19": "0.1100" 180 | } 181 | ] 182 | }, 183 | "text/html": [ 184 | "", 185 | "", 186 | "", 187 | "
execution_datedatabase_nameschema_nametable_namepartition_numberpartition_schemeobject_idindex_namerow_countdeleted_row_countrow_group_countcompressed_row_countcompressed_rowgroup_countopen_rowgroup_countopen_row_countcompressed_row_maxcompressed_row_avgfragmentation_densityfragmentation_deletesfragmentation_open
2022-07-08 13:12:32.560AdventureWorksDWdboFactFinance100mNULLNULL162867697ClusteredIndex_bfe60ded6e5d4ff498d34447e5d3f9bf8606475864758239860000001805964758100053247777754.44000.11000.1100
" 188 | ] 189 | } 190 | } 191 | ], 192 | "execution_count": 3 193 | }, 194 | { 195 | "cell_type": "markdown", 196 | "source": [ 197 | "A simple REORGANIZE wil not close the open row groups.\n", 198 | "\n", 199 | "While a REORGANIZE WITH (COMPRESS_ALL_ROW_GROUPS = ON) will close the open RowGroup for inserts it will also not take care of deleted row. See below - after REGORG we still have the deleted rows \n", 200 | "" 201 | ], 202 | "metadata": { 203 | "azdata_cell_guid": "2f22701e-ae9a-4027-babd-f5ece18c6208" 204 | }, 205 | "attachments": {} 206 | }, 207 | { 208 | "cell_type": "code", 209 | "source": [ 210 | "exec [dbo].[ColumnstoreOptimize] @Tables='FactFinance100m'\r\n", 211 | ",@DensityThreshold=null\r\n", 212 | ",@OpenThreshold=10000\r\n", 213 | ",@DeleteThreshold=null\r\n", 214 | ",@TimeLimit =null\r\n", 215 | ", @Execute='Y'\r\n", 216 | "\r\n", 217 | "SELECT * FROM dbo.vColumnstoreStats WHERE table_name='FactFinance100m'" 218 | ], 219 | "metadata": { 220 | "azdata_cell_guid": "94a67e55-db37-4367-b56d-b5051e8470fd", 221 | "language": "sql" 222 | }, 223 | "outputs": [ 224 | { 225 | "output_type": "display_data", 226 | "data": { 227 | "text/html": "Date and time: 2022-07-08 13:13:36" 228 | }, 229 | "metadata": {} 230 | }, 231 | { 232 | "output_type": "display_data", 233 | "data": { 234 | "text/html": "Server: aw-dev" 235 | }, 236 | "metadata": {} 237 | }, 238 | { 239 | "output_type": "display_data", 240 | "data": { 241 | "text/html": "Version: 10.0.15665.0" 242 | }, 243 | "metadata": {} 244 | }, 245 | { 246 | "output_type": "display_data", 247 | "data": { 248 | "text/html": "Edition: SQL Azure" 249 | }, 250 | "metadata": {} 251 | }, 252 | { 253 | "output_type": "display_data", 254 | "data": { 255 | "text/html": "Parameters: @Tables= 'FactFinance100m', @DensityThreshold = 25.00, @OpenThreshold = 10000, @DeleteThreshold = 200000, @Execute = Y" 256 | }, 257 | "metadata": {} 258 | }, 259 | { 260 | "output_type": "display_data", 261 | "data": { 262 | "text/html": "Version: Microsoft Azure SQL Data Warehouse - 10.0.15665.0 Jul 7 2022 07:03:56 Copyright (c) Microsoft Corporation" 263 | }, 264 | "metadata": {} 265 | }, 266 | { 267 | "output_type": "display_data", 268 | "data": { 269 | "text/html": "Source: https://github.com/ProdataSQL/SynapseTools/tree/main/SqlPools/Maintenance" 270 | }, 271 | "metadata": {} 272 | }, 273 | { 274 | "output_type": "display_data", 275 | "data": { 276 | "text/html": "Date and time: 2022-07-08 13:13:39" 277 | }, 278 | "metadata": {} 279 | }, 280 | { 281 | "output_type": "display_data", 282 | "data": { 283 | "text/html": "SqlCommand: ALTER INDEX ClusteredIndex_bfe60ded6e5d4ff498d34447e5d3f9bf ON [dbo].[FactFinance100m] REORGANIZE WITH (COMPRESS_ALL_ROW_GROUPS = ON)" 284 | }, 285 | "metadata": {} 286 | }, 287 | { 288 | "output_type": "display_data", 289 | "data": { 290 | "text/html": "(1 row affected)" 291 | }, 292 | "metadata": {} 293 | }, 294 | { 295 | "output_type": "display_data", 296 | "data": { 297 | "text/html": "Total execution time: 00:00:05.651" 298 | }, 299 | "metadata": {} 300 | }, 301 | { 302 | "output_type": "execute_result", 303 | "metadata": {}, 304 | "execution_count": 6, 305 | "data": { 306 | "application/vnd.dataresource+json": { 307 | "schema": { 308 | "fields": [ 309 | { 310 | "name": "execution_date" 311 | }, 312 | { 313 | "name": "database_name" 314 | }, 315 | { 316 | "name": "schema_name" 317 | }, 318 | { 319 | "name": "table_name" 320 | }, 321 | { 322 | "name": "partition_number" 323 | }, 324 | { 325 | "name": "partition_scheme" 326 | }, 327 | { 328 | "name": "object_id" 329 | }, 330 | { 331 | "name": "index_name" 332 | }, 333 | { 334 | "name": "row_count" 335 | }, 336 | { 337 | "name": "deleted_row_count" 338 | }, 339 | { 340 | "name": "row_group_count" 341 | }, 342 | { 343 | "name": "compressed_row_count" 344 | }, 345 | { 346 | "name": "compressed_rowgroup_count" 347 | }, 348 | { 349 | "name": "open_rowgroup_count" 350 | }, 351 | { 352 | "name": "open_row_count" 353 | }, 354 | { 355 | "name": "compressed_row_max" 356 | }, 357 | { 358 | "name": "compressed_row_avg" 359 | }, 360 | { 361 | "name": "fragmentation_density" 362 | }, 363 | { 364 | "name": "fragmentation_deletes" 365 | }, 366 | { 367 | "name": "fragmentation_open" 368 | } 369 | ] 370 | }, 371 | "data": [ 372 | { 373 | "0": "2022-07-08 13:13:40.537", 374 | "1": "AdventureWorksDW", 375 | "2": "dbo", 376 | "3": "FactFinance100m", 377 | "4": "NULL", 378 | "5": "NULL", 379 | "6": "162867697", 380 | "7": "ClusteredIndex_bfe60ded6e5d4ff498d34447e5d3f9bf", 381 | "8": "86044901", 382 | "9": "44901", 383 | "10": "179", 384 | "11": "86044901", 385 | "12": "179", 386 | "13": "0", 387 | "14": "0", 388 | "15": "1000532", 389 | "16": "480697", 390 | "17": "54.1600", 391 | "18": "0.0700", 392 | "19": "0.0000" 393 | } 394 | ] 395 | }, 396 | "text/html": [ 397 | "", 398 | "", 399 | "", 400 | "
execution_datedatabase_nameschema_nametable_namepartition_numberpartition_schemeobject_idindex_namerow_countdeleted_row_countrow_group_countcompressed_row_countcompressed_rowgroup_countopen_rowgroup_countopen_row_countcompressed_row_maxcompressed_row_avgfragmentation_densityfragmentation_deletesfragmentation_open
2022-07-08 13:13:40.537AdventureWorksDWdboFactFinance100mNULLNULL162867697ClusteredIndex_bfe60ded6e5d4ff498d34447e5d3f9bf86044901449011798604490117900100053248069754.16000.07000.0000
" 401 | ] 402 | } 403 | } 404 | ], 405 | "execution_count": 6 406 | }, 407 | { 408 | "cell_type": "markdown", 409 | "source": [ 410 | "Using the ColumnStoreOrganise we can pass a DeleteThrehsold and force a REBUILD, which is an offline operation\n", 411 | "\n", 412 | "(queries blocked until finished)" 413 | ], 414 | "metadata": { 415 | "azdata_cell_guid": "d246c5a9-fdd0-47c1-adc7-bde91bf39b29" 416 | }, 417 | "attachments": {} 418 | }, 419 | { 420 | "cell_type": "code", 421 | "source": [ 422 | "exec [dbo].[ColumnstoreOptimize] @Tables='FactFinance100m'\r\n", 423 | ",@DensityThreshold=null\r\n", 424 | ",@OpenThreshold=null\r\n", 425 | ",@DeleteThreshold=10000\r\n", 426 | ",@TimeLimit =null\r\n", 427 | ",@Execute=null\r\n", 428 | "\r\n", 429 | "SELECT * FROM dbo.vColumnstoreStats WHERE table_name='FactFinance100m'" 430 | ], 431 | "metadata": { 432 | "azdata_cell_guid": "af850ae7-d039-4615-8dae-f75011ca4e3f", 433 | "language": "sql" 434 | }, 435 | "outputs": [ 436 | { 437 | "output_type": "display_data", 438 | "data": { 439 | "text/html": "Date and time: 2022-07-08 13:14:00" 440 | }, 441 | "metadata": {} 442 | }, 443 | { 444 | "output_type": "display_data", 445 | "data": { 446 | "text/html": "Server: aw-dev" 447 | }, 448 | "metadata": {} 449 | }, 450 | { 451 | "output_type": "display_data", 452 | "data": { 453 | "text/html": "Version: 10.0.15665.0" 454 | }, 455 | "metadata": {} 456 | }, 457 | { 458 | "output_type": "display_data", 459 | "data": { 460 | "text/html": "Edition: SQL Azure" 461 | }, 462 | "metadata": {} 463 | }, 464 | { 465 | "output_type": "display_data", 466 | "data": { 467 | "text/html": "Parameters: @Tables= 'FactFinance100m', @DensityThreshold = 25.00, @OpenThreshold = 200000, @DeleteThreshold = 10000, @Execute = Y" 468 | }, 469 | "metadata": {} 470 | }, 471 | { 472 | "output_type": "display_data", 473 | "data": { 474 | "text/html": "Version: Microsoft Azure SQL Data Warehouse - 10.0.15665.0 Jul 7 2022 07:03:56 Copyright (c) Microsoft Corporation" 475 | }, 476 | "metadata": {} 477 | }, 478 | { 479 | "output_type": "display_data", 480 | "data": { 481 | "text/html": "Source: https://github.com/ProdataSQL/SynapseTools/tree/main/SqlPools/Maintenance" 482 | }, 483 | "metadata": {} 484 | }, 485 | { 486 | "output_type": "display_data", 487 | "data": { 488 | "text/html": "Date and time: 2022-07-08 13:14:03" 489 | }, 490 | "metadata": {} 491 | }, 492 | { 493 | "output_type": "display_data", 494 | "data": { 495 | "text/html": "SqlCommand: ALTER INDEX ClusteredIndex_bfe60ded6e5d4ff498d34447e5d3f9bf ON [dbo].[FactFinance100m] REBUILD " 496 | }, 497 | "metadata": {} 498 | }, 499 | { 500 | "output_type": "display_data", 501 | "data": { 502 | "text/html": "(1 row affected)" 503 | }, 504 | "metadata": {} 505 | }, 506 | { 507 | "output_type": "display_data", 508 | "data": { 509 | "text/html": "Total execution time: 00:00:11.818" 510 | }, 511 | "metadata": {} 512 | }, 513 | { 514 | "output_type": "execute_result", 515 | "metadata": {}, 516 | "execution_count": 7, 517 | "data": { 518 | "application/vnd.dataresource+json": { 519 | "schema": { 520 | "fields": [ 521 | { 522 | "name": "execution_date" 523 | }, 524 | { 525 | "name": "database_name" 526 | }, 527 | { 528 | "name": "schema_name" 529 | }, 530 | { 531 | "name": "table_name" 532 | }, 533 | { 534 | "name": "partition_number" 535 | }, 536 | { 537 | "name": "partition_scheme" 538 | }, 539 | { 540 | "name": "object_id" 541 | }, 542 | { 543 | "name": "index_name" 544 | }, 545 | { 546 | "name": "row_count" 547 | }, 548 | { 549 | "name": "deleted_row_count" 550 | }, 551 | { 552 | "name": "row_group_count" 553 | }, 554 | { 555 | "name": "compressed_row_count" 556 | }, 557 | { 558 | "name": "compressed_rowgroup_count" 559 | }, 560 | { 561 | "name": "open_rowgroup_count" 562 | }, 563 | { 564 | "name": "open_row_count" 565 | }, 566 | { 567 | "name": "compressed_row_max" 568 | }, 569 | { 570 | "name": "compressed_row_avg" 571 | }, 572 | { 573 | "name": "fragmentation_density" 574 | }, 575 | { 576 | "name": "fragmentation_deletes" 577 | }, 578 | { 579 | "name": "fragmentation_open" 580 | } 581 | ] 582 | }, 583 | "data": [ 584 | { 585 | "0": "2022-07-08 13:14:10.910", 586 | "1": "AdventureWorksDW", 587 | "2": "dbo", 588 | "3": "FactFinance100m", 589 | "4": "NULL", 590 | "5": "NULL", 591 | "6": "162867697", 592 | "7": "ClusteredIndex_bfe60ded6e5d4ff498d34447e5d3f9bf", 593 | "8": "86000000", 594 | "9": "0", 595 | "10": "180", 596 | "11": "86000000", 597 | "12": "180", 598 | "13": "0", 599 | "14": "0", 600 | "15": "1048576", 601 | "16": "477777", 602 | "17": "54.4400", 603 | "18": "0.0000", 604 | "19": "0.0000" 605 | } 606 | ] 607 | }, 608 | "text/html": [ 609 | "", 610 | "", 611 | "", 612 | "
execution_datedatabase_nameschema_nametable_namepartition_numberpartition_schemeobject_idindex_namerow_countdeleted_row_countrow_group_countcompressed_row_countcompressed_rowgroup_countopen_rowgroup_countopen_row_countcompressed_row_maxcompressed_row_avgfragmentation_densityfragmentation_deletesfragmentation_open
2022-07-08 13:14:10.910AdventureWorksDWdboFactFinance100mNULLNULL162867697ClusteredIndex_bfe60ded6e5d4ff498d34447e5d3f9bf8600000001808600000018000104857647777754.44000.00000.0000
" 613 | ] 614 | } 615 | } 616 | ], 617 | "execution_count": 7 618 | } 619 | ] 620 | } --------------------------------------------------------------------------------